mirror of
https://github.com/gchq/CyberChef.git
synced 2024-11-02 14:11:02 +01:00
Merge branch 'jarrodconnolly-avro-to-json'
This commit is contained in:
n1474335
commit
4c4d7b5d26
5
package-lock.json
generated
5
package-lock.json
generated
@ -2149,6 +2149,11 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"avsc": {
|
||||
"version": "5.4.16",
|
||||
"resolved": "https://registry.npmjs.org/avsc/-/avsc-5.4.16.tgz",
|
||||
"integrity": "sha512-Z85B8ZaEU2PWNPRJYuMSp5Hg7Nw3KPKW47lW/Kus7AcwV7fr6uJG3UckagqIPLydIeO/Cm+yjnJG7g0tliICOg=="
|
||||
},
|
||||
"aws-sign2": {
|
||||
"version": "0.7.0",
|
||||
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
|
||||
|
||||
@ -86,6 +86,7 @@
|
||||
"@babel/polyfill": "^7.4.4",
|
||||
"@babel/runtime": "^7.5.5",
|
||||
"arrive": "^2.4.1",
|
||||
"avsc": "^5.4.16",
|
||||
"babel-plugin-transform-builtin-extend": "1.1.2",
|
||||
"bcryptjs": "^2.4.3",
|
||||
"bignumber.js": "^9.0.0",
|
||||
|
||||
@ -59,7 +59,8 @@
|
||||
"From Braille",
|
||||
"Parse TLV",
|
||||
"CSV to JSON",
|
||||
"JSON to CSV"
|
||||
"JSON to CSV",
|
||||
"Avro to JSON"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
76
src/core/operations/AvroToJSON.mjs
Normal file
76
src/core/operations/AvroToJSON.mjs
Normal file
@ -0,0 +1,76 @@
|
||||
/**
|
||||
* @author jarrodconnolly [jarrod@nestedquotes.ca]
|
||||
* @copyright Crown Copyright 2019
|
||||
* @license Apache-2.0
|
||||
*/
|
||||
|
||||
import Operation from "../Operation.mjs";
|
||||
import OperationError from "../errors/OperationError.mjs";
|
||||
import avro from "avsc";
|
||||
|
||||
/**
|
||||
* Avro to JSON operation
|
||||
*/
|
||||
class AvroToJSON extends Operation {
|
||||
|
||||
/**
|
||||
* AvroToJSON constructor
|
||||
*/
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this.name = "Avro to JSON";
|
||||
this.module = "Serialise";
|
||||
this.description = "Converts Avro encoded data into JSON.";
|
||||
this.infoURL = "https://wikipedia.org/wiki/Apache_Avro";
|
||||
this.inputType = "ArrayBuffer";
|
||||
this.outputType = "string";
|
||||
this.args = [
|
||||
{
|
||||
name: "Force Valid JSON",
|
||||
type: "boolean",
|
||||
value: true
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {ArrayBuffer} input
|
||||
* @param {Object[]} args
|
||||
* @returns {string}
|
||||
*/
|
||||
run(input, args) {
|
||||
if (input.byteLength <= 0) {
|
||||
throw new OperationError("Please provide an input.");
|
||||
}
|
||||
|
||||
const forceJSON = args[0];
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const result = [];
|
||||
const inpArray = new Uint8Array(input);
|
||||
const decoder = new avro.streams.BlockDecoder();
|
||||
|
||||
decoder
|
||||
.on("data", function (obj) {
|
||||
result.push(obj);
|
||||
})
|
||||
.on("error", function () {
|
||||
reject(new OperationError("Error parsing Avro file."));
|
||||
})
|
||||
.on("end", function () {
|
||||
if (forceJSON) {
|
||||
resolve(result.length === 1 ? JSON.stringify(result[0], null, 4) : JSON.stringify(result, null, 4));
|
||||
} else {
|
||||
const data = result.reduce((result, current) => result + JSON.stringify(current) + "\n", "");
|
||||
resolve(data);
|
||||
}
|
||||
});
|
||||
|
||||
decoder.write(inpArray);
|
||||
decoder.end();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export default AvroToJSON;
|
||||
@ -20,7 +20,7 @@ class BSONDeserialise extends Operation {
|
||||
super();
|
||||
|
||||
this.name = "BSON deserialise";
|
||||
this.module = "BSON";
|
||||
this.module = "Serialise";
|
||||
this.description = "BSON is a computer data interchange format used mainly as a data storage and network transfer format in the MongoDB database. It is a binary form for representing simple data structures, associative arrays (called objects or documents in MongoDB), and various data types of specific interest to MongoDB. The name 'BSON' is based on the term JSON and stands for 'Binary JSON'.<br><br>Input data should be in a raw bytes format.";
|
||||
this.infoURL = "https://wikipedia.org/wiki/BSON";
|
||||
this.inputType = "ArrayBuffer";
|
||||
|
||||
@ -20,7 +20,7 @@ class BSONSerialise extends Operation {
|
||||
super();
|
||||
|
||||
this.name = "BSON serialise";
|
||||
this.module = "BSON";
|
||||
this.module = "Serialise";
|
||||
this.description = "BSON is a computer data interchange format used mainly as a data storage and network transfer format in the MongoDB database. It is a binary form for representing simple data structures, associative arrays (called objects or documents in MongoDB), and various data types of specific interest to MongoDB. The name 'BSON' is based on the term JSON and stands for 'Binary JSON'.<br><br>Input data should be valid JSON.";
|
||||
this.infoURL = "https://wikipedia.org/wiki/BSON";
|
||||
this.inputType = "string";
|
||||
|
||||
@ -91,6 +91,7 @@ import "./tests/Protobuf.mjs";
|
||||
import "./tests/ParseSSHHostKey.mjs";
|
||||
import "./tests/DefangIP.mjs";
|
||||
import "./tests/ParseUDP.mjs";
|
||||
import "./tests/AvroToJSON.mjs";
|
||||
|
||||
// Cannot test operations that use the File type yet
|
||||
// import "./tests/SplitColourChannels.mjs";
|
||||
|
||||
66
tests/operations/tests/AvroToJSON.mjs
Normal file
66
tests/operations/tests/AvroToJSON.mjs
Normal file
@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Avro to JSON tests.
|
||||
*
|
||||
* @author jarrodconnolly [jarrod@nestedquotes.ca]
|
||||
* @copyright Crown Copyright 2019
|
||||
* @license Apache-2.0
|
||||
*/
|
||||
|
||||
import TestRegister from "../../lib/TestRegister";
|
||||
|
||||
TestRegister.addTests([
|
||||
{
|
||||
name: "Avro to JSON: no input (force JSON true)",
|
||||
input: "",
|
||||
expectedOutput: "Please provide an input.",
|
||||
recipeConfig: [
|
||||
{
|
||||
op: "Avro to JSON",
|
||||
args: [true]
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "Avro to JSON: no input (force JSON false)",
|
||||
input: "",
|
||||
expectedOutput: "Please provide an input.",
|
||||
recipeConfig: [
|
||||
{
|
||||
op: "Avro to JSON",
|
||||
args: [false]
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "Avro to JSON: small (force JSON true)",
|
||||
input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" +
|
||||
"\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" +
|
||||
"\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" +
|
||||
"\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" +
|
||||
"\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" +
|
||||
"\x15\x41",
|
||||
expectedOutput: "{\n \"name\": \"myname\"\n}",
|
||||
recipeConfig: [
|
||||
{
|
||||
op: "Avro to JSON",
|
||||
args: [true]
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "Avro to JSON: small (force JSON false)",
|
||||
input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" +
|
||||
"\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" +
|
||||
"\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" +
|
||||
"\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" +
|
||||
"\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" +
|
||||
"\x15\x41",
|
||||
expectedOutput: "{\"name\":\"myname\"}\n",
|
||||
recipeConfig: [
|
||||
{
|
||||
op: "Avro to JSON",
|
||||
args: [false]
|
||||
}
|
||||
],
|
||||
}
|
||||
]);
|
||||
Loading…
Reference in New Issue
Block a user