Mirror
/
CyberChef
mirror of https://github.com/gchq/CyberChef.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add Avro to JSON data format conversion

This commit is contained in:
Jarrod Connolly 2019-10-30 22:09:42 -07:00
parent 05e65a74ce
commit 2d12a16771
6 changed files with 155 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
package-lock.json generated
View File

@ -2149,6 +2149,11 @@
}
}
},
"avsc": {
"version": "5.4.16",
"resolved": "https://registry.npmjs.org/avsc/-/avsc-5.4.16.tgz",
"integrity": "sha512-Z85B8ZaEU2PWNPRJYuMSp5Hg7Nw3KPKW47lW/Kus7AcwV7fr6uJG3UckagqIPLydIeO/Cm+yjnJG7g0tliICOg=="
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",

1
package.json
View File

@ -86,6 +86,7 @@
"@babel/polyfill": "^7.4.4",
"@babel/runtime": "^7.5.5",
"arrive": "^2.4.1",
"avsc": "^5.4.16",
"babel-plugin-transform-builtin-extend": "1.1.2",
"bcryptjs": "^2.4.3",
"bignumber.js": "^9.0.0",

3
src/core/config/Categories.json
View File

@ -59,7 +59,8 @@
"From Braille",
"Parse TLV",
"CSV to JSON",
"JSON to CSV"
"JSON to CSV",
"Avro to JSON"
]
},
{

79
src/core/operations/AvroToJSON.mjs Normal file
View File

@ -0,0 +1,79 @@
/**
* @author jarrodconnolly [jarrod@nestedquotes.ca]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation.mjs";
import OperationError from "../errors/OperationError.mjs";
import avro from "avsc";
/**
* Avro to JSON operation
*/
class AvroToJSON extends Operation {
/**
* AvroToJSON constructor
*/
constructor() {
super();
this.name = "Avro to JSON";
this.module = "Avro";
this.description = "Converts Avro encoded data into JSON.";
this.infoURL = "https://avro.apache.org/docs/current/spec.html";
this.inputType = "ArrayBuffer";
this.outputType = "JSON";
this.args = [{
name: "Force Valid JSON",
type: "boolean",
value: true
}];
}
/**
* @param {ArrayBuffer} input
* @param {Object[]} args
* @returns {JSON}
*/
run(input, args) {
const self = this;
if (input.byteLength <= 0) {
throw new OperationError("Please provide an input.");
}
const forceJSON = args[0];
return new Promise((resolve, reject) => {
const result = [];
const inpArray = new Uint8Array(input);
const decoder = new avro.streams.BlockDecoder();
decoder
.on("data", function (obj) {
result.push(obj);
})
.on("error", function () {
reject(new OperationError("Error parsing Avro file."));
})
.on("end", function () {
if (forceJSON) {
self.presentType = "JSON";
self.outputType = "JSON";
resolve(result.length === 1 ? result[0] : result);
} else {
self.presentType = "string";
self.outputType = "string";
const data = result.reduce((result, current) => result + JSON.stringify(current) + "\n", "");
resolve(data);
}
});
decoder.write(inpArray);
decoder.end();
});
}
}
export default AvroToJSON;

1
tests/operations/index.mjs
View File

@ -91,6 +91,7 @@ import "./tests/Protobuf.mjs";
import "./tests/ParseSSHHostKey.mjs";
import "./tests/DefangIP.mjs";
import "./tests/ParseUDP.mjs";
import "./tests/AvroToJSON";
// Cannot test operations that use the File type yet
// import "./tests/SplitColourChannels.mjs";

67
tests/operations/tests/AvroToJSON.mjs Normal file
View File

@ -0,0 +1,67 @@
/**
*
* Avro to JSON tests.
*
* @author jarrodconnolly [jarrod@nestedquotes.ca]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import TestRegister from "../../lib/TestRegister";
TestRegister.addTests([
{
name: "Avro to JSON: no input (force JSON true)",
input: "",
expectedOutput: "Please provide an input.",
recipeConfig: [
{
op: "Avro to JSON",
args: [true]
}
],
},
{
name: "Avro to JSON: no input (force JSON false)",
input: "",
expectedOutput: "Please provide an input.",
recipeConfig: [
{
op: "Avro to JSON",
args: [false]
}
],
},
{
name: "Avro to JSON: small (force JSON true)",
input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" +
"\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" +
"\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" +
"\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" +
"\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" +
"\x15\x41",
expectedOutput: "{\n \"name\": \"myname\"\n}",
recipeConfig: [
{
op: "Avro to JSON",
args: [true]
}
],
},
{
name: "Avro to JSON: small (force JSON false)",
input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" +
"\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" +
"\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" +
"\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" +
"\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" +
"\x15\x41",
expectedOutput: "{\"name\":\"myname\"}\n",
recipeConfig: [
{
op: "Avro to JSON",
args: [false]
}
],
}
]);