From 2d12a167710d019635c001d5328fef8c3d9263cb Mon Sep 17 00:00:00 2001 From: Jarrod Connolly Date: Wed, 30 Oct 2019 22:09:42 -0700 Subject: [PATCH] Add Avro to JSON data format conversion --- package-lock.json | 5 ++ package.json | 1 + src/core/config/Categories.json | 3 +- src/core/operations/AvroToJSON.mjs | 79 +++++++++++++++++++++++++++ tests/operations/index.mjs | 1 + tests/operations/tests/AvroToJSON.mjs | 67 +++++++++++++++++++++++ 6 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/AvroToJSON.mjs create mode 100644 tests/operations/tests/AvroToJSON.mjs diff --git a/package-lock.json b/package-lock.json index b3c6f415..45f1afd7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2149,6 +2149,11 @@ } } }, + "avsc": { + "version": "5.4.16", + "resolved": "https://registry.npmjs.org/avsc/-/avsc-5.4.16.tgz", + "integrity": "sha512-Z85B8ZaEU2PWNPRJYuMSp5Hg7Nw3KPKW47lW/Kus7AcwV7fr6uJG3UckagqIPLydIeO/Cm+yjnJG7g0tliICOg==" + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", diff --git a/package.json b/package.json index 518fb29b..84ba3bbd 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "@babel/polyfill": "^7.4.4", "@babel/runtime": "^7.5.5", "arrive": "^2.4.1", + "avsc": "^5.4.16", "babel-plugin-transform-builtin-extend": "1.1.2", "bcryptjs": "^2.4.3", "bignumber.js": "^9.0.0", diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index db2ab3a6..19b702d0 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -59,7 +59,8 @@ "From Braille", "Parse TLV", "CSV to JSON", - "JSON to CSV" + "JSON to CSV", + "Avro to JSON" ] }, { diff --git a/src/core/operations/AvroToJSON.mjs b/src/core/operations/AvroToJSON.mjs new file mode 100644 index 00000000..ee2b81a9 --- /dev/null +++ b/src/core/operations/AvroToJSON.mjs @@ -0,0 +1,79 @@ +/** + * @author jarrodconnolly [jarrod@nestedquotes.ca] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import avro from "avsc"; + +/** + * Avro to JSON operation + */ +class AvroToJSON extends Operation { + + /** + * AvroToJSON constructor + */ + constructor() { + super(); + + this.name = "Avro to JSON"; + this.module = "Avro"; + this.description = "Converts Avro encoded data into JSON."; + this.infoURL = "https://avro.apache.org/docs/current/spec.html"; + this.inputType = "ArrayBuffer"; + this.outputType = "JSON"; + this.args = [{ + name: "Force Valid JSON", + type: "boolean", + value: true + }]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {JSON} + */ + run(input, args) { + const self = this; + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); + } + + const forceJSON = args[0]; + + return new Promise((resolve, reject) => { + const result = []; + const inpArray = new Uint8Array(input); + const decoder = new avro.streams.BlockDecoder(); + + decoder + .on("data", function (obj) { + result.push(obj); + }) + .on("error", function () { + reject(new OperationError("Error parsing Avro file.")); + }) + .on("end", function () { + if (forceJSON) { + self.presentType = "JSON"; + self.outputType = "JSON"; + resolve(result.length === 1 ? result[0] : result); + } else { + self.presentType = "string"; + self.outputType = "string"; + const data = result.reduce((result, current) => result + JSON.stringify(current) + "\n", ""); + resolve(data); + } + }); + + decoder.write(inpArray); + decoder.end(); + }); + } +} + +export default AvroToJSON; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index d64a7737..a1503814 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -91,6 +91,7 @@ import "./tests/Protobuf.mjs"; import "./tests/ParseSSHHostKey.mjs"; import "./tests/DefangIP.mjs"; import "./tests/ParseUDP.mjs"; +import "./tests/AvroToJSON"; // Cannot test operations that use the File type yet // import "./tests/SplitColourChannels.mjs"; diff --git a/tests/operations/tests/AvroToJSON.mjs b/tests/operations/tests/AvroToJSON.mjs new file mode 100644 index 00000000..b6e763ad --- /dev/null +++ b/tests/operations/tests/AvroToJSON.mjs @@ -0,0 +1,67 @@ +/** + * + * Avro to JSON tests. + * + * @author jarrodconnolly [jarrod@nestedquotes.ca] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import TestRegister from "../../lib/TestRegister"; + +TestRegister.addTests([ + { + name: "Avro to JSON: no input (force JSON true)", + input: "", + expectedOutput: "Please provide an input.", + recipeConfig: [ + { + op: "Avro to JSON", + args: [true] + } + ], + }, + { + name: "Avro to JSON: no input (force JSON false)", + input: "", + expectedOutput: "Please provide an input.", + recipeConfig: [ + { + op: "Avro to JSON", + args: [false] + } + ], + }, + { + name: "Avro to JSON: small (force JSON true)", + input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" + + "\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" + + "\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" + + "\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" + + "\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" + + "\x15\x41", + expectedOutput: "{\n \"name\": \"myname\"\n}", + recipeConfig: [ + { + op: "Avro to JSON", + args: [true] + } + ], + }, + { + name: "Avro to JSON: small (force JSON false)", + input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" + + "\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" + + "\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" + + "\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" + + "\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" + + "\x15\x41", + expectedOutput: "{\"name\":\"myname\"}\n", + recipeConfig: [ + { + op: "Avro to JSON", + args: [false] + } + ], + } +]);