From 2d12a167710d019635c001d5328fef8c3d9263cb Mon Sep 17 00:00:00 2001 From: Jarrod Connolly Date: Wed, 30 Oct 2019 22:09:42 -0700 Subject: [PATCH 1/2] Add Avro to JSON data format conversion --- package-lock.json | 5 ++ package.json | 1 + src/core/config/Categories.json | 3 +- src/core/operations/AvroToJSON.mjs | 79 +++++++++++++++++++++++++++ tests/operations/index.mjs | 1 + tests/operations/tests/AvroToJSON.mjs | 67 +++++++++++++++++++++++ 6 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/AvroToJSON.mjs create mode 100644 tests/operations/tests/AvroToJSON.mjs diff --git a/package-lock.json b/package-lock.json index b3c6f415..45f1afd7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2149,6 +2149,11 @@ } } }, + "avsc": { + "version": "5.4.16", + "resolved": "https://registry.npmjs.org/avsc/-/avsc-5.4.16.tgz", + "integrity": "sha512-Z85B8ZaEU2PWNPRJYuMSp5Hg7Nw3KPKW47lW/Kus7AcwV7fr6uJG3UckagqIPLydIeO/Cm+yjnJG7g0tliICOg==" + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", diff --git a/package.json b/package.json index 518fb29b..84ba3bbd 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "@babel/polyfill": "^7.4.4", "@babel/runtime": "^7.5.5", "arrive": "^2.4.1", + "avsc": "^5.4.16", "babel-plugin-transform-builtin-extend": "1.1.2", "bcryptjs": "^2.4.3", "bignumber.js": "^9.0.0", diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index db2ab3a6..19b702d0 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -59,7 +59,8 @@ "From Braille", "Parse TLV", "CSV to JSON", - "JSON to CSV" + "JSON to CSV", + "Avro to JSON" ] }, { diff --git a/src/core/operations/AvroToJSON.mjs b/src/core/operations/AvroToJSON.mjs new file mode 100644 index 00000000..ee2b81a9 --- /dev/null +++ b/src/core/operations/AvroToJSON.mjs @@ -0,0 +1,79 @@ +/** + * @author jarrodconnolly [jarrod@nestedquotes.ca] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import avro from "avsc"; + +/** + * Avro to JSON operation + */ +class AvroToJSON extends Operation { + + /** + * AvroToJSON constructor + */ + constructor() { + super(); + + this.name = "Avro to JSON"; + this.module = "Avro"; + this.description = "Converts Avro encoded data into JSON."; + this.infoURL = "https://avro.apache.org/docs/current/spec.html"; + this.inputType = "ArrayBuffer"; + this.outputType = "JSON"; + this.args = [{ + name: "Force Valid JSON", + type: "boolean", + value: true + }]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {JSON} + */ + run(input, args) { + const self = this; + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); + } + + const forceJSON = args[0]; + + return new Promise((resolve, reject) => { + const result = []; + const inpArray = new Uint8Array(input); + const decoder = new avro.streams.BlockDecoder(); + + decoder + .on("data", function (obj) { + result.push(obj); + }) + .on("error", function () { + reject(new OperationError("Error parsing Avro file.")); + }) + .on("end", function () { + if (forceJSON) { + self.presentType = "JSON"; + self.outputType = "JSON"; + resolve(result.length === 1 ? result[0] : result); + } else { + self.presentType = "string"; + self.outputType = "string"; + const data = result.reduce((result, current) => result + JSON.stringify(current) + "\n", ""); + resolve(data); + } + }); + + decoder.write(inpArray); + decoder.end(); + }); + } +} + +export default AvroToJSON; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index d64a7737..a1503814 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -91,6 +91,7 @@ import "./tests/Protobuf.mjs"; import "./tests/ParseSSHHostKey.mjs"; import "./tests/DefangIP.mjs"; import "./tests/ParseUDP.mjs"; +import "./tests/AvroToJSON"; // Cannot test operations that use the File type yet // import "./tests/SplitColourChannels.mjs"; diff --git a/tests/operations/tests/AvroToJSON.mjs b/tests/operations/tests/AvroToJSON.mjs new file mode 100644 index 00000000..b6e763ad --- /dev/null +++ b/tests/operations/tests/AvroToJSON.mjs @@ -0,0 +1,67 @@ +/** + * + * Avro to JSON tests. + * + * @author jarrodconnolly [jarrod@nestedquotes.ca] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import TestRegister from "../../lib/TestRegister"; + +TestRegister.addTests([ + { + name: "Avro to JSON: no input (force JSON true)", + input: "", + expectedOutput: "Please provide an input.", + recipeConfig: [ + { + op: "Avro to JSON", + args: [true] + } + ], + }, + { + name: "Avro to JSON: no input (force JSON false)", + input: "", + expectedOutput: "Please provide an input.", + recipeConfig: [ + { + op: "Avro to JSON", + args: [false] + } + ], + }, + { + name: "Avro to JSON: small (force JSON true)", + input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" + + "\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" + + "\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" + + "\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" + + "\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" + + "\x15\x41", + expectedOutput: "{\n \"name\": \"myname\"\n}", + recipeConfig: [ + { + op: "Avro to JSON", + args: [true] + } + ], + }, + { + name: "Avro to JSON: small (force JSON false)", + input: "\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x01\x7b\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65" + + "\x63\x6f\x72\x64\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x73\x6d\x61\x6c\x6c\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a" + + "\x5b\x7b\x22\x6e\x61\x6d\x65\x22\x3a\x22\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67" + + "\x22\x7d\x5d\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7" + + "\x5c\xda\xb9\xa6\x2f\x15\x41\x02\x0e\x0c\x6d\x79\x6e\x61\x6d\x65\x4e\x02\x47\x63\x2e\x37\x02\xe5\xb7\x5c\xda\xb9\xa6\x2f" + + "\x15\x41", + expectedOutput: "{\"name\":\"myname\"}\n", + recipeConfig: [ + { + op: "Avro to JSON", + args: [false] + } + ], + } +]); From daad633195fc7f9494f8dacb665f8b7a5dbcecf1 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 31 Oct 2019 14:17:07 +0000 Subject: [PATCH 2/2] Tidied up Avro to JSON operation --- src/core/operations/AvroToJSON.mjs | 27 +++++++++++-------------- src/core/operations/BSONDeserialise.mjs | 2 +- src/core/operations/BSONSerialise.mjs | 2 +- tests/operations/index.mjs | 2 +- tests/operations/tests/AvroToJSON.mjs | 1 - 5 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/core/operations/AvroToJSON.mjs b/src/core/operations/AvroToJSON.mjs index ee2b81a9..497a3872 100644 --- a/src/core/operations/AvroToJSON.mjs +++ b/src/core/operations/AvroToJSON.mjs @@ -20,25 +20,26 @@ class AvroToJSON extends Operation { super(); this.name = "Avro to JSON"; - this.module = "Avro"; + this.module = "Serialise"; this.description = "Converts Avro encoded data into JSON."; - this.infoURL = "https://avro.apache.org/docs/current/spec.html"; + this.infoURL = "https://wikipedia.org/wiki/Apache_Avro"; this.inputType = "ArrayBuffer"; - this.outputType = "JSON"; - this.args = [{ - name: "Force Valid JSON", - type: "boolean", - value: true - }]; + this.outputType = "string"; + this.args = [ + { + name: "Force Valid JSON", + type: "boolean", + value: true + } + ]; } /** * @param {ArrayBuffer} input * @param {Object[]} args - * @returns {JSON} + * @returns {string} */ run(input, args) { - const self = this; if (input.byteLength <= 0) { throw new OperationError("Please provide an input."); } @@ -59,12 +60,8 @@ class AvroToJSON extends Operation { }) .on("end", function () { if (forceJSON) { - self.presentType = "JSON"; - self.outputType = "JSON"; - resolve(result.length === 1 ? result[0] : result); + resolve(result.length === 1 ? JSON.stringify(result[0], null, 4) : JSON.stringify(result, null, 4)); } else { - self.presentType = "string"; - self.outputType = "string"; const data = result.reduce((result, current) => result + JSON.stringify(current) + "\n", ""); resolve(data); } diff --git a/src/core/operations/BSONDeserialise.mjs b/src/core/operations/BSONDeserialise.mjs index a21eaadd..cb46b357 100644 --- a/src/core/operations/BSONDeserialise.mjs +++ b/src/core/operations/BSONDeserialise.mjs @@ -20,7 +20,7 @@ class BSONDeserialise extends Operation { super(); this.name = "BSON deserialise"; - this.module = "BSON"; + this.module = "Serialise"; this.description = "BSON is a computer data interchange format used mainly as a data storage and network transfer format in the MongoDB database. It is a binary form for representing simple data structures, associative arrays (called objects or documents in MongoDB), and various data types of specific interest to MongoDB. The name 'BSON' is based on the term JSON and stands for 'Binary JSON'.

Input data should be in a raw bytes format."; this.infoURL = "https://wikipedia.org/wiki/BSON"; this.inputType = "ArrayBuffer"; diff --git a/src/core/operations/BSONSerialise.mjs b/src/core/operations/BSONSerialise.mjs index 6d33c6be..25eed876 100644 --- a/src/core/operations/BSONSerialise.mjs +++ b/src/core/operations/BSONSerialise.mjs @@ -20,7 +20,7 @@ class BSONSerialise extends Operation { super(); this.name = "BSON serialise"; - this.module = "BSON"; + this.module = "Serialise"; this.description = "BSON is a computer data interchange format used mainly as a data storage and network transfer format in the MongoDB database. It is a binary form for representing simple data structures, associative arrays (called objects or documents in MongoDB), and various data types of specific interest to MongoDB. The name 'BSON' is based on the term JSON and stands for 'Binary JSON'.

Input data should be valid JSON."; this.infoURL = "https://wikipedia.org/wiki/BSON"; this.inputType = "string"; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index a1503814..71c4432c 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -91,7 +91,7 @@ import "./tests/Protobuf.mjs"; import "./tests/ParseSSHHostKey.mjs"; import "./tests/DefangIP.mjs"; import "./tests/ParseUDP.mjs"; -import "./tests/AvroToJSON"; +import "./tests/AvroToJSON.mjs"; // Cannot test operations that use the File type yet // import "./tests/SplitColourChannels.mjs"; diff --git a/tests/operations/tests/AvroToJSON.mjs b/tests/operations/tests/AvroToJSON.mjs index b6e763ad..045abddb 100644 --- a/tests/operations/tests/AvroToJSON.mjs +++ b/tests/operations/tests/AvroToJSON.mjs @@ -1,5 +1,4 @@ /** - * * Avro to JSON tests. * * @author jarrodconnolly [jarrod@nestedquotes.ca]