From 1ec7033d46d04c168a1b8d70fd27bb637f86f95d Mon Sep 17 00:00:00 2001 From: Matt C Date: Mon, 19 Sep 2022 14:05:13 +0100 Subject: [PATCH 1/3] Add LZMA Compress operation --- package-lock.json | 14 +++++++ package.json | 1 + src/core/config/Categories.json | 3 +- src/core/operations/LZMACompress.mjs | 62 ++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/LZMACompress.mjs diff --git a/package-lock.json b/package-lock.json index e6b2082e..45d18e0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "license": "Apache-2.0", "dependencies": { "@babel/polyfill": "^7.12.1", + "@blu3r4y/lzma": "^2.3.3", "arrive": "^2.4.1", "avsc": "^5.7.4", "bcryptjs": "^2.4.3", @@ -1771,6 +1772,14 @@ "node": ">=6.9.0" } }, + "node_modules/@blu3r4y/lzma": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/@blu3r4y/lzma/-/lzma-2.3.3.tgz", + "integrity": "sha512-2ckRSsYewLAgq/s8tUW3o5gurtCNYga1f9l0egV4QlT8hgVEilQHRt18s+behmPL2M/BPBxUINaOz67u++r0wA==", + "bin": { + "lzma.js": "bin/lzma.js" + } + }, "node_modules/@colors/colors": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", @@ -15446,6 +15455,11 @@ "to-fast-properties": "^2.0.0" } }, + "@blu3r4y/lzma": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/@blu3r4y/lzma/-/lzma-2.3.3.tgz", + "integrity": "sha512-2ckRSsYewLAgq/s8tUW3o5gurtCNYga1f9l0egV4QlT8hgVEilQHRt18s+behmPL2M/BPBxUINaOz67u++r0wA==" + }, "@colors/colors": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", diff --git a/package.json b/package.json index a0aa75c1..3c847422 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ }, "dependencies": { "@babel/polyfill": "^7.12.1", + "@blu3r4y/lzma": "^2.3.3", "arrive": "^2.4.1", "avsc": "^5.7.4", "bcryptjs": "^2.4.3", diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 8ac60048..24963c99 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -331,7 +331,8 @@ "Tar", "Untar", "LZString Compress", - "LZString Decompress" + "LZString Decompress", + "LZMA Compress" ] }, { diff --git a/src/core/operations/LZMACompress.mjs b/src/core/operations/LZMACompress.mjs new file mode 100644 index 00000000..5c038786 --- /dev/null +++ b/src/core/operations/LZMACompress.mjs @@ -0,0 +1,62 @@ +/** + * @author Matt C [me@mitt.dev] + * @copyright Crown Copyright 2022 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; + +import { compress } from "@blu3r4y/lzma"; + +/** + * LZMA Compress operation + */ +class LZMACompress extends Operation { + + /** + * LZMACompress constructor + */ + constructor() { + super(); + + this.name = "LZMA Compress"; + this.module = "Compression"; + this.description = "Compresses data using the Lempel\u2013Ziv\u2013Markov chain algorithm. Compression mode determines the speed and effectiveness of the compression: 1 is fastest and less effective, 9 is slowest and most effective"; + this.infoURL = "https://wikipedia.org/wiki/Lempel%E2%80%93Ziv%E2%80%93Markov_chain_algorithm"; + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + this.args = [ + { + name: "Compression Mode", + type: "option", + value: [ + "1", "2", "3", "4", "5", "6", "7", "8", "9" + ], + "defaultIndex": 6 + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {ArrayBuffer} + */ + run(input, args) { + return new Promise((resolve, reject) => { + compress(new Uint8Array(input), Number(args[0]), (result, error) => { + if (error) { + reject(new OperationError(`Failed to compress input: ${error.message}`)); + } + // The compression returns as an Int8Array, but we can just get the unsigned data from the buffer + resolve(new Int8Array(result).buffer); + }, (percent) => { + self.sendStatusMessage(`Compressing input: ${(percent*100).toFixed(2)}%`); + }); + }); + } + +} + +export default LZMACompress; From d502dd9857f3880e483200bf46bcb01b7ad63dd5 Mon Sep 17 00:00:00 2001 From: Matt C Date: Mon, 19 Sep 2022 14:20:27 +0100 Subject: [PATCH 2/3] Add LZMA Decompress operation --- src/core/config/Categories.json | 3 +- src/core/operations/LZMACompress.mjs | 3 +- src/core/operations/LZMADecompress.mjs | 51 ++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 src/core/operations/LZMADecompress.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 24963c99..7869893a 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -330,8 +330,9 @@ "Bzip2 Compress", "Tar", "Untar", - "LZString Compress", "LZString Decompress", + "LZString Compress", + "LZMA Decompress", "LZMA Compress" ] }, diff --git a/src/core/operations/LZMACompress.mjs b/src/core/operations/LZMACompress.mjs index 5c038786..0be27089 100644 --- a/src/core/operations/LZMACompress.mjs +++ b/src/core/operations/LZMACompress.mjs @@ -44,8 +44,9 @@ class LZMACompress extends Operation { * @returns {ArrayBuffer} */ run(input, args) { + const mode = Number(args[0]); return new Promise((resolve, reject) => { - compress(new Uint8Array(input), Number(args[0]), (result, error) => { + compress(new Uint8Array(input), mode, (result, error) => { if (error) { reject(new OperationError(`Failed to compress input: ${error.message}`)); } diff --git a/src/core/operations/LZMADecompress.mjs b/src/core/operations/LZMADecompress.mjs new file mode 100644 index 00000000..44908f32 --- /dev/null +++ b/src/core/operations/LZMADecompress.mjs @@ -0,0 +1,51 @@ +/** + * @author Matt C [me@mitt.dev] + * @copyright Crown Copyright 2022 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import {decompress} from "@blu3r4y/lzma"; + +/** + * LZMA Decompress operation + */ +class LZMADecompress extends Operation { + + /** + * LZMADecompress constructor + */ + constructor() { + super(); + + this.name = "LZMA Decompress"; + this.module = "Compression"; + this.description = "Decompresses data using the Lempel-Ziv-Markov chain Algorithm."; + this.infoURL = "https://wikipedia.org/wiki/Lempel%E2%80%93Ziv%E2%80%93Markov_chain_algorithm"; + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {ArrayBuffer} + */ + run(input, args) { + return new Promise((resolve, reject) => { + decompress(new Uint8Array(input), (result, error) => { + if (error) { + reject(new OperationError(`Failed to decompress input: ${error.message}`)); + } + // The decompression returns as an Int8Array, but we can just get the unsigned data from the buffer + resolve(new Int8Array(result).buffer); + }, (percent) => { + self.sendStatusMessage(`Decompressing input: ${(percent*100).toFixed(2)}%`); + }); + }); + } + +} + +export default LZMADecompress; From 98a70c2dd26ab954e7bfcb8bc5ee032ff346d5ff Mon Sep 17 00:00:00 2001 From: Matt C Date: Mon, 19 Sep 2022 17:33:55 +0100 Subject: [PATCH 3/3] Add tests and handle decompress returning string or array --- src/core/operations/LZMACompress.mjs | 5 ++- src/core/operations/LZMADecompress.mjs | 14 +++++-- tests/operations/tests/Compress.mjs | 52 ++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/src/core/operations/LZMACompress.mjs b/src/core/operations/LZMACompress.mjs index 0be27089..5a252db2 100644 --- a/src/core/operations/LZMACompress.mjs +++ b/src/core/operations/LZMACompress.mjs @@ -8,6 +8,7 @@ import Operation from "../Operation.mjs"; import OperationError from "../errors/OperationError.mjs"; import { compress } from "@blu3r4y/lzma"; +import {isWorkerEnvironment} from "../Utils.mjs"; /** * LZMA Compress operation @@ -43,7 +44,7 @@ class LZMACompress extends Operation { * @param {Object[]} args * @returns {ArrayBuffer} */ - run(input, args) { + async run(input, args) { const mode = Number(args[0]); return new Promise((resolve, reject) => { compress(new Uint8Array(input), mode, (result, error) => { @@ -53,7 +54,7 @@ class LZMACompress extends Operation { // The compression returns as an Int8Array, but we can just get the unsigned data from the buffer resolve(new Int8Array(result).buffer); }, (percent) => { - self.sendStatusMessage(`Compressing input: ${(percent*100).toFixed(2)}%`); + if (isWorkerEnvironment()) self.sendStatusMessage(`Compressing input: ${(percent*100).toFixed(2)}%`); }); }); } diff --git a/src/core/operations/LZMADecompress.mjs b/src/core/operations/LZMADecompress.mjs index 44908f32..3bebb860 100644 --- a/src/core/operations/LZMADecompress.mjs +++ b/src/core/operations/LZMADecompress.mjs @@ -7,6 +7,7 @@ import Operation from "../Operation.mjs"; import OperationError from "../errors/OperationError.mjs"; import {decompress} from "@blu3r4y/lzma"; +import Utils, {isWorkerEnvironment} from "../Utils.mjs"; /** * LZMA Decompress operation @@ -32,16 +33,21 @@ class LZMADecompress extends Operation { * @param {Object[]} args * @returns {ArrayBuffer} */ - run(input, args) { + async run(input, args) { return new Promise((resolve, reject) => { decompress(new Uint8Array(input), (result, error) => { if (error) { reject(new OperationError(`Failed to decompress input: ${error.message}`)); } - // The decompression returns as an Int8Array, but we can just get the unsigned data from the buffer - resolve(new Int8Array(result).buffer); + // The decompression returns either a String or an untyped unsigned int8 array, but we can just get the unsigned data from the buffer + + if (typeof result == "string") { + resolve(Utils.strToArrayBuffer(result)); + } else { + resolve(new Int8Array(result).buffer); + } }, (percent) => { - self.sendStatusMessage(`Decompressing input: ${(percent*100).toFixed(2)}%`); + if (isWorkerEnvironment()) self.sendStatusMessage(`Decompressing input: ${(percent*100).toFixed(2)}%`); }); }); } diff --git a/tests/operations/tests/Compress.mjs b/tests/operations/tests/Compress.mjs index a1e895bb..015277b1 100644 --- a/tests/operations/tests/Compress.mjs +++ b/tests/operations/tests/Compress.mjs @@ -23,4 +23,56 @@ TestRegister.addTests([ } ], }, + { + name: "LZMA compress & decompress", + input: "The cat sat on the mat.", + // Generated using command `echo -n "The cat sat on the mat." | lzma -z -6 | xxd -p` + expectedOutput: "The cat sat on the mat.", + recipeConfig: [ + { + "op": "LZMA Compress", + "args": ["6"] + }, + { + "op": "LZMA Decompress", + "args": [] + }, + ], + }, + { + name: "LZMA decompress: binary", + // Generated using command `echo "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10" | xxd -r -p | lzma -z -6 | xxd -p` + input: "5d00008000ffffffffffffffff00000052500a84f99bb28021a969d627e03e8a922effffbd160000", + expectedOutput: "00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10", + recipeConfig: [ + { + "op": "From Hex", + "args": ["Space"] + }, + { + "op": "LZMA Decompress", + "args": [] + }, + { + "op": "To Hex", + "args": ["Space", 0] + } + ], + }, + { + name: "LZMA decompress: string", + // Generated using command `echo -n "The cat sat on the mat." | lzma -z -6 | xxd -p` + input: "5d00008000ffffffffffffffff002a1a08a202b1a4b814b912c94c4152e1641907d3fd8cd903ffff4fec0000", + expectedOutput: "The cat sat on the mat.", + recipeConfig: [ + { + "op": "From Hex", + "args": ["Space"] + }, + { + "op": "LZMA Decompress", + "args": [] + } + ], + }, ]);