From 3cc66e9db980cab8edf9c749ae231438894bce97 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 2 Apr 2019 11:55:59 +0100 Subject: [PATCH 01/27] Added Bzip2 compression support --- package-lock.json | 13 +++-- package.json | 1 + src/core/operations/Bzip2Compress.mjs | 74 +++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 src/core/operations/Bzip2Compress.mjs diff --git a/package-lock.json b/package-lock.json index 9fd4a068..57cde40d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5726,7 +5726,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, @@ -6482,7 +6482,7 @@ }, "html-webpack-plugin": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", + "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=", "dev": true, "requires": { @@ -6607,7 +6607,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -7797,6 +7797,11 @@ "resolved": "https://registry.npmjs.org/lex-parser/-/lex-parser-0.1.4.tgz", "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, + "libbzip2-wasm": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/libbzip2-wasm/-/libbzip2-wasm-0.0.3.tgz", + "integrity": "sha512-eJpB5ITwsdyjWu7DUJirHaBSFLhSbcz8txvL0Gf9NOb+HIzp/lRxFSDXyNAi59ncbhHe3xX+3gj0o+l8rSdJHQ==" + }, "libyara-wasm": { "version": "0.0.12", "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.12.tgz", @@ -9526,7 +9531,7 @@ }, "parse-asn1": { "version": "5.1.1", - "resolved": "http://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==", "dev": true, "requires": { diff --git a/package.json b/package.json index e650b272..412c1da1 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "jsqr": "^1.1.1", "jsrsasign": "8.0.12", "kbpgp": "^2.0.82", + "libbzip2-wasm": "0.0.3", "libyara-wasm": "0.0.12", "lodash": "^4.17.11", "loglevel": "^1.6.1", diff --git a/src/core/operations/Bzip2Compress.mjs b/src/core/operations/Bzip2Compress.mjs new file mode 100644 index 00000000..acc69593 --- /dev/null +++ b/src/core/operations/Bzip2Compress.mjs @@ -0,0 +1,74 @@ +/** + * @author Matt C [me@mitt.dev] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Bzip2 from "libbzip2-wasm"; + +/** + * Bzip2 Compress operation + */ +class Bzip2Compress extends Operation { + + /** + * Bzip2Compress constructor + */ + constructor() { + super(); + + this.name = "Bzip2 Compress"; + this.module = "Compression"; + this.description = "yeet"; + this.infoURL = "https://en.wikipedia.org/wiki/Bzip2"; + this.inputType = "ArrayBuffer"; + this.outputType = "File"; + this.args = [ + { + name: "Block size (100s of kb)", + type: "number", + value: 9 + }, + { + name: "Work factor", + type: "number", + value: 30 + }, + { + name: "Compressed filename", + type: "string", + value: "compressed.bz2" + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {File} + */ + run(input, args) { + const [blockSize, workFactor, filename] = args; + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); + } + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Loading Bzip2..."); + return new Promise((resolve, reject) => { + Bzip2().then(bzip2 => { + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Compressing data..."); + const inpArray = new Uint8Array(input); + const bzip2cc = bzip2.compressBZ2(inpArray, blockSize, workFactor); + if (bzip2cc.error !== 0) { + reject(new OperationError(bzip2cc.error_msg)); + } else { + resolve(new File([bzip2cc.output], filename)); + } + }); + }); + } + +} + +export default Bzip2Compress; From e1492c3bb1e65f56dcfa89f31cdf001a94191c08 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 2 Apr 2019 12:05:17 +0100 Subject: [PATCH 02/27] Added (non-garbage) description and fixed wikipedia link. --- src/core/operations/Bzip2Compress.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/operations/Bzip2Compress.mjs b/src/core/operations/Bzip2Compress.mjs index acc69593..22353e68 100644 --- a/src/core/operations/Bzip2Compress.mjs +++ b/src/core/operations/Bzip2Compress.mjs @@ -21,8 +21,8 @@ class Bzip2Compress extends Operation { this.name = "Bzip2 Compress"; this.module = "Compression"; - this.description = "yeet"; - this.infoURL = "https://en.wikipedia.org/wiki/Bzip2"; + this.description = "Bzip2 is a compression library developed by Julian Seward (of GHC fame) that uses the Burrows-Wheeler algorithm. It only supports compressing single files and its compression is slow, however is more effective than Deflate (.gz & .zip)."; + this.infoURL = "https://wikipedia.org/wiki/Bzip2"; this.inputType = "ArrayBuffer"; this.outputType = "File"; this.args = [ From 8445165491fdf8233656a1edef4519a7a839ac19 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 2 Apr 2019 16:47:38 +0100 Subject: [PATCH 03/27] Use all the arraybuffers cuts a solid 1/3 off the compression time --- src/core/operations/Bzip2Compress.mjs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/core/operations/Bzip2Compress.mjs b/src/core/operations/Bzip2Compress.mjs index 22353e68..e0dd9a12 100644 --- a/src/core/operations/Bzip2Compress.mjs +++ b/src/core/operations/Bzip2Compress.mjs @@ -24,7 +24,7 @@ class Bzip2Compress extends Operation { this.description = "Bzip2 is a compression library developed by Julian Seward (of GHC fame) that uses the Burrows-Wheeler algorithm. It only supports compressing single files and its compression is slow, however is more effective than Deflate (.gz & .zip)."; this.infoURL = "https://wikipedia.org/wiki/Bzip2"; this.inputType = "ArrayBuffer"; - this.outputType = "File"; + this.outputType = "ArrayBuffer"; this.args = [ { name: "Block size (100s of kb)", @@ -35,11 +35,6 @@ class Bzip2Compress extends Operation { name: "Work factor", type: "number", value: 30 - }, - { - name: "Compressed filename", - type: "string", - value: "compressed.bz2" } ]; } @@ -63,7 +58,8 @@ class Bzip2Compress extends Operation { if (bzip2cc.error !== 0) { reject(new OperationError(bzip2cc.error_msg)); } else { - resolve(new File([bzip2cc.output], filename)); + const output = bzip2cc.output; + resolve(output.buffer.slice(output.byteOffset, output.byteLength + output.byteOffset)); } }); }); From 7796c473aeb3e14e3d76ef7972727cfee7e20b09 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 2 Apr 2019 17:01:47 +0100 Subject: [PATCH 04/27] Fix lint issue --- src/core/operations/Bzip2Compress.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Bzip2Compress.mjs b/src/core/operations/Bzip2Compress.mjs index e0dd9a12..5c029c61 100644 --- a/src/core/operations/Bzip2Compress.mjs +++ b/src/core/operations/Bzip2Compress.mjs @@ -45,7 +45,7 @@ class Bzip2Compress extends Operation { * @returns {File} */ run(input, args) { - const [blockSize, workFactor, filename] = args; + const [blockSize, workFactor] = args; if (input.byteLength <= 0) { throw new OperationError("Please provide an input."); } From a339eacd450e283926b30ada409c9aac777bd5f9 Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 7 Apr 2019 18:59:03 +0100 Subject: [PATCH 05/27] Bzip2 compression support changed to use wasm backend x4 speed. --- package-lock.json | 47 ++++++++----------------- package.json | 2 +- src/core/operations/Bzip2Decompress.mjs | 38 ++++++++++++++------ 3 files changed, 42 insertions(+), 45 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3e1011f4..96d1ff25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5263,8 +5263,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -5285,14 +5284,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5307,20 +5304,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -5437,8 +5431,7 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -5450,7 +5443,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5465,7 +5457,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5473,14 +5464,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5499,7 +5488,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -5580,8 +5568,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -5593,7 +5580,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -5679,8 +5665,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -5716,7 +5701,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5736,7 +5720,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5780,14 +5763,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, @@ -7905,9 +7886,9 @@ "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, "libbzip2-wasm": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/libbzip2-wasm/-/libbzip2-wasm-0.0.3.tgz", - "integrity": "sha512-eJpB5ITwsdyjWu7DUJirHaBSFLhSbcz8txvL0Gf9NOb+HIzp/lRxFSDXyNAi59ncbhHe3xX+3gj0o+l8rSdJHQ==" + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/libbzip2-wasm/-/libbzip2-wasm-0.0.4.tgz", + "integrity": "sha512-RqscTx95+RTKhFAyjedsboR0Lmo3zd8//EuRwQXkdWmsCwYlzarVRaiYg6kS1O8m10MCQkGdrnlK9L4eAmZUwA==" }, "libyara-wasm": { "version": "0.0.12", diff --git a/package.json b/package.json index d84e7d17..929e038d 100644 --- a/package.json +++ b/package.json @@ -117,7 +117,7 @@ "jsqr": "^1.2.0", "jsrsasign": "8.0.12", "kbpgp": "2.1.0", - "libbzip2-wasm": "0.0.3", + "libbzip2-wasm": "0.0.4", "libyara-wasm": "0.0.12", "lodash": "^4.17.11", "loglevel": "^1.6.1", diff --git a/src/core/operations/Bzip2Decompress.mjs b/src/core/operations/Bzip2Decompress.mjs index 3b357486..2598e207 100644 --- a/src/core/operations/Bzip2Decompress.mjs +++ b/src/core/operations/Bzip2Decompress.mjs @@ -5,8 +5,8 @@ */ import Operation from "../Operation"; -import bzip2 from "../vendor/bzip2"; import OperationError from "../errors/OperationError"; +import Bzip2 from "libbzip2-wasm"; /** * Bzip2 Decompress operation @@ -23,9 +23,15 @@ class Bzip2Decompress extends Operation { this.module = "Compression"; this.description = "Decompresses data using the Bzip2 algorithm."; this.infoURL = "https://wikipedia.org/wiki/Bzip2"; - this.inputType = "byteArray"; - this.outputType = "string"; - this.args = []; + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + this.args = [ + { + name: "Use low-memory, slower decompression algorithm", + type: "boolean", + value: false + } + ]; this.patterns = [ { "match": "^\\x42\\x5a\\x68", @@ -41,14 +47,24 @@ class Bzip2Decompress extends Operation { * @returns {string} */ run(input, args) { - const compressed = new Uint8Array(input); - - try { - const bzip2Reader = bzip2.array(compressed); - return bzip2.simple(bzip2Reader); - } catch (err) { - throw new OperationError(err); + const [small] = args; + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); } + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Loading Bzip2..."); + return new Promise((resolve, reject) => { + Bzip2().then(bzip2 => { + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Decompressing data..."); + const inpArray = new Uint8Array(input); + const bzip2cc = bzip2.decompressBZ2(inpArray, small ? 1 : 0); + if (bzip2cc.error !== 0) { + reject(new OperationError(bzip2cc.error_msg)); + } else { + const output = bzip2cc.output; + resolve(output.buffer.slice(output.byteOffset, output.byteLength + output.byteOffset)); + } + }); + }); } } From 982c915931e41a0a855668f5966e693684f86c7a Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 7 Apr 2019 19:02:27 +0100 Subject: [PATCH 06/27] Change author --- src/core/operations/Bzip2Decompress.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/operations/Bzip2Decompress.mjs b/src/core/operations/Bzip2Decompress.mjs index 2598e207..08a1f3d7 100644 --- a/src/core/operations/Bzip2Decompress.mjs +++ b/src/core/operations/Bzip2Decompress.mjs @@ -1,6 +1,6 @@ /** - * @author n1474335 [n1474335@gmail.com] - * @copyright Crown Copyright 2016 + * @author Matt C [me@mitt.dev] + * @copyright Crown Copyright 2019 * @license Apache-2.0 */ From 18408901bee44bd8b0300bb0c0ad27661649f67f Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 7 Apr 2019 19:11:46 +0100 Subject: [PATCH 07/27] removed old bzip2 dependency --- src/core/vendor/bzip2.mjs | 265 -------------------------------------- 1 file changed, 265 deletions(-) delete mode 100755 src/core/vendor/bzip2.mjs diff --git a/src/core/vendor/bzip2.mjs b/src/core/vendor/bzip2.mjs deleted file mode 100755 index df0573d8..00000000 --- a/src/core/vendor/bzip2.mjs +++ /dev/null @@ -1,265 +0,0 @@ -/** @license -======================================================================== - bzip2.js - a small bzip2 decompression implementation - - Copyright 2011 by antimatter15 (antimatter15@gmail.com) - - Based on micro-bunzip by Rob Landley (rob@landley.net). - - Copyright (c) 2011 by antimatter15 (antimatter15@gmail.com). - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, - DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH - THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ -"use strict"; - -var bzip2 = {}; - -bzip2.array = function(bytes){ - var bit = 0, byte = 0; - var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF ]; - return function(n){ - var result = 0; - while(n > 0){ - var left = 8 - bit; - if(n >= left){ - result <<= left; - result |= (BITMASK[left] & bytes[byte++]); - bit = 0; - n -= left; - }else{ - result <<= n; - result |= ((bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit)); - bit += n; - n = 0; - } - } - return result - } -} - -bzip2.simple = function(bits){ - var size = bzip2.header(bits); - var all = '', chunk = ''; - do{ - all += chunk; - chunk = bzip2.decompress(bits, size); - }while(chunk != -1); - return all; -} - -bzip2.header = function(bits){ - if(bits(8*3) != 4348520) throw "No magic number found"; - var i = bits(8) - 48; - if(i < 1 || i > 9) throw "Not a BZIP archive"; - return i; -}; - - -//takes a function for reading the block data (starting with 0x314159265359) -//a block size (0-9) (optional, defaults to 9) -//a length at which to stop decompressing and return the output -bzip2.decompress = function(bits, size, len){ - var MAX_HUFCODE_BITS = 20; - var MAX_SYMBOLS = 258; - var SYMBOL_RUNA = 0; - var SYMBOL_RUNB = 1; - var GROUP_SIZE = 50; - - var bufsize = 100000 * size; - for(var h = '', i = 0; i < 6; i++) h += bits(8).toString(16); - if(h == "177245385090") return -1; //last block - if(h != "314159265359") throw "Not valid bzip data"; - bits(32); //ignore CRC codes - if(bits(1)) throw "Unsupported obsolete version"; - var origPtr = bits(24); - if(origPtr > bufsize) throw "Initial position larger than buffer size"; - var t = bits(16); - var symToByte = new Uint8Array(256), - symTotal = 0; - for (i = 0; i < 16; i++) { - if(t & (1 << (15 - i))) { - var k = bits(16); - for(j = 0; j < 16; j++){ - if(k & (1 << (15 - j))){ - symToByte[symTotal++] = (16 * i) + j; - } - } - } - } - - var groupCount = bits(3); - if(groupCount < 2 || groupCount > 6) throw "Error 1"; - var nSelectors = bits(15); - if(nSelectors == 0) throw "Error"; - var mtfSymbol = []; //TODO: possibly replace JS array with typed arrays - for(var i = 0; i < groupCount; i++) mtfSymbol[i] = i; - var selectors = new Uint8Array(32768); - - for(var i = 0; i < nSelectors; i++){ - for(var j = 0; bits(1); j++) if(j >= groupCount) throw "Error 2"; - var uc = mtfSymbol[j]; - mtfSymbol.splice(j, 1); //this is a probably inefficient MTF transform - mtfSymbol.splice(0, 0, uc); - selectors[i] = uc; - } - - var symCount = symTotal + 2; - var groups = []; - for(var j = 0; j < groupCount; j++){ - var length = new Uint8Array(MAX_SYMBOLS), - temp = new Uint8Array(MAX_HUFCODE_BITS+1); - t = bits(5); //lengths - for(var i = 0; i < symCount; i++){ - while(true){ - if (t < 1 || t > MAX_HUFCODE_BITS) throw "Error 3"; - if(!bits(1)) break; - if(!bits(1)) t++; - else t--; - } - length[i] = t; - } - var minLen, maxLen; - minLen = maxLen = length[0]; - for(var i = 1; i < symCount; i++){ - if(length[i] > maxLen) maxLen = length[i]; - else if(length[i] < minLen) minLen = length[i]; - } - var hufGroup; - hufGroup = groups[j] = {}; - hufGroup.permute = new Uint32Array(MAX_SYMBOLS); - hufGroup.limit = new Uint32Array(MAX_HUFCODE_BITS + 1); - hufGroup.base = new Uint32Array(MAX_HUFCODE_BITS + 1); - hufGroup.minLen = minLen; - hufGroup.maxLen = maxLen; - var base = hufGroup.base.subarray(1); - var limit = hufGroup.limit.subarray(1); - var pp = 0; - for(var i = minLen; i <= maxLen; i++) - for(var t = 0; t < symCount; t++) - if(length[t] == i) hufGroup.permute[pp++] = t; - for(i = minLen; i <= maxLen; i++) temp[i] = limit[i] = 0; - for(i = 0; i < symCount; i++) temp[length[i]]++; - pp = t = 0; - for(i = minLen; i < maxLen; i++) { - pp += temp[i]; - limit[i] = pp - 1; - pp <<= 1; - base[i+1] = pp - (t += temp[i]); - } - limit[maxLen]=pp+temp[maxLen]-1; - base[minLen]=0; - } - var byteCount = new Uint32Array(256); - for(var i = 0; i < 256; i++) mtfSymbol[i] = i; - var runPos, count, symCount, selector; - runPos = count = symCount = selector = 0; - var buf = new Uint32Array(bufsize); - while(true){ - if(!(symCount--)){ - symCount = GROUP_SIZE - 1; - if(selector >= nSelectors) throw "Error 4"; - hufGroup = groups[selectors[selector++]]; - base = hufGroup.base.subarray(1); - limit = hufGroup.limit.subarray(1); - } - i = hufGroup.minLen; - j = bits(i); - while(true){ - if(i > hufGroup.maxLen) throw "Error 5"; - if(j <= limit[i]) break; - i++; - j = (j << 1) | bits(1); - } - j -= base[i]; - if(j < 0 || j >= MAX_SYMBOLS) throw "Error 6"; - var nextSym = hufGroup.permute[j]; - if (nextSym == SYMBOL_RUNA || nextSym == SYMBOL_RUNB) { - if(!runPos){ - runPos = 1; - t = 0; - } - if(nextSym == SYMBOL_RUNA) t += runPos; - else t += 2 * runPos; - runPos <<= 1; - continue; - } - if(runPos){ - runPos = 0; - if(count + t >= bufsize) throw "Error 7"; - uc = symToByte[mtfSymbol[0]]; - byteCount[uc] += t; - while(t--) buf[count++] = uc; - } - if(nextSym > symTotal) break; - if(count >= bufsize) throw "Error 8"; - i = nextSym -1; - uc = mtfSymbol[i]; - mtfSymbol.splice(i, 1); - mtfSymbol.splice(0, 0, uc); - uc = symToByte[uc]; - byteCount[uc]++; - buf[count++] = uc; - } - if(origPtr < 0 || origPtr >= count) throw "Error 9"; - var j = 0; - for(var i = 0; i < 256; i++){ - k = j + byteCount[i]; - byteCount[i] = j; - j = k; - } - for(var i = 0; i < count; i++){ - uc = buf[i] & 0xff; - buf[byteCount[uc]] |= (i << 8); - byteCount[uc]++; - } - var pos = 0, current = 0, run = 0; - if(count) { - pos = buf[origPtr]; - current = (pos & 0xff); - pos >>= 8; - run = -1; - } - count = count; - var output = ''; - var copies, previous, outbyte; - if(!len) len = Infinity; - while(count){ - count--; - previous = current; - pos = buf[pos]; - current = pos & 0xff; - pos >>= 8; - if(run++ == 3){ - copies = current; - outbyte = previous; - current = -1; - }else{ - copies = 1; - outbyte = current; - } - while(copies--){ - output += (String.fromCharCode(outbyte)); - if(!--len) return output; - } - if(current != previous) run = 0; - } - return output; -} - -export default bzip2; From 8fa8e3402703b309b3c2e83b3cf4c0660ad137f4 Mon Sep 17 00:00:00 2001 From: mshwed Date: Sun, 28 Apr 2019 16:29:15 -0400 Subject: [PATCH 08/27] Added support for parsing JSON with number type values. Added support for non-array JSON objects. Added extra tests for JSON to CSV operation. --- src/core/operations/JSONToCSV.mjs | 6 ++ tests/operations/index.mjs | 1 + tests/operations/tests/JSONtoCSV.mjs | 93 ++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 tests/operations/tests/JSONtoCSV.mjs diff --git a/src/core/operations/JSONToCSV.mjs b/src/core/operations/JSONToCSV.mjs index c3d078e3..a20d1728 100644 --- a/src/core/operations/JSONToCSV.mjs +++ b/src/core/operations/JSONToCSV.mjs @@ -51,6 +51,10 @@ class JSONToCSV extends Operation { this.rowDelim = rowDelim; const self = this; + if (!(input instanceof Array)) { + input = [input]; + } + try { // If the JSON is an array of arrays, this is easy if (input[0] instanceof Array) { @@ -89,6 +93,8 @@ class JSONToCSV extends Operation { * @returns {string} */ escapeCellContents(data) { + if (typeof data === "number") data = data.toString(); + // Double quotes should be doubled up data = data.replace(/"/g, '""'); diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 41d78c35..8129d4eb 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -52,6 +52,7 @@ import "./tests/Image"; import "./tests/Jump"; import "./tests/JSONBeautify"; import "./tests/JSONMinify"; +import "./tests/JSONtoCSV"; import "./tests/JWTDecode"; import "./tests/JWTSign"; import "./tests/JWTVerify"; diff --git a/tests/operations/tests/JSONtoCSV.mjs b/tests/operations/tests/JSONtoCSV.mjs new file mode 100644 index 00000000..67d46e64 --- /dev/null +++ b/tests/operations/tests/JSONtoCSV.mjs @@ -0,0 +1,93 @@ +/** + * JSON to CSV tests. + * + * @author mshwed [m@ttshwed.com] + * + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import TestRegister from "../TestRegister"; + +const EXPECTED_CSV_SINGLE = "a,b,c\r\n1,2,3\r\n"; +const EXPECTED_CSV_MULTIPLE = "a,b,c\r\n1,2,3\r\n1,2,3\r\n"; +const EXPECTED_CSV_EMPTY = "\r\n\r\n"; + +TestRegister.addTests([ + { + name: "JSON to CSV: strings as values", + input: JSON.stringify({a: "1", b: "2", c: "3"}), + expectedOutput: EXPECTED_CSV_SINGLE, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: numbers as values", + input: JSON.stringify({a: 1, b: 2, c: 3}), + expectedOutput: EXPECTED_CSV_SINGLE, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: numbers and strings as values", + input: JSON.stringify({a: 1, b: "2", c: 3}), + expectedOutput: EXPECTED_CSV_SINGLE, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: JSON as an array", + input: JSON.stringify([{a: 1, b: "2", c: 3}]), + expectedOutput: EXPECTED_CSV_SINGLE, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: multiple JSON values in an array", + input: JSON.stringify([{a: 1, b: "2", c: 3}, {a: 1, b: "2", c: 3}]), + expectedOutput: EXPECTED_CSV_MULTIPLE, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: empty JSON", + input: JSON.stringify({}), + expectedOutput: EXPECTED_CSV_EMPTY, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: empty JSON in array", + input: JSON.stringify([{}]), + expectedOutput: EXPECTED_CSV_EMPTY, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + } +]); From 1d130c88a82ca7bfc50408c02f39b0e4aa46f94d Mon Sep 17 00:00:00 2001 From: h345983745 Date: Sun, 19 May 2019 15:40:17 +0100 Subject: [PATCH 09/27] Inital commit --- .github/ISSUE_TEMPLATE.md | 15 +-------- .github/ISSUE_TEMPLATE/bug_report.md | 35 +++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++++ .github/ISSUE_TEMPLATE/operation-request.md | 16 ++++++++++ 4 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/operation-request.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 8e7dee82..e90ab51f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,14 +1 @@ - - - - - - -### Summary - - -### Example - - - - + diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..b293da9a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,35 @@ +--- +name: Bug report +about: Create a report to help us improve +title: 'Bug report: ' +labels: bug +assignees: '' + +--- + + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior or a link to the recipe / input used to cause the bug: + +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (if relevant, please complete the following information):** + - OS: [e.g. Windows] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..9a38f0e5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for the project +title: 'Feature request: ' +labels: feature +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/operation-request.md b/.github/ISSUE_TEMPLATE/operation-request.md new file mode 100644 index 00000000..03a45a14 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/operation-request.md @@ -0,0 +1,16 @@ +--- +name: Operation request +about: Suggest an operation +title: 'Operation request: ' +labels: operation +assignees: '' + +--- + + + +## Summary + +### Example Input + +### Example Output From 466d872d30305c2b8ab7c42ad22769e57a3b4dff Mon Sep 17 00:00:00 2001 From: George O <16269580+Ge0rg3@users.noreply.github.com> Date: Sat, 8 Jun 2019 21:43:30 +0100 Subject: [PATCH 10/27] Added Index of Coincidence Operation --- src/core/config/Categories.json | 1 + src/core/operations/IndexOfCoincidence.mjs | 107 +++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 src/core/operations/IndexOfCoincidence.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 2d194c37..102d76aa 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -395,6 +395,7 @@ "ops": [ "Entropy", "Frequency distribution", + "Index of Coincidence", "Chi Square", "Disassemble x86", "Pseudo-Random Number Generator", diff --git a/src/core/operations/IndexOfCoincidence.mjs b/src/core/operations/IndexOfCoincidence.mjs new file mode 100644 index 00000000..8edf5e33 --- /dev/null +++ b/src/core/operations/IndexOfCoincidence.mjs @@ -0,0 +1,107 @@ +/** + * @author George O [georgeomnet+cyberchef@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import Utils from "../Utils"; + +/** + * Index of Coincidence operation + */ +class IndexOfCoincidence extends Operation { + + /** + * IndexOfCoincidence constructor + */ + constructor() { + super(); + + this.name = "Index of Coincidence"; + this.module = "Default"; + this.description = "Index of Coincidence (IC) is the probability of two randomly selected characters being the same. This can be used to determine whether text is readable or random, with English text having an IC of around 0.066. IC can therefore be a sound method to automate frequency analysis."; + this.infoURL = "https://wikipedia.org/wiki/Index_of_coincidence"; + this.inputType = "string"; + this.outputType = "number"; + this.presentType = "html"; + this.args = []; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {number} + */ + run(input, args) { + const text = input.toLowerCase().replace(/[^a-z]/g, ""), + frequencies = new Array(26).fill(0), + alphabet = Utils.expandAlphRange("a-z"); + let coincidence = 0.00, + density = 0.00, + result = 0.00, + i; + + for (i=0; i < alphabet.length; i++) { + frequencies[i] = text.count(alphabet[i]); + } + + for (i=0; i < frequencies.length; i++) { + coincidence += frequencies[i] * (frequencies[i] - 1); + } + + density = frequencies.sum(); + + // Ensure that we don't divide by 0 + if (density < 2) density = 2; + + result = coincidence / (density * (density - 1)); + + return result; + } + + /** + * Displays the IC as a scale bar for web apps. + * + * @param {number} ic + * @returns {html} + */ + present(ic) { + return `Index of Coincidence: ${ic} +Normalized: ${ic * 26} +

+- 0 represents complete randomness (all characters are unique), whereas 1 represents no randomness (all characters are identical). +- English text generally has an IC of between 0.67 to 0.78. +- 'Random' text is determined by the probability that each letter occurs the same number of times as another. + +The graph shows the IC of the input data. A low IC generally means that the text is random, encoded or encrypted. + + + `; + } + +} + +export default IndexOfCoincidence; From a6732ba81524fc34e2ce58a1dff7304f3ac1c31f Mon Sep 17 00:00:00 2001 From: George O <16269580+Ge0rg3@users.noreply.github.com> Date: Sat, 8 Jun 2019 21:49:31 +0100 Subject: [PATCH 11/27] Added Index of Coincidence Tests --- tests/operations/index.mjs | 1 + tests/operations/tests/IndexOfCoincidence.mjs | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/operations/tests/IndexOfCoincidence.mjs diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 41d78c35..e40a5bd9 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -49,6 +49,7 @@ import "./tests/Hash"; import "./tests/HaversineDistance"; import "./tests/Hexdump"; import "./tests/Image"; +import "./tests/IndexOfCoincidence"; import "./tests/Jump"; import "./tests/JSONBeautify"; import "./tests/JSONMinify"; diff --git a/tests/operations/tests/IndexOfCoincidence.mjs b/tests/operations/tests/IndexOfCoincidence.mjs new file mode 100644 index 00000000..3dc4cd35 --- /dev/null +++ b/tests/operations/tests/IndexOfCoincidence.mjs @@ -0,0 +1,22 @@ +/** + * Index of Coincidence tests. + * + * @author George O [georgeomnet+cyberchef@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import TestRegister from "../TestRegister"; + +TestRegister.addTests([ + { + name: "Index of Coincidence", + input: "Hello world, this is a test to determine the correct IC value.", + expectedMatch: /^Index of Coincidence: 0\.07142857142857142\nNormalized: 1\.857142857142857/, + recipeConfig: [ + { + "op": "Index of Coincidence", + "args": [] + }, + ], + }, +]); From e7095820629b27cccbbaf81a5509de2780a666b6 Mon Sep 17 00:00:00 2001 From: George O <16269580+Ge0rg3@users.noreply.github.com> Date: Mon, 10 Jun 2019 00:06:15 +0100 Subject: [PATCH 12/27] Disappearing Popover Fix --- src/web/RecipeWaiter.mjs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/web/RecipeWaiter.mjs b/src/web/RecipeWaiter.mjs index f2edd725..b8b523dc 100755 --- a/src/web/RecipeWaiter.mjs +++ b/src/web/RecipeWaiter.mjs @@ -124,16 +124,21 @@ class RecipeWaiter { * @param {event} evt */ opSortEnd(evt) { - if (this.removeIntent) { - if (evt.item.parentNode.id === "rec-list") { - evt.item.remove(); - } + if (this.removeIntent && evt.item.parentNode.id === "rec-list") { + evt.item.remove(); return; } // Reinitialise the popover on the original element in the ops list because for some reason it - // gets destroyed and recreated. - this.manager.ops.enableOpsListPopovers(evt.clone); + // gets destroyed and recreated. If the clone isn't in the ops list, we use the original item instead. + let enableOpsElement; + if (evt.clone.parentNode && evt.clone.parentNode.classList.contains("op-list")) { + enableOpsElement = evt.clone; + } else { + enableOpsElement = evt.item; + $(evt.item).attr("data-toggle", "popover"); + } + this.manager.ops.enableOpsListPopovers(enableOpsElement); if (evt.item.parentNode.id !== "rec-list") { return; From f29d8eeda89a01219d3a449eae1cd697555f4fcb Mon Sep 17 00:00:00 2001 From: wh0 Date: Tue, 25 Jun 2019 18:35:25 -0700 Subject: [PATCH 13/27] Parse octal escape sequences --- src/core/Utils.mjs | 11 +++++++++-- src/core/operations/UnescapeString.mjs | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/core/Utils.mjs b/src/core/Utils.mjs index e4411fe0..273eca1e 100755 --- a/src/core/Utils.mjs +++ b/src/core/Utils.mjs @@ -201,11 +201,18 @@ class Utils { * Utils.parseEscapedChars("\\n"); */ static parseEscapedChars(str) { - return str.replace(/(\\)?\\([bfnrtv0'"]|x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]{1,6}\})/g, function(m, a, b) { + return str.replace(/(\\)?\\([bfnrtv'"]|[0-3][0-7]{2}|[0-7]{1,2}|x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]{1,6}\})/g, function(m, a, b) { if (a === "\\") return "\\"+b; switch (b[0]) { case "0": - return "\0"; + case "1": + case "2": + case "3": + case "4": + case "5": + case "6": + case "7": + return String.fromCharCode(parseInt(b, 8)); case "b": return "\b"; case "t": diff --git a/src/core/operations/UnescapeString.mjs b/src/core/operations/UnescapeString.mjs index 62eab48e..1a625582 100644 --- a/src/core/operations/UnescapeString.mjs +++ b/src/core/operations/UnescapeString.mjs @@ -20,7 +20,7 @@ class UnescapeString extends Operation { this.name = "Unescape string"; this.module = "Default"; - this.description = "Unescapes characters in a string that have been escaped. For example, Don\\'t stop me now becomes Don't stop me now.

Supports the following escape sequences:
  • \\n (Line feed/newline)
  • \\r (Carriage return)
  • \\t (Horizontal tab)
  • \\b (Backspace)
  • \\f (Form feed)
  • \\xnn (Hex, where n is 0-f)
  • \\\\ (Backslash)
  • \\' (Single quote)
  • \\" (Double quote)
  • \\unnnn (Unicode character)
  • \\u{nnnnnn} (Unicode code point)
"; + this.description = "Unescapes characters in a string that have been escaped. For example, Don\\'t stop me now becomes Don't stop me now.

Supports the following escape sequences:
  • \\n (Line feed/newline)
  • \\r (Carriage return)
  • \\t (Horizontal tab)
  • \\b (Backspace)
  • \\f (Form feed)
  • \\nnn (Octal, where n is 0-7)
  • \\xnn (Hex, where n is 0-f)
  • \\\\ (Backslash)
  • \\' (Single quote)
  • \\" (Double quote)
  • \\unnnn (Unicode character)
  • \\u{nnnnnn} (Unicode code point)
"; this.infoURL = "https://wikipedia.org/wiki/Escape_sequence"; this.inputType = "string"; this.outputType = "string"; From ccf2348cd6cd8fc9ca11f01ecbf933b1085ecf60 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 27 Jun 2019 15:34:22 +0100 Subject: [PATCH 14/27] 8.31.7 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 88652e86..96135781 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.31.6", + "version": "8.31.7", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index ed1d1cd3..a47f521a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.31.6", + "version": "8.31.7", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 393d070b0544dec864177e868a41bf6a9cd44b78 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 27 Jun 2019 15:37:12 +0100 Subject: [PATCH 15/27] 8.31.8 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 96135781..067887df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.31.7", + "version": "8.31.8", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index a47f521a..c6801479 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.31.7", + "version": "8.31.8", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From e11aec64cd2b8d6f20ebb3b4f6e0d8b179376437 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 27 Jun 2019 15:42:32 +0100 Subject: [PATCH 16/27] Modified wording for IC op --- src/core/operations/IndexOfCoincidence.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/IndexOfCoincidence.mjs b/src/core/operations/IndexOfCoincidence.mjs index 8edf5e33..4a417769 100644 --- a/src/core/operations/IndexOfCoincidence.mjs +++ b/src/core/operations/IndexOfCoincidence.mjs @@ -74,7 +74,7 @@ Normalized: ${ic * 26} - English text generally has an IC of between 0.67 to 0.78. - 'Random' text is determined by the probability that each letter occurs the same number of times as another. -The graph shows the IC of the input data. A low IC generally means that the text is random, encoded or encrypted. +The graph shows the IC of the input data. A low IC generally means that the text is random, compressed or encrypted.