From 3cc66e9db980cab8edf9c749ae231438894bce97 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 2 Apr 2019 11:55:59 +0100 Subject: [PATCH 01/42] Added Bzip2 compression support --- package-lock.json | 13 +++-- package.json | 1 + src/core/operations/Bzip2Compress.mjs | 74 +++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 src/core/operations/Bzip2Compress.mjs diff --git a/package-lock.json b/package-lock.json index 9fd4a068..57cde40d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5726,7 +5726,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, @@ -6482,7 +6482,7 @@ }, "html-webpack-plugin": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", + "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=", "dev": true, "requires": { @@ -6607,7 +6607,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -7797,6 +7797,11 @@ "resolved": "https://registry.npmjs.org/lex-parser/-/lex-parser-0.1.4.tgz", "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, + "libbzip2-wasm": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/libbzip2-wasm/-/libbzip2-wasm-0.0.3.tgz", + "integrity": "sha512-eJpB5ITwsdyjWu7DUJirHaBSFLhSbcz8txvL0Gf9NOb+HIzp/lRxFSDXyNAi59ncbhHe3xX+3gj0o+l8rSdJHQ==" + }, "libyara-wasm": { "version": "0.0.12", "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.12.tgz", @@ -9526,7 +9531,7 @@ }, "parse-asn1": { "version": "5.1.1", - "resolved": "http://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==", "dev": true, "requires": { diff --git a/package.json b/package.json index e650b272..412c1da1 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "jsqr": "^1.1.1", "jsrsasign": "8.0.12", "kbpgp": "^2.0.82", + "libbzip2-wasm": "0.0.3", "libyara-wasm": "0.0.12", "lodash": "^4.17.11", "loglevel": "^1.6.1", diff --git a/src/core/operations/Bzip2Compress.mjs b/src/core/operations/Bzip2Compress.mjs new file mode 100644 index 00000000..acc69593 --- /dev/null +++ b/src/core/operations/Bzip2Compress.mjs @@ -0,0 +1,74 @@ +/** + * @author Matt C [me@mitt.dev] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Bzip2 from "libbzip2-wasm"; + +/** + * Bzip2 Compress operation + */ +class Bzip2Compress extends Operation { + + /** + * Bzip2Compress constructor + */ + constructor() { + super(); + + this.name = "Bzip2 Compress"; + this.module = "Compression"; + this.description = "yeet"; + this.infoURL = "https://en.wikipedia.org/wiki/Bzip2"; + this.inputType = "ArrayBuffer"; + this.outputType = "File"; + this.args = [ + { + name: "Block size (100s of kb)", + type: "number", + value: 9 + }, + { + name: "Work factor", + type: "number", + value: 30 + }, + { + name: "Compressed filename", + type: "string", + value: "compressed.bz2" + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {File} + */ + run(input, args) { + const [blockSize, workFactor, filename] = args; + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); + } + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Loading Bzip2..."); + return new Promise((resolve, reject) => { + Bzip2().then(bzip2 => { + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Compressing data..."); + const inpArray = new Uint8Array(input); + const bzip2cc = bzip2.compressBZ2(inpArray, blockSize, workFactor); + if (bzip2cc.error !== 0) { + reject(new OperationError(bzip2cc.error_msg)); + } else { + resolve(new File([bzip2cc.output], filename)); + } + }); + }); + } + +} + +export default Bzip2Compress; From e1492c3bb1e65f56dcfa89f31cdf001a94191c08 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 2 Apr 2019 12:05:17 +0100 Subject: [PATCH 02/42] Added (non-garbage) description and fixed wikipedia link. --- src/core/operations/Bzip2Compress.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/operations/Bzip2Compress.mjs b/src/core/operations/Bzip2Compress.mjs index acc69593..22353e68 100644 --- a/src/core/operations/Bzip2Compress.mjs +++ b/src/core/operations/Bzip2Compress.mjs @@ -21,8 +21,8 @@ class Bzip2Compress extends Operation { this.name = "Bzip2 Compress"; this.module = "Compression"; - this.description = "yeet"; - this.infoURL = "https://en.wikipedia.org/wiki/Bzip2"; + this.description = "Bzip2 is a compression library developed by Julian Seward (of GHC fame) that uses the Burrows-Wheeler algorithm. It only supports compressing single files and its compression is slow, however is more effective than Deflate (.gz & .zip)."; + this.infoURL = "https://wikipedia.org/wiki/Bzip2"; this.inputType = "ArrayBuffer"; this.outputType = "File"; this.args = [ From 8445165491fdf8233656a1edef4519a7a839ac19 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 2 Apr 2019 16:47:38 +0100 Subject: [PATCH 03/42] Use all the arraybuffers cuts a solid 1/3 off the compression time --- src/core/operations/Bzip2Compress.mjs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/core/operations/Bzip2Compress.mjs b/src/core/operations/Bzip2Compress.mjs index 22353e68..e0dd9a12 100644 --- a/src/core/operations/Bzip2Compress.mjs +++ b/src/core/operations/Bzip2Compress.mjs @@ -24,7 +24,7 @@ class Bzip2Compress extends Operation { this.description = "Bzip2 is a compression library developed by Julian Seward (of GHC fame) that uses the Burrows-Wheeler algorithm. It only supports compressing single files and its compression is slow, however is more effective than Deflate (.gz & .zip)."; this.infoURL = "https://wikipedia.org/wiki/Bzip2"; this.inputType = "ArrayBuffer"; - this.outputType = "File"; + this.outputType = "ArrayBuffer"; this.args = [ { name: "Block size (100s of kb)", @@ -35,11 +35,6 @@ class Bzip2Compress extends Operation { name: "Work factor", type: "number", value: 30 - }, - { - name: "Compressed filename", - type: "string", - value: "compressed.bz2" } ]; } @@ -63,7 +58,8 @@ class Bzip2Compress extends Operation { if (bzip2cc.error !== 0) { reject(new OperationError(bzip2cc.error_msg)); } else { - resolve(new File([bzip2cc.output], filename)); + const output = bzip2cc.output; + resolve(output.buffer.slice(output.byteOffset, output.byteLength + output.byteOffset)); } }); }); From 7796c473aeb3e14e3d76ef7972727cfee7e20b09 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 2 Apr 2019 17:01:47 +0100 Subject: [PATCH 04/42] Fix lint issue --- src/core/operations/Bzip2Compress.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Bzip2Compress.mjs b/src/core/operations/Bzip2Compress.mjs index e0dd9a12..5c029c61 100644 --- a/src/core/operations/Bzip2Compress.mjs +++ b/src/core/operations/Bzip2Compress.mjs @@ -45,7 +45,7 @@ class Bzip2Compress extends Operation { * @returns {File} */ run(input, args) { - const [blockSize, workFactor, filename] = args; + const [blockSize, workFactor] = args; if (input.byteLength <= 0) { throw new OperationError("Please provide an input."); } From dec28e16d4457397371112bab42c2b28fb86c461 Mon Sep 17 00:00:00 2001 From: mshwed Date: Fri, 5 Apr 2019 11:12:44 -0400 Subject: [PATCH 05/42] Added histogram visualization for text entropy --- src/core/operations/AdvancedEntropy.mjs | 180 ++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 src/core/operations/AdvancedEntropy.mjs diff --git a/src/core/operations/AdvancedEntropy.mjs b/src/core/operations/AdvancedEntropy.mjs new file mode 100644 index 00000000..b3316583 --- /dev/null +++ b/src/core/operations/AdvancedEntropy.mjs @@ -0,0 +1,180 @@ +/** + * @author mshwed [m@ttshwed.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import * as d3temp from "d3"; +import * as nodomtemp from "nodom"; + +import Operation from "../Operation"; +import Utils from "../Utils"; + +const d3 = d3temp.default ? d3temp.default : d3temp; +const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp; + +/** + * Advanced Entropy operation + */ +class AdvancedEntropy extends Operation { + + /** + * AdvancedEntropy constructor + */ + constructor() { + super(); + + this.name = "Advanced Entropy"; + this.module = "Default"; + this.description = "Adds advanced views for examining entropy"; + this.infoURL = ""; + this.inputType = "byteArray"; + this.outputType = "html"; + this.presentType = "html"; + this.args = []; + } + + /** + * Calculates the frequency of bytes in the input. + * + * @param {byteArray} input + * @returns {frequency} + */ + calculateShannonEntropy(input) { + const prob = [], + uniques = input.unique(), + str = Utils.byteArrayToChars(input); + let i; + + for (i = 0; i < uniques.length; i++) { + prob.push(str.count(Utils.chr(uniques[i])) / input.length); + } + + let entropy = 0, + p; + + for (i = 0; i < prob.length; i++) { + p = prob[i]; + entropy += p * Math.log(p) / Math.log(2); + } + + return -entropy; + } + + /** + * Calculates the frequency of bytes in the input. + * + * @param {byteArray} inputBytes + * @returns {frequency} + */ + calculateByteFrequency(inputBytes) { + console.log(inputBytes); + + let byteFrequency = []; + for (let i = 0; i < 256; i++) { + let count = 0; + for (let byte of inputBytes) { + if (byte === i) { + count++; + } + } + + byteFrequency.push(count / (inputBytes.length + 1)) + } + + return byteFrequency; + } + + /** + * Creates a byte frequency histogram + * + * @param {byteArray} entropyData + * @returns {HTML} + */ + createByteFrequencyHistogram(entropyData) { + const byteFrequency = entropyData.byteFrequency; + + const svgWidth = 500, + svgHeight = 500, + binWidth = 1; + + const document = new nodom.Document(); + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + + const margins = {top: 30, right: 20, bottom: 50, left: 30}; + + const yScale = d3.scaleLinear() + .domain(d3.extent(byteFrequency, d => d)) + .range([svgHeight - margins.bottom, margins.top]) + + const xScale = d3.scaleLinear() + .domain([0, byteFrequency.length - 1]) + .range([margins.left - binWidth, svgWidth - margins.right]) + + svg.selectAll("rect") + .data(byteFrequency) + .enter().append("rect") + .attr("x", (_, i) => xScale(i) + binWidth) + .attr("y", (dataPoint) => yScale(dataPoint)) + .attr("width", binWidth) + .attr("height", dataPoint => yScale(0) - yScale(dataPoint)) + .attr("fill", "blue") + + // Axes + const yAxis = d3.axisLeft() + .scale(yScale); + + const xAxis = d3.axisBottom() + .scale(xScale) + + svg.append("g") + .attr("transform", `translate(${binWidth}, ${svgHeight - margins.bottom})`) + .call(xAxis); + + svg.append("g") + .attr("transform", `translate(${margins.left},0)`) + .call(yAxis); + + // Axes labels + svg.append("text") + .attr("transform", "rotate(-90)") + .attr("y", 0 - margins.left) + .attr("x", 0 - (svgHeight / 2)) + .attr("dy", "1em") + .style("text-anchor", "middle") + .text("Frequency (%)") + + svg.append("text") + .attr("transform", `translate(${svgWidth / 2}, ${svgHeight - margins.bottom + 40})`) + .style("text-anchor", "middle") + .text("Byte") + + // Add title + svg.append("text") + .attr("transform", `translate(${svgWidth / 2}, ${margins.top - 10})`) + .style("text-anchor", "middle") + .text("Byte Frequency") + + return svg._groups[0][0].outerHTML; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {html} + */ + run(input, args) { + const entropyData = { + entropy: this.calculateShannonEntropy(input), + byteFrequency: this.calculateByteFrequency(input) + }; + let svgData = this.createByteFrequencyHistogram(entropyData) + return svgData + } +} + +export default AdvancedEntropy; From c80cb57b076baf69ba3b24e2f404e3a3807b6569 Mon Sep 17 00:00:00 2001 From: mshwed Date: Fri, 5 Apr 2019 14:30:24 -0400 Subject: [PATCH 06/42] Added histogram line, refactored axes generation --- src/core/operations/AdvancedEntropy.mjs | 202 ++++++++++++++++-------- 1 file changed, 137 insertions(+), 65 deletions(-) diff --git a/src/core/operations/AdvancedEntropy.mjs b/src/core/operations/AdvancedEntropy.mjs index b3316583..26ae25a6 100644 --- a/src/core/operations/AdvancedEntropy.mjs +++ b/src/core/operations/AdvancedEntropy.mjs @@ -30,8 +30,13 @@ class AdvancedEntropy extends Operation { this.infoURL = ""; this.inputType = "byteArray"; this.outputType = "html"; - this.presentType = "html"; - this.args = []; + this.args = [ + { + "name": "Visualization", + "type": "option", + "value": ["Histogram (Bar)", "Histogram (Line)"] + } + ]; } /** @@ -64,66 +69,15 @@ class AdvancedEntropy extends Operation { /** * Calculates the frequency of bytes in the input. * - * @param {byteArray} inputBytes - * @returns {frequency} + * @param {object} svg + * @param {function} xScale + * @param {function} yScale + * @param {integer} svgHeight + * @param {integer} svgWidth + * @param {object} margins + * @returns {undefined} */ - calculateByteFrequency(inputBytes) { - console.log(inputBytes); - - let byteFrequency = []; - for (let i = 0; i < 256; i++) { - let count = 0; - for (let byte of inputBytes) { - if (byte === i) { - count++; - } - } - - byteFrequency.push(count / (inputBytes.length + 1)) - } - - return byteFrequency; - } - - /** - * Creates a byte frequency histogram - * - * @param {byteArray} entropyData - * @returns {HTML} - */ - createByteFrequencyHistogram(entropyData) { - const byteFrequency = entropyData.byteFrequency; - - const svgWidth = 500, - svgHeight = 500, - binWidth = 1; - - const document = new nodom.Document(); - let svg = document.createElement("svg"); - svg = d3.select(svg) - .attr("width", "100%") - .attr("height", "100%") - .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); - - const margins = {top: 30, right: 20, bottom: 50, left: 30}; - - const yScale = d3.scaleLinear() - .domain(d3.extent(byteFrequency, d => d)) - .range([svgHeight - margins.bottom, margins.top]) - - const xScale = d3.scaleLinear() - .domain([0, byteFrequency.length - 1]) - .range([margins.left - binWidth, svgWidth - margins.right]) - - svg.selectAll("rect") - .data(byteFrequency) - .enter().append("rect") - .attr("x", (_, i) => xScale(i) + binWidth) - .attr("y", (dataPoint) => yScale(dataPoint)) - .attr("width", binWidth) - .attr("height", dataPoint => yScale(0) - yScale(dataPoint)) - .attr("fill", "blue") - + createHistogramAxes(svg, xScale, yScale, svgHeight, svgWidth, margins) { // Axes const yAxis = d3.axisLeft() .scale(yScale); @@ -132,7 +86,7 @@ class AdvancedEntropy extends Operation { .scale(xScale) svg.append("g") - .attr("transform", `translate(${binWidth}, ${svgHeight - margins.bottom})`) + .attr("transform", `translate(0, ${svgHeight - margins.bottom})`) .call(xAxis); svg.append("g") @@ -158,6 +112,118 @@ class AdvancedEntropy extends Operation { .attr("transform", `translate(${svgWidth / 2}, ${margins.top - 10})`) .style("text-anchor", "middle") .text("Byte Frequency") + } + + /** + * Calculates the frequency of bytes in the input. + * + * @param {byteArray} inputBytes + * @returns {frequency} + */ + calculateByteFrequency(inputBytes) { + console.log(inputBytes); + + let byteFrequency = []; + for (let i = 0; i < 256; i++) { + let count = 0; + for (let byte of inputBytes) { + if (byte === i) { + count++; + } + } + + byteFrequency.push(count / (inputBytes.length + 1)) + } + + return byteFrequency; + } + + /** + * Calculates the frequency of bytes in the input. + * + * @param {byteArray} input + * @returns {frequency} + */ + createByteFrequencyLineHistogram(entropyData) { + const byteFrequency = entropyData.byteFrequency; + + const margins = {top: 30, right: 20, bottom: 50, left: 30}; + + const svgWidth = 500, + svgHeight = 500; + + const document = new nodom.Document(); + let svg = document.createElement("svg"); + + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + + const yScale = d3.scaleLinear() + .domain(d3.extent(byteFrequency, d => d)) + .range([svgHeight - margins.bottom, margins.top]); + + const xScale = d3.scaleLinear() + .domain([0, byteFrequency.length - 1]) + .range([margins.left, svgWidth - margins.right]); + + const line = d3.line() + .x((d, i) => { return xScale(i)}) + .y((d) => { return yScale(d)}) + .curve(d3.curveMonotoneX); + + svg.append('path') + .datum(byteFrequency) + .attr("class", "line") + .attr("d", line) + .attr("fill", "steelblue"); + + this.createHistogramAxes(svg, xScale, yScale, svgHeight, svgWidth, margins); + + return svg._groups[0][0].outerHTML; + } + + /** + * Creates a byte frequency histogram + * + * @param {byteArray} entropyData + * @returns {HTML} + */ + createByteFrequencyBarHistogram(entropyData) { + const byteFrequency = entropyData.byteFrequency; + + const svgWidth = 500, + svgHeight = 500, + binWidth = 1; + + const document = new nodom.Document(); + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + + const margins = {top: 30, right: 20, bottom: 50, left: 30}; + + const yScale = d3.scaleLinear() + .domain(d3.extent(byteFrequency, d => d)) + .range([svgHeight - margins.bottom, margins.top]); + + const xScale = d3.scaleLinear() + .domain([0, byteFrequency.length - 1]) + .range([margins.left - binWidth, svgWidth - margins.right]); + + svg.selectAll("rect") + .data(byteFrequency) + .enter().append("rect") + .attr("x", (_, i) => xScale(i) + binWidth) + .attr("y", (dataPoint) => yScale(dataPoint)) + .attr("width", binWidth) + .attr("height", dataPoint => yScale(0) - yScale(dataPoint)) + .attr("fill", "blue"); + + this.createHistogramAxes(svg, xScale, yScale, svgHeight, svgWidth, margins); return svg._groups[0][0].outerHTML; } @@ -167,13 +233,19 @@ class AdvancedEntropy extends Operation { * @param {Object[]} args * @returns {html} */ - run(input, args) { + run(input, args) { + const visualizationType = args[0]; + const entropyData = { entropy: this.calculateShannonEntropy(input), byteFrequency: this.calculateByteFrequency(input) }; - let svgData = this.createByteFrequencyHistogram(entropyData) - return svgData + + let svgData; + if (visualizationType === "Histogram (Bar)") svgData = this.createByteFrequencyBarHistogram(entropyData); + else if (visualizationType === "Histogram (Line)") svgData = this.createByteFrequencyLineHistogram(entropyData); + + return svgData; } } From f988a958bb12a61a07f532c9f0fb6dc7578d4953 Mon Sep 17 00:00:00 2001 From: mshwed Date: Sat, 6 Apr 2019 15:59:36 -0400 Subject: [PATCH 07/42] Added support for generating an entropy curve based on the input data --- src/core/operations/AdvancedEntropy.mjs | 90 ++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 9 deletions(-) diff --git a/src/core/operations/AdvancedEntropy.mjs b/src/core/operations/AdvancedEntropy.mjs index 26ae25a6..af76b0de 100644 --- a/src/core/operations/AdvancedEntropy.mjs +++ b/src/core/operations/AdvancedEntropy.mjs @@ -34,7 +34,7 @@ class AdvancedEntropy extends Operation { { "name": "Visualization", "type": "option", - "value": ["Histogram (Bar)", "Histogram (Line)"] + "value": ["Histogram (Bar)", "Histogram (Line)", "Curve"] } ]; } @@ -66,6 +66,26 @@ class AdvancedEntropy extends Operation { return -entropy; } + /** + * + * @param inputBytes + * @returns {entropyData} + */ + calculateScanningEntropy(inputBytes) { + let entropyData = []; + let binSelection = Math.ceil(inputBytes.length / 256); + let binWidth = binSelection < 256 ? 256 : binSelection; + + for (let bytePos = 0; bytePos < inputBytes.length; bytePos+=binWidth) { + const block = inputBytes.slice(bytePos, bytePos+binWidth) + const blockEntropy = this.calculateShannonEntropy(block); + entropyData.push(blockEntropy); + } + + return { entropyData, binWidth }; + } + + /** * Calculates the frequency of bytes in the input. * @@ -75,9 +95,11 @@ class AdvancedEntropy extends Operation { * @param {integer} svgHeight * @param {integer} svgWidth * @param {object} margins + * @param {string} xTitle + * @param {string} yTitle * @returns {undefined} */ - createHistogramAxes(svg, xScale, yScale, svgHeight, svgWidth, margins) { + createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, title, xTitle, yTitle) { // Axes const yAxis = d3.axisLeft() .scale(yScale); @@ -100,18 +122,18 @@ class AdvancedEntropy extends Operation { .attr("x", 0 - (svgHeight / 2)) .attr("dy", "1em") .style("text-anchor", "middle") - .text("Frequency (%)") + .text(yTitle) svg.append("text") .attr("transform", `translate(${svgWidth / 2}, ${svgHeight - margins.bottom + 40})`) .style("text-anchor", "middle") - .text("Byte") + .text(xTitle) // Add title svg.append("text") .attr("transform", `translate(${svgWidth / 2}, ${margins.top - 10})`) .style("text-anchor", "middle") - .text("Byte Frequency") + .text(title) } /** @@ -175,11 +197,10 @@ class AdvancedEntropy extends Operation { svg.append('path') .datum(byteFrequency) - .attr("class", "line") .attr("d", line) .attr("fill", "steelblue"); - this.createHistogramAxes(svg, xScale, yScale, svgHeight, svgWidth, margins); + this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency"); return svg._groups[0][0].outerHTML; } @@ -223,7 +244,57 @@ class AdvancedEntropy extends Operation { .attr("height", dataPoint => yScale(0) - yScale(dataPoint)) .attr("fill", "blue"); - this.createHistogramAxes(svg, xScale, yScale, svgHeight, svgWidth, margins); + this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency"); + + return svg._groups[0][0].outerHTML; + } + + /** + * Creates a byte frequency histogram + * + * @param {byteArray} input + * @param {number} blockSize + * @returns {HTML} + */ + createEntropyCurve(input) { + const { entropyData, binWidth } = this.calculateScanningEntropy(input); + + const svgWidth = 500, + svgHeight = 500; + + const document = new nodom.Document(); + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + + const margins = {top: 30, right: 20, bottom: 50, left: 30}; + + const yScale = d3.scaleLinear() + .domain([0, d3.max(entropyData, d => d)]) + .range([svgHeight - margins.bottom, margins.top]); + + const xScale = d3.scaleLinear() + .domain([0, entropyData.length]) + .range([margins.left, svgWidth - margins.right]); + + const line = d3.line() + .x((d, i) => { return xScale(i)}) + .y((d) => { return yScale(d)}) + .curve(d3.curveMonotoneX); + + if (entropyData.length > 0 ) { + svg.append('path') + .datum(entropyData) + .attr("d", line); + + svg.selectAll("path").attr("fill", "none").attr("stroke", "steelblue"); + } + + this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "Scanning Entropy" , `Block (${binWidth}B)`, "Entropy"); + + console.log('TEST', entropyData); return svg._groups[0][0].outerHTML; } @@ -244,7 +315,8 @@ class AdvancedEntropy extends Operation { let svgData; if (visualizationType === "Histogram (Bar)") svgData = this.createByteFrequencyBarHistogram(entropyData); else if (visualizationType === "Histogram (Line)") svgData = this.createByteFrequencyLineHistogram(entropyData); - + else if (visualizationType === "Curve") svgData = this.createEntropyCurve(input); + return svgData; } } From b7fb9635e59ce5e34a00bb3104ae6ab356dcbb49 Mon Sep 17 00:00:00 2001 From: mshwed Date: Sat, 6 Apr 2019 19:40:07 -0400 Subject: [PATCH 08/42] Added operation for entropy visualization as an image --- src/core/operations/AdvancedEntropy.mjs | 62 ++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/src/core/operations/AdvancedEntropy.mjs b/src/core/operations/AdvancedEntropy.mjs index af76b0de..23d8748b 100644 --- a/src/core/operations/AdvancedEntropy.mjs +++ b/src/core/operations/AdvancedEntropy.mjs @@ -34,7 +34,7 @@ class AdvancedEntropy extends Operation { { "name": "Visualization", "type": "option", - "value": ["Histogram (Bar)", "Histogram (Line)", "Curve"] + "value": ["Histogram (Bar)", "Histogram (Line)", "Curve", "Image"] } ]; } @@ -71,10 +71,11 @@ class AdvancedEntropy extends Operation { * @param inputBytes * @returns {entropyData} */ - calculateScanningEntropy(inputBytes) { - let entropyData = []; - let binSelection = Math.ceil(inputBytes.length / 256); - let binWidth = binSelection < 256 ? 256 : binSelection; + calculateScanningEntropy(inputBytes, binWidth) { + const entropyData = []; + binWidth = binWidth + ? Math.floor(inputBytes.length / binWidth) + : Math.floor(inputBytes.length / 256); for (let bytePos = 0; bytePos < inputBytes.length; bytePos+=binWidth) { const block = inputBytes.slice(bytePos, bytePos+binWidth) @@ -294,7 +295,53 @@ class AdvancedEntropy extends Operation { this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "Scanning Entropy" , `Block (${binWidth}B)`, "Entropy"); - console.log('TEST', entropyData); + return svg._groups[0][0].outerHTML; + } + + /** + * Creates an image representation of the entropy + * + * @param {byteArray} input + * @param {number} blockSize + * @returns {HTML} + */ + createEntropyImage(inputBytes) { + const svgHeight = 100, + svgWidth = 100, + cellSize = 1, + nodes = []; + + const { entropyData } = this.calculateScanningEntropy(inputBytes, svgWidth*svgHeight); + + for (let i = 0; i < entropyData.length; i++) { + nodes.push({ + x: i % svgWidth, + y: Math.floor(i / svgWidth), + entropy: entropyData[i] + }); + } + + const document = new nodom.Document(); + let svg = document.createElement("svg"); + svg = d3.select(svg) + .attr("width", "100%") + .attr("height", "100%") + .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + + const greyScale = d3.scaleLinear() + .domain([0, d3.max(entropyData, d => d)]) + .range(["#000000", "#FFFFFF"]) + .interpolate(d3.interpolateRgb); + + svg + .selectAll("rect") + .data(nodes) + .enter().append("rect") + .attr("x", (d, i) => d.x * cellSize) + .attr("y", (d, i) => d.y * cellSize) + .attr("width", cellSize) + .attr("height", cellSize) + .style("fill", (d) => greyScale(d.entropy)) return svg._groups[0][0].outerHTML; } @@ -316,7 +363,8 @@ class AdvancedEntropy extends Operation { if (visualizationType === "Histogram (Bar)") svgData = this.createByteFrequencyBarHistogram(entropyData); else if (visualizationType === "Histogram (Line)") svgData = this.createByteFrequencyLineHistogram(entropyData); else if (visualizationType === "Curve") svgData = this.createEntropyCurve(input); - + else if (visualizationType === "Image") svgData = this.createEntropyImage(input); + return svgData; } } From 8fc0e012e33c3a63382b08db9e63276371885490 Mon Sep 17 00:00:00 2001 From: mshwed Date: Sat, 6 Apr 2019 23:09:46 -0400 Subject: [PATCH 09/42] Fixed formatting issues --- src/core/operations/AdvancedEntropy.mjs | 134 +++++++++++------------- 1 file changed, 60 insertions(+), 74 deletions(-) diff --git a/src/core/operations/AdvancedEntropy.mjs b/src/core/operations/AdvancedEntropy.mjs index 23d8748b..d7cbd8c1 100644 --- a/src/core/operations/AdvancedEntropy.mjs +++ b/src/core/operations/AdvancedEntropy.mjs @@ -26,8 +26,8 @@ class AdvancedEntropy extends Operation { this.name = "Advanced Entropy"; this.module = "Default"; - this.description = "Adds advanced views for examining entropy"; - this.infoURL = ""; + this.description = "Shannon Entropy, in the context of information theory, is a measure of the rate at which information is produced by a source of data. It can be used, in a broad sense, to detect whether data is likely to be structured or unstructured. 8 is the maximum, representing highly unstructured, 'random' data. English language text usually falls somewhere between 3.5 and 5. Properly encrypted or compressed data should have an entropy of over 7.5."; + this.infoURL = "https://wikipedia.org/wiki/Entropy_(information_theory)"; this.inputType = "byteArray"; this.outputType = "html"; this.args = [ @@ -41,16 +41,16 @@ class AdvancedEntropy extends Operation { /** * Calculates the frequency of bytes in the input. - * + * * @param {byteArray} input * @returns {frequency} */ calculateShannonEntropy(input) { const prob = [], - uniques = input.unique(), - str = Utils.byteArrayToChars(input); - let i; + uniques = input.unique(), + str = Utils.byteArrayToChars(input); + let i; for (i = 0; i < uniques.length; i++) { prob.push(str.count(Utils.chr(uniques[i])) / input.length); } @@ -66,21 +66,20 @@ class AdvancedEntropy extends Operation { return -entropy; } - /** - * - * @param inputBytes + /** + * + * @param inputBytes * @returns {entropyData} */ calculateScanningEntropy(inputBytes, binWidth) { const entropyData = []; - binWidth = binWidth - ? Math.floor(inputBytes.length / binWidth) - : Math.floor(inputBytes.length / 256); - + binWidth = binWidth ? + Math.floor(inputBytes.length / binWidth) : + Math.floor(inputBytes.length / 256); + for (let bytePos = 0; bytePos < inputBytes.length; bytePos+=binWidth) { - const block = inputBytes.slice(bytePos, bytePos+binWidth) - const blockEntropy = this.calculateShannonEntropy(block); - entropyData.push(blockEntropy); + const block = inputBytes.slice(bytePos, bytePos+binWidth); + entropyData.push(this.calculateShannonEntropy(block)); } return { entropyData, binWidth }; @@ -89,7 +88,7 @@ class AdvancedEntropy extends Operation { /** * Calculates the frequency of bytes in the input. - * + * * @param {object} svg * @param {function} xScale * @param {function} yScale @@ -106,7 +105,7 @@ class AdvancedEntropy extends Operation { .scale(yScale); const xAxis = d3.axisBottom() - .scale(xScale) + .scale(xScale); svg.append("g") .attr("transform", `translate(0, ${svgHeight - margins.bottom})`) @@ -123,39 +122,37 @@ class AdvancedEntropy extends Operation { .attr("x", 0 - (svgHeight / 2)) .attr("dy", "1em") .style("text-anchor", "middle") - .text(yTitle) + .text(yTitle); svg.append("text") .attr("transform", `translate(${svgWidth / 2}, ${svgHeight - margins.bottom + 40})`) .style("text-anchor", "middle") - .text(xTitle) + .text(xTitle); // Add title svg.append("text") .attr("transform", `translate(${svgWidth / 2}, ${margins.top - 10})`) .style("text-anchor", "middle") - .text(title) + .text(title); } /** * Calculates the frequency of bytes in the input. - * + * * @param {byteArray} inputBytes * @returns {frequency} */ calculateByteFrequency(inputBytes) { - console.log(inputBytes); - - let byteFrequency = []; + const byteFrequency = []; for (let i = 0; i < 256; i++) { let count = 0; - for (let byte of inputBytes) { + for (const byte of inputBytes) { if (byte === i) { count++; } } - byteFrequency.push(count / (inputBytes.length + 1)) + byteFrequency.push(count / (inputBytes.length + 1)); } return byteFrequency; @@ -163,14 +160,12 @@ class AdvancedEntropy extends Operation { /** * Calculates the frequency of bytes in the input. - * - * @param {byteArray} input + * + * @param {byteArray} byteFrequency * @returns {frequency} */ - createByteFrequencyLineHistogram(entropyData) { - const byteFrequency = entropyData.byteFrequency; - - const margins = {top: 30, right: 20, bottom: 50, left: 30}; + createByteFrequencyLineHistogram(byteFrequency) { + const margins = { top: 30, right: 20, bottom: 50, left: 30 }; const svgWidth = 500, svgHeight = 500; @@ -192,28 +187,28 @@ class AdvancedEntropy extends Operation { .range([margins.left, svgWidth - margins.right]); const line = d3.line() - .x((d, i) => { return xScale(i)}) - .y((d) => { return yScale(d)}) + .x((_, i) => xScale(i)) + .y(d => yScale(d)) .curve(d3.curveMonotoneX); - svg.append('path') + svg.append("path") .datum(byteFrequency) .attr("d", line) .attr("fill", "steelblue"); - this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency"); + this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency"); return svg._groups[0][0].outerHTML; } /** * Creates a byte frequency histogram - * - * @param {byteArray} entropyData + * + * @param {byteArray} byteFrequency * @returns {HTML} */ - createByteFrequencyBarHistogram(entropyData) { - const byteFrequency = entropyData.byteFrequency; + createByteFrequencyBarHistogram(byteFrequency) { + const margins = { top: 30, right: 20, bottom: 50, left: 30 }; const svgWidth = 500, svgHeight = 500, @@ -225,8 +220,6 @@ class AdvancedEntropy extends Operation { .attr("width", "100%") .attr("height", "100%") .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); - - const margins = {top: 30, right: 20, bottom: 50, left: 30}; const yScale = d3.scaleLinear() .domain(d3.extent(byteFrequency, d => d)) @@ -240,7 +233,7 @@ class AdvancedEntropy extends Operation { .data(byteFrequency) .enter().append("rect") .attr("x", (_, i) => xScale(i) + binWidth) - .attr("y", (dataPoint) => yScale(dataPoint)) + .attr("y", dataPoint => yScale(dataPoint)) .attr("width", binWidth) .attr("height", dataPoint => yScale(0) - yScale(dataPoint)) .attr("fill", "blue"); @@ -252,13 +245,13 @@ class AdvancedEntropy extends Operation { /** * Creates a byte frequency histogram - * + * * @param {byteArray} input * @param {number} blockSize * @returns {HTML} */ - createEntropyCurve(input) { - const { entropyData, binWidth } = this.calculateScanningEntropy(input); + createEntropyCurve(entropyData) { + const margins = { top: 30, right: 20, bottom: 50, left: 30 }; const svgWidth = 500, svgHeight = 500; @@ -270,8 +263,6 @@ class AdvancedEntropy extends Operation { .attr("height", "100%") .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); - const margins = {top: 30, right: 20, bottom: 50, left: 30}; - const yScale = d3.scaleLinear() .domain([0, d3.max(entropyData, d => d)]) .range([svgHeight - margins.bottom, margins.top]); @@ -281,37 +272,35 @@ class AdvancedEntropy extends Operation { .range([margins.left, svgWidth - margins.right]); const line = d3.line() - .x((d, i) => { return xScale(i)}) - .y((d) => { return yScale(d)}) + .x((_, i) => xScale(i)) + .y(d => yScale(d)) .curve(d3.curveMonotoneX); - if (entropyData.length > 0 ) { - svg.append('path') + if (entropyData.length > 0) { + svg.append("path") .datum(entropyData) .attr("d", line); svg.selectAll("path").attr("fill", "none").attr("stroke", "steelblue"); } - this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "Scanning Entropy" , `Block (${binWidth}B)`, "Entropy"); + this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "Scanning Entropy", "Block", "Entropy"); return svg._groups[0][0].outerHTML; } /** * Creates an image representation of the entropy - * + * * @param {byteArray} input * @param {number} blockSize * @returns {HTML} */ - createEntropyImage(inputBytes) { + createEntropyImage(entropyData) { const svgHeight = 100, svgWidth = 100, cellSize = 1, nodes = []; - - const { entropyData } = this.calculateScanningEntropy(inputBytes, svgWidth*svgHeight); for (let i = 0; i < entropyData.length; i++) { nodes.push({ @@ -337,11 +326,11 @@ class AdvancedEntropy extends Operation { .selectAll("rect") .data(nodes) .enter().append("rect") - .attr("x", (d, i) => d.x * cellSize) - .attr("y", (d, i) => d.y * cellSize) + .attr("x", d => d.x * cellSize) + .attr("y", d => d.y * cellSize) .attr("width", cellSize) .attr("height", cellSize) - .style("fill", (d) => greyScale(d.entropy)) + .style("fill", d => greyScale(d.entropy)); return svg._groups[0][0].outerHTML; } @@ -351,21 +340,18 @@ class AdvancedEntropy extends Operation { * @param {Object[]} args * @returns {html} */ - run(input, args) { + run(input, args) { const visualizationType = args[0]; - const entropyData = { - entropy: this.calculateShannonEntropy(input), - byteFrequency: this.calculateByteFrequency(input) - }; - - let svgData; - if (visualizationType === "Histogram (Bar)") svgData = this.createByteFrequencyBarHistogram(entropyData); - else if (visualizationType === "Histogram (Line)") svgData = this.createByteFrequencyLineHistogram(entropyData); - else if (visualizationType === "Curve") svgData = this.createEntropyCurve(input); - else if (visualizationType === "Image") svgData = this.createEntropyImage(input); - - return svgData; + if (visualizationType === "Histogram (Bar)") { + return this.createByteFrequencyBarHistogram(this.calculateByteFrequency(input)); + } else if (visualizationType === "Histogram (Line)") { + return this.createByteFrequencyLineHistogram(this.calculateByteFrequency(input)); + } else if (visualizationType === "Curve") { + return this.createEntropyCurve(this.calculateScanningEntropy(input).entropyData); + } else if (visualizationType === "Image") { + return this.createEntropyImage(this.calculateScanningEntropy(input, 10000).entropyData); + } } } From a339eacd450e283926b30ada409c9aac777bd5f9 Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 7 Apr 2019 18:59:03 +0100 Subject: [PATCH 10/42] Bzip2 compression support changed to use wasm backend x4 speed. --- package-lock.json | 47 ++++++++----------------- package.json | 2 +- src/core/operations/Bzip2Decompress.mjs | 38 ++++++++++++++------ 3 files changed, 42 insertions(+), 45 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3e1011f4..96d1ff25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5263,8 +5263,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -5285,14 +5284,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5307,20 +5304,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -5437,8 +5431,7 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -5450,7 +5443,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5465,7 +5457,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5473,14 +5464,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5499,7 +5488,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -5580,8 +5568,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -5593,7 +5580,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -5679,8 +5665,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -5716,7 +5701,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5736,7 +5720,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5780,14 +5763,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, @@ -7905,9 +7886,9 @@ "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, "libbzip2-wasm": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/libbzip2-wasm/-/libbzip2-wasm-0.0.3.tgz", - "integrity": "sha512-eJpB5ITwsdyjWu7DUJirHaBSFLhSbcz8txvL0Gf9NOb+HIzp/lRxFSDXyNAi59ncbhHe3xX+3gj0o+l8rSdJHQ==" + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/libbzip2-wasm/-/libbzip2-wasm-0.0.4.tgz", + "integrity": "sha512-RqscTx95+RTKhFAyjedsboR0Lmo3zd8//EuRwQXkdWmsCwYlzarVRaiYg6kS1O8m10MCQkGdrnlK9L4eAmZUwA==" }, "libyara-wasm": { "version": "0.0.12", diff --git a/package.json b/package.json index d84e7d17..929e038d 100644 --- a/package.json +++ b/package.json @@ -117,7 +117,7 @@ "jsqr": "^1.2.0", "jsrsasign": "8.0.12", "kbpgp": "2.1.0", - "libbzip2-wasm": "0.0.3", + "libbzip2-wasm": "0.0.4", "libyara-wasm": "0.0.12", "lodash": "^4.17.11", "loglevel": "^1.6.1", diff --git a/src/core/operations/Bzip2Decompress.mjs b/src/core/operations/Bzip2Decompress.mjs index 3b357486..2598e207 100644 --- a/src/core/operations/Bzip2Decompress.mjs +++ b/src/core/operations/Bzip2Decompress.mjs @@ -5,8 +5,8 @@ */ import Operation from "../Operation"; -import bzip2 from "../vendor/bzip2"; import OperationError from "../errors/OperationError"; +import Bzip2 from "libbzip2-wasm"; /** * Bzip2 Decompress operation @@ -23,9 +23,15 @@ class Bzip2Decompress extends Operation { this.module = "Compression"; this.description = "Decompresses data using the Bzip2 algorithm."; this.infoURL = "https://wikipedia.org/wiki/Bzip2"; - this.inputType = "byteArray"; - this.outputType = "string"; - this.args = []; + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + this.args = [ + { + name: "Use low-memory, slower decompression algorithm", + type: "boolean", + value: false + } + ]; this.patterns = [ { "match": "^\\x42\\x5a\\x68", @@ -41,14 +47,24 @@ class Bzip2Decompress extends Operation { * @returns {string} */ run(input, args) { - const compressed = new Uint8Array(input); - - try { - const bzip2Reader = bzip2.array(compressed); - return bzip2.simple(bzip2Reader); - } catch (err) { - throw new OperationError(err); + const [small] = args; + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); } + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Loading Bzip2..."); + return new Promise((resolve, reject) => { + Bzip2().then(bzip2 => { + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Decompressing data..."); + const inpArray = new Uint8Array(input); + const bzip2cc = bzip2.decompressBZ2(inpArray, small ? 1 : 0); + if (bzip2cc.error !== 0) { + reject(new OperationError(bzip2cc.error_msg)); + } else { + const output = bzip2cc.output; + resolve(output.buffer.slice(output.byteOffset, output.byteLength + output.byteOffset)); + } + }); + }); } } From 982c915931e41a0a855668f5966e693684f86c7a Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 7 Apr 2019 19:02:27 +0100 Subject: [PATCH 11/42] Change author --- src/core/operations/Bzip2Decompress.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/operations/Bzip2Decompress.mjs b/src/core/operations/Bzip2Decompress.mjs index 2598e207..08a1f3d7 100644 --- a/src/core/operations/Bzip2Decompress.mjs +++ b/src/core/operations/Bzip2Decompress.mjs @@ -1,6 +1,6 @@ /** - * @author n1474335 [n1474335@gmail.com] - * @copyright Crown Copyright 2016 + * @author Matt C [me@mitt.dev] + * @copyright Crown Copyright 2019 * @license Apache-2.0 */ From 18408901bee44bd8b0300bb0c0ad27661649f67f Mon Sep 17 00:00:00 2001 From: Matt Date: Sun, 7 Apr 2019 19:11:46 +0100 Subject: [PATCH 12/42] removed old bzip2 dependency --- src/core/vendor/bzip2.mjs | 265 -------------------------------------- 1 file changed, 265 deletions(-) delete mode 100755 src/core/vendor/bzip2.mjs diff --git a/src/core/vendor/bzip2.mjs b/src/core/vendor/bzip2.mjs deleted file mode 100755 index df0573d8..00000000 --- a/src/core/vendor/bzip2.mjs +++ /dev/null @@ -1,265 +0,0 @@ -/** @license -======================================================================== - bzip2.js - a small bzip2 decompression implementation - - Copyright 2011 by antimatter15 (antimatter15@gmail.com) - - Based on micro-bunzip by Rob Landley (rob@landley.net). - - Copyright (c) 2011 by antimatter15 (antimatter15@gmail.com). - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and associated documentation files (the "Software"), - to deal in the Software without restriction, including without limitation - the rights to use, copy, modify, merge, publish, distribute, sublicense, - and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, - DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH - THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ -"use strict"; - -var bzip2 = {}; - -bzip2.array = function(bytes){ - var bit = 0, byte = 0; - var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF ]; - return function(n){ - var result = 0; - while(n > 0){ - var left = 8 - bit; - if(n >= left){ - result <<= left; - result |= (BITMASK[left] & bytes[byte++]); - bit = 0; - n -= left; - }else{ - result <<= n; - result |= ((bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit)); - bit += n; - n = 0; - } - } - return result - } -} - -bzip2.simple = function(bits){ - var size = bzip2.header(bits); - var all = '', chunk = ''; - do{ - all += chunk; - chunk = bzip2.decompress(bits, size); - }while(chunk != -1); - return all; -} - -bzip2.header = function(bits){ - if(bits(8*3) != 4348520) throw "No magic number found"; - var i = bits(8) - 48; - if(i < 1 || i > 9) throw "Not a BZIP archive"; - return i; -}; - - -//takes a function for reading the block data (starting with 0x314159265359) -//a block size (0-9) (optional, defaults to 9) -//a length at which to stop decompressing and return the output -bzip2.decompress = function(bits, size, len){ - var MAX_HUFCODE_BITS = 20; - var MAX_SYMBOLS = 258; - var SYMBOL_RUNA = 0; - var SYMBOL_RUNB = 1; - var GROUP_SIZE = 50; - - var bufsize = 100000 * size; - for(var h = '', i = 0; i < 6; i++) h += bits(8).toString(16); - if(h == "177245385090") return -1; //last block - if(h != "314159265359") throw "Not valid bzip data"; - bits(32); //ignore CRC codes - if(bits(1)) throw "Unsupported obsolete version"; - var origPtr = bits(24); - if(origPtr > bufsize) throw "Initial position larger than buffer size"; - var t = bits(16); - var symToByte = new Uint8Array(256), - symTotal = 0; - for (i = 0; i < 16; i++) { - if(t & (1 << (15 - i))) { - var k = bits(16); - for(j = 0; j < 16; j++){ - if(k & (1 << (15 - j))){ - symToByte[symTotal++] = (16 * i) + j; - } - } - } - } - - var groupCount = bits(3); - if(groupCount < 2 || groupCount > 6) throw "Error 1"; - var nSelectors = bits(15); - if(nSelectors == 0) throw "Error"; - var mtfSymbol = []; //TODO: possibly replace JS array with typed arrays - for(var i = 0; i < groupCount; i++) mtfSymbol[i] = i; - var selectors = new Uint8Array(32768); - - for(var i = 0; i < nSelectors; i++){ - for(var j = 0; bits(1); j++) if(j >= groupCount) throw "Error 2"; - var uc = mtfSymbol[j]; - mtfSymbol.splice(j, 1); //this is a probably inefficient MTF transform - mtfSymbol.splice(0, 0, uc); - selectors[i] = uc; - } - - var symCount = symTotal + 2; - var groups = []; - for(var j = 0; j < groupCount; j++){ - var length = new Uint8Array(MAX_SYMBOLS), - temp = new Uint8Array(MAX_HUFCODE_BITS+1); - t = bits(5); //lengths - for(var i = 0; i < symCount; i++){ - while(true){ - if (t < 1 || t > MAX_HUFCODE_BITS) throw "Error 3"; - if(!bits(1)) break; - if(!bits(1)) t++; - else t--; - } - length[i] = t; - } - var minLen, maxLen; - minLen = maxLen = length[0]; - for(var i = 1; i < symCount; i++){ - if(length[i] > maxLen) maxLen = length[i]; - else if(length[i] < minLen) minLen = length[i]; - } - var hufGroup; - hufGroup = groups[j] = {}; - hufGroup.permute = new Uint32Array(MAX_SYMBOLS); - hufGroup.limit = new Uint32Array(MAX_HUFCODE_BITS + 1); - hufGroup.base = new Uint32Array(MAX_HUFCODE_BITS + 1); - hufGroup.minLen = minLen; - hufGroup.maxLen = maxLen; - var base = hufGroup.base.subarray(1); - var limit = hufGroup.limit.subarray(1); - var pp = 0; - for(var i = minLen; i <= maxLen; i++) - for(var t = 0; t < symCount; t++) - if(length[t] == i) hufGroup.permute[pp++] = t; - for(i = minLen; i <= maxLen; i++) temp[i] = limit[i] = 0; - for(i = 0; i < symCount; i++) temp[length[i]]++; - pp = t = 0; - for(i = minLen; i < maxLen; i++) { - pp += temp[i]; - limit[i] = pp - 1; - pp <<= 1; - base[i+1] = pp - (t += temp[i]); - } - limit[maxLen]=pp+temp[maxLen]-1; - base[minLen]=0; - } - var byteCount = new Uint32Array(256); - for(var i = 0; i < 256; i++) mtfSymbol[i] = i; - var runPos, count, symCount, selector; - runPos = count = symCount = selector = 0; - var buf = new Uint32Array(bufsize); - while(true){ - if(!(symCount--)){ - symCount = GROUP_SIZE - 1; - if(selector >= nSelectors) throw "Error 4"; - hufGroup = groups[selectors[selector++]]; - base = hufGroup.base.subarray(1); - limit = hufGroup.limit.subarray(1); - } - i = hufGroup.minLen; - j = bits(i); - while(true){ - if(i > hufGroup.maxLen) throw "Error 5"; - if(j <= limit[i]) break; - i++; - j = (j << 1) | bits(1); - } - j -= base[i]; - if(j < 0 || j >= MAX_SYMBOLS) throw "Error 6"; - var nextSym = hufGroup.permute[j]; - if (nextSym == SYMBOL_RUNA || nextSym == SYMBOL_RUNB) { - if(!runPos){ - runPos = 1; - t = 0; - } - if(nextSym == SYMBOL_RUNA) t += runPos; - else t += 2 * runPos; - runPos <<= 1; - continue; - } - if(runPos){ - runPos = 0; - if(count + t >= bufsize) throw "Error 7"; - uc = symToByte[mtfSymbol[0]]; - byteCount[uc] += t; - while(t--) buf[count++] = uc; - } - if(nextSym > symTotal) break; - if(count >= bufsize) throw "Error 8"; - i = nextSym -1; - uc = mtfSymbol[i]; - mtfSymbol.splice(i, 1); - mtfSymbol.splice(0, 0, uc); - uc = symToByte[uc]; - byteCount[uc]++; - buf[count++] = uc; - } - if(origPtr < 0 || origPtr >= count) throw "Error 9"; - var j = 0; - for(var i = 0; i < 256; i++){ - k = j + byteCount[i]; - byteCount[i] = j; - j = k; - } - for(var i = 0; i < count; i++){ - uc = buf[i] & 0xff; - buf[byteCount[uc]] |= (i << 8); - byteCount[uc]++; - } - var pos = 0, current = 0, run = 0; - if(count) { - pos = buf[origPtr]; - current = (pos & 0xff); - pos >>= 8; - run = -1; - } - count = count; - var output = ''; - var copies, previous, outbyte; - if(!len) len = Infinity; - while(count){ - count--; - previous = current; - pos = buf[pos]; - current = pos & 0xff; - pos >>= 8; - if(run++ == 3){ - copies = current; - outbyte = previous; - current = -1; - }else{ - copies = 1; - outbyte = current; - } - while(copies--){ - output += (String.fromCharCode(outbyte)); - if(!--len) return output; - } - if(current != previous) run = 0; - } - return output; -} - -export default bzip2; From 5225874498d8ef15eafce5e5389d40f96c54ad4a Mon Sep 17 00:00:00 2001 From: mshwed Date: Sun, 28 Apr 2019 14:38:03 -0400 Subject: [PATCH 13/42] Fixed handling of large files and fixed issue with line histogram colour fill --- src/core/operations/AdvancedEntropy.mjs | 33 ++++++++++++++----------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/core/operations/AdvancedEntropy.mjs b/src/core/operations/AdvancedEntropy.mjs index d7cbd8c1..e60816ac 100644 --- a/src/core/operations/AdvancedEntropy.mjs +++ b/src/core/operations/AdvancedEntropy.mjs @@ -73,9 +73,9 @@ class AdvancedEntropy extends Operation { */ calculateScanningEntropy(inputBytes, binWidth) { const entropyData = []; - binWidth = binWidth ? - Math.floor(inputBytes.length / binWidth) : - Math.floor(inputBytes.length / 256); + + if (inputBytes.length < 256) binWidth = 8; + else binWidth = 256; for (let bytePos = 0; bytePos < inputBytes.length; bytePos+=binWidth) { const block = inputBytes.slice(bytePos, bytePos+binWidth); @@ -145,14 +145,15 @@ class AdvancedEntropy extends Operation { calculateByteFrequency(inputBytes) { const byteFrequency = []; for (let i = 0; i < 256; i++) { - let count = 0; - for (const byte of inputBytes) { - if (byte === i) { - count++; + if (inputBytes.length > 0) { + let count = 0; + for (const byte of inputBytes) { + if (byte === i) count++; } + byteFrequency.push(count / inputBytes.length); + } else { + byteFrequency.push(0); } - - byteFrequency.push(count / (inputBytes.length + 1)); } return byteFrequency; @@ -179,7 +180,7 @@ class AdvancedEntropy extends Operation { .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); const yScale = d3.scaleLinear() - .domain(d3.extent(byteFrequency, d => d)) + .domain([0, d3.max(byteFrequency, d => d)]) .range([svgHeight - margins.bottom, margins.top]); const xScale = d3.scaleLinear() @@ -193,8 +194,9 @@ class AdvancedEntropy extends Operation { svg.append("path") .datum(byteFrequency) - .attr("d", line) - .attr("fill", "steelblue"); + .attr("fill", "none") + .attr("stroke", "steelblue") + .attr("d", line); this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency"); @@ -221,8 +223,9 @@ class AdvancedEntropy extends Operation { .attr("height", "100%") .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`); + const yExtent = d3.extent(byteFrequency, d => d); const yScale = d3.scaleLinear() - .domain(d3.extent(byteFrequency, d => d)) + .domain(yExtent) .range([svgHeight - margins.bottom, margins.top]); const xScale = d3.scaleLinear() @@ -235,7 +238,7 @@ class AdvancedEntropy extends Operation { .attr("x", (_, i) => xScale(i) + binWidth) .attr("y", dataPoint => yScale(dataPoint)) .attr("width", binWidth) - .attr("height", dataPoint => yScale(0) - yScale(dataPoint)) + .attr("height", dataPoint => yScale(yExtent[0]) - yScale(dataPoint)) .attr("fill", "blue"); this.createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, "", "Byte", "Byte Frequency"); @@ -350,7 +353,7 @@ class AdvancedEntropy extends Operation { } else if (visualizationType === "Curve") { return this.createEntropyCurve(this.calculateScanningEntropy(input).entropyData); } else if (visualizationType === "Image") { - return this.createEntropyImage(this.calculateScanningEntropy(input, 10000).entropyData); + return this.createEntropyImage(this.calculateScanningEntropy(input).entropyData); } } } From 8fa8e3402703b309b3c2e83b3cf4c0660ad137f4 Mon Sep 17 00:00:00 2001 From: mshwed Date: Sun, 28 Apr 2019 16:29:15 -0400 Subject: [PATCH 14/42] Added support for parsing JSON with number type values. Added support for non-array JSON objects. Added extra tests for JSON to CSV operation. --- src/core/operations/JSONToCSV.mjs | 6 ++ tests/operations/index.mjs | 1 + tests/operations/tests/JSONtoCSV.mjs | 93 ++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 tests/operations/tests/JSONtoCSV.mjs diff --git a/src/core/operations/JSONToCSV.mjs b/src/core/operations/JSONToCSV.mjs index c3d078e3..a20d1728 100644 --- a/src/core/operations/JSONToCSV.mjs +++ b/src/core/operations/JSONToCSV.mjs @@ -51,6 +51,10 @@ class JSONToCSV extends Operation { this.rowDelim = rowDelim; const self = this; + if (!(input instanceof Array)) { + input = [input]; + } + try { // If the JSON is an array of arrays, this is easy if (input[0] instanceof Array) { @@ -89,6 +93,8 @@ class JSONToCSV extends Operation { * @returns {string} */ escapeCellContents(data) { + if (typeof data === "number") data = data.toString(); + // Double quotes should be doubled up data = data.replace(/"/g, '""'); diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 41d78c35..8129d4eb 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -52,6 +52,7 @@ import "./tests/Image"; import "./tests/Jump"; import "./tests/JSONBeautify"; import "./tests/JSONMinify"; +import "./tests/JSONtoCSV"; import "./tests/JWTDecode"; import "./tests/JWTSign"; import "./tests/JWTVerify"; diff --git a/tests/operations/tests/JSONtoCSV.mjs b/tests/operations/tests/JSONtoCSV.mjs new file mode 100644 index 00000000..67d46e64 --- /dev/null +++ b/tests/operations/tests/JSONtoCSV.mjs @@ -0,0 +1,93 @@ +/** + * JSON to CSV tests. + * + * @author mshwed [m@ttshwed.com] + * + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import TestRegister from "../TestRegister"; + +const EXPECTED_CSV_SINGLE = "a,b,c\r\n1,2,3\r\n"; +const EXPECTED_CSV_MULTIPLE = "a,b,c\r\n1,2,3\r\n1,2,3\r\n"; +const EXPECTED_CSV_EMPTY = "\r\n\r\n"; + +TestRegister.addTests([ + { + name: "JSON to CSV: strings as values", + input: JSON.stringify({a: "1", b: "2", c: "3"}), + expectedOutput: EXPECTED_CSV_SINGLE, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: numbers as values", + input: JSON.stringify({a: 1, b: 2, c: 3}), + expectedOutput: EXPECTED_CSV_SINGLE, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: numbers and strings as values", + input: JSON.stringify({a: 1, b: "2", c: 3}), + expectedOutput: EXPECTED_CSV_SINGLE, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: JSON as an array", + input: JSON.stringify([{a: 1, b: "2", c: 3}]), + expectedOutput: EXPECTED_CSV_SINGLE, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: multiple JSON values in an array", + input: JSON.stringify([{a: 1, b: "2", c: 3}, {a: 1, b: "2", c: 3}]), + expectedOutput: EXPECTED_CSV_MULTIPLE, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: empty JSON", + input: JSON.stringify({}), + expectedOutput: EXPECTED_CSV_EMPTY, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + }, + { + name: "JSON to CSV: empty JSON in array", + input: JSON.stringify([{}]), + expectedOutput: EXPECTED_CSV_EMPTY, + recipeConfig: [ + { + op: "JSON to CSV", + args: [",", "\\r\\n"] + }, + ], + } +]); From 1d130c88a82ca7bfc50408c02f39b0e4aa46f94d Mon Sep 17 00:00:00 2001 From: h345983745 Date: Sun, 19 May 2019 15:40:17 +0100 Subject: [PATCH 15/42] Inital commit --- .github/ISSUE_TEMPLATE.md | 15 +-------- .github/ISSUE_TEMPLATE/bug_report.md | 35 +++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++++ .github/ISSUE_TEMPLATE/operation-request.md | 16 ++++++++++ 4 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/operation-request.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 8e7dee82..e90ab51f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -1,14 +1 @@ - - - - - - -### Summary - - -### Example - - - - + diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..b293da9a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,35 @@ +--- +name: Bug report +about: Create a report to help us improve +title: 'Bug report: ' +labels: bug +assignees: '' + +--- + + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior or a link to the recipe / input used to cause the bug: + +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (if relevant, please complete the following information):** + - OS: [e.g. Windows] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..9a38f0e5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for the project +title: 'Feature request: ' +labels: feature +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/operation-request.md b/.github/ISSUE_TEMPLATE/operation-request.md new file mode 100644 index 00000000..03a45a14 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/operation-request.md @@ -0,0 +1,16 @@ +--- +name: Operation request +about: Suggest an operation +title: 'Operation request: ' +labels: operation +assignees: '' + +--- + + + +## Summary + +### Example Input + +### Example Output From 466d872d30305c2b8ab7c42ad22769e57a3b4dff Mon Sep 17 00:00:00 2001 From: George O <16269580+Ge0rg3@users.noreply.github.com> Date: Sat, 8 Jun 2019 21:43:30 +0100 Subject: [PATCH 16/42] Added Index of Coincidence Operation --- src/core/config/Categories.json | 1 + src/core/operations/IndexOfCoincidence.mjs | 107 +++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 src/core/operations/IndexOfCoincidence.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 2d194c37..102d76aa 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -395,6 +395,7 @@ "ops": [ "Entropy", "Frequency distribution", + "Index of Coincidence", "Chi Square", "Disassemble x86", "Pseudo-Random Number Generator", diff --git a/src/core/operations/IndexOfCoincidence.mjs b/src/core/operations/IndexOfCoincidence.mjs new file mode 100644 index 00000000..8edf5e33 --- /dev/null +++ b/src/core/operations/IndexOfCoincidence.mjs @@ -0,0 +1,107 @@ +/** + * @author George O [georgeomnet+cyberchef@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import Utils from "../Utils"; + +/** + * Index of Coincidence operation + */ +class IndexOfCoincidence extends Operation { + + /** + * IndexOfCoincidence constructor + */ + constructor() { + super(); + + this.name = "Index of Coincidence"; + this.module = "Default"; + this.description = "Index of Coincidence (IC) is the probability of two randomly selected characters being the same. This can be used to determine whether text is readable or random, with English text having an IC of around 0.066. IC can therefore be a sound method to automate frequency analysis."; + this.infoURL = "https://wikipedia.org/wiki/Index_of_coincidence"; + this.inputType = "string"; + this.outputType = "number"; + this.presentType = "html"; + this.args = []; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {number} + */ + run(input, args) { + const text = input.toLowerCase().replace(/[^a-z]/g, ""), + frequencies = new Array(26).fill(0), + alphabet = Utils.expandAlphRange("a-z"); + let coincidence = 0.00, + density = 0.00, + result = 0.00, + i; + + for (i=0; i < alphabet.length; i++) { + frequencies[i] = text.count(alphabet[i]); + } + + for (i=0; i < frequencies.length; i++) { + coincidence += frequencies[i] * (frequencies[i] - 1); + } + + density = frequencies.sum(); + + // Ensure that we don't divide by 0 + if (density < 2) density = 2; + + result = coincidence / (density * (density - 1)); + + return result; + } + + /** + * Displays the IC as a scale bar for web apps. + * + * @param {number} ic + * @returns {html} + */ + present(ic) { + return `Index of Coincidence: ${ic} +Normalized: ${ic * 26} +

+- 0 represents complete randomness (all characters are unique), whereas 1 represents no randomness (all characters are identical). +- English text generally has an IC of between 0.67 to 0.78. +- 'Random' text is determined by the probability that each letter occurs the same number of times as another. + +The graph shows the IC of the input data. A low IC generally means that the text is random, encoded or encrypted. + + + `; + } + +} + +export default IndexOfCoincidence; From a6732ba81524fc34e2ce58a1dff7304f3ac1c31f Mon Sep 17 00:00:00 2001 From: George O <16269580+Ge0rg3@users.noreply.github.com> Date: Sat, 8 Jun 2019 21:49:31 +0100 Subject: [PATCH 17/42] Added Index of Coincidence Tests --- tests/operations/index.mjs | 1 + tests/operations/tests/IndexOfCoincidence.mjs | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 tests/operations/tests/IndexOfCoincidence.mjs diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 41d78c35..e40a5bd9 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -49,6 +49,7 @@ import "./tests/Hash"; import "./tests/HaversineDistance"; import "./tests/Hexdump"; import "./tests/Image"; +import "./tests/IndexOfCoincidence"; import "./tests/Jump"; import "./tests/JSONBeautify"; import "./tests/JSONMinify"; diff --git a/tests/operations/tests/IndexOfCoincidence.mjs b/tests/operations/tests/IndexOfCoincidence.mjs new file mode 100644 index 00000000..3dc4cd35 --- /dev/null +++ b/tests/operations/tests/IndexOfCoincidence.mjs @@ -0,0 +1,22 @@ +/** + * Index of Coincidence tests. + * + * @author George O [georgeomnet+cyberchef@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import TestRegister from "../TestRegister"; + +TestRegister.addTests([ + { + name: "Index of Coincidence", + input: "Hello world, this is a test to determine the correct IC value.", + expectedMatch: /^Index of Coincidence: 0\.07142857142857142\nNormalized: 1\.857142857142857/, + recipeConfig: [ + { + "op": "Index of Coincidence", + "args": [] + }, + ], + }, +]); From e7095820629b27cccbbaf81a5509de2780a666b6 Mon Sep 17 00:00:00 2001 From: George O <16269580+Ge0rg3@users.noreply.github.com> Date: Mon, 10 Jun 2019 00:06:15 +0100 Subject: [PATCH 18/42] Disappearing Popover Fix --- src/web/RecipeWaiter.mjs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/web/RecipeWaiter.mjs b/src/web/RecipeWaiter.mjs index f2edd725..b8b523dc 100755 --- a/src/web/RecipeWaiter.mjs +++ b/src/web/RecipeWaiter.mjs @@ -124,16 +124,21 @@ class RecipeWaiter { * @param {event} evt */ opSortEnd(evt) { - if (this.removeIntent) { - if (evt.item.parentNode.id === "rec-list") { - evt.item.remove(); - } + if (this.removeIntent && evt.item.parentNode.id === "rec-list") { + evt.item.remove(); return; } // Reinitialise the popover on the original element in the ops list because for some reason it - // gets destroyed and recreated. - this.manager.ops.enableOpsListPopovers(evt.clone); + // gets destroyed and recreated. If the clone isn't in the ops list, we use the original item instead. + let enableOpsElement; + if (evt.clone.parentNode && evt.clone.parentNode.classList.contains("op-list")) { + enableOpsElement = evt.clone; + } else { + enableOpsElement = evt.item; + $(evt.item).attr("data-toggle", "popover"); + } + this.manager.ops.enableOpsListPopovers(enableOpsElement); if (evt.item.parentNode.id !== "rec-list") { return; From f29d8eeda89a01219d3a449eae1cd697555f4fcb Mon Sep 17 00:00:00 2001 From: wh0 Date: Tue, 25 Jun 2019 18:35:25 -0700 Subject: [PATCH 19/42] Parse octal escape sequences --- src/core/Utils.mjs | 11 +++++++++-- src/core/operations/UnescapeString.mjs | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/core/Utils.mjs b/src/core/Utils.mjs index e4411fe0..273eca1e 100755 --- a/src/core/Utils.mjs +++ b/src/core/Utils.mjs @@ -201,11 +201,18 @@ class Utils { * Utils.parseEscapedChars("\\n"); */ static parseEscapedChars(str) { - return str.replace(/(\\)?\\([bfnrtv0'"]|x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]{1,6}\})/g, function(m, a, b) { + return str.replace(/(\\)?\\([bfnrtv'"]|[0-3][0-7]{2}|[0-7]{1,2}|x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]{1,6}\})/g, function(m, a, b) { if (a === "\\") return "\\"+b; switch (b[0]) { case "0": - return "\0"; + case "1": + case "2": + case "3": + case "4": + case "5": + case "6": + case "7": + return String.fromCharCode(parseInt(b, 8)); case "b": return "\b"; case "t": diff --git a/src/core/operations/UnescapeString.mjs b/src/core/operations/UnescapeString.mjs index 62eab48e..1a625582 100644 --- a/src/core/operations/UnescapeString.mjs +++ b/src/core/operations/UnescapeString.mjs @@ -20,7 +20,7 @@ class UnescapeString extends Operation { this.name = "Unescape string"; this.module = "Default"; - this.description = "Unescapes characters in a string that have been escaped. For example, Don\\'t stop me now becomes Don't stop me now.

Supports the following escape sequences:
  • \\n (Line feed/newline)
  • \\r (Carriage return)
  • \\t (Horizontal tab)
  • \\b (Backspace)
  • \\f (Form feed)
  • \\xnn (Hex, where n is 0-f)
  • \\\\ (Backslash)
  • \\' (Single quote)
  • \\" (Double quote)
  • \\unnnn (Unicode character)
  • \\u{nnnnnn} (Unicode code point)
"; + this.description = "Unescapes characters in a string that have been escaped. For example, Don\\'t stop me now becomes Don't stop me now.

Supports the following escape sequences:
  • \\n (Line feed/newline)
  • \\r (Carriage return)
  • \\t (Horizontal tab)
  • \\b (Backspace)
  • \\f (Form feed)
  • \\nnn (Octal, where n is 0-7)
  • \\xnn (Hex, where n is 0-f)
  • \\\\ (Backslash)
  • \\' (Single quote)
  • \\" (Double quote)
  • \\unnnn (Unicode character)
  • \\u{nnnnnn} (Unicode code point)
"; this.infoURL = "https://wikipedia.org/wiki/Escape_sequence"; this.inputType = "string"; this.outputType = "string"; From ccf2348cd6cd8fc9ca11f01ecbf933b1085ecf60 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 27 Jun 2019 15:34:22 +0100 Subject: [PATCH 20/42] 8.31.7 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 88652e86..96135781 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.31.6", + "version": "8.31.7", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index ed1d1cd3..a47f521a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.31.6", + "version": "8.31.7", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 393d070b0544dec864177e868a41bf6a9cd44b78 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 27 Jun 2019 15:37:12 +0100 Subject: [PATCH 21/42] 8.31.8 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 96135781..067887df 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.31.7", + "version": "8.31.8", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index a47f521a..c6801479 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.31.7", + "version": "8.31.8", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From e11aec64cd2b8d6f20ebb3b4f6e0d8b179376437 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 27 Jun 2019 15:42:32 +0100 Subject: [PATCH 22/42] Modified wording for IC op --- src/core/operations/IndexOfCoincidence.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/IndexOfCoincidence.mjs b/src/core/operations/IndexOfCoincidence.mjs index 8edf5e33..4a417769 100644 --- a/src/core/operations/IndexOfCoincidence.mjs +++ b/src/core/operations/IndexOfCoincidence.mjs @@ -74,7 +74,7 @@ Normalized: ${ic * 26} - English text generally has an IC of between 0.67 to 0.78. - 'Random' text is determined by the probability that each letter occurs the same number of times as another. -The graph shows the IC of the input data. A low IC generally means that the text is random, encoded or encrypted. +The graph shows the IC of the input data. A low IC generally means that the text is random, compressed or encrypted. `; + CanvasComponents.drawScaleBar(canvas, entropy, 8, [ + { + label: "English text", + min: 3.5, + max: 5 + },{ + label: "Encrypted/compressed", + min: 7.5, + max: 8 + } + ]); + `; } + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {html} + */ + run(input, args) { + const visualizationType = args[0]; + + if (visualizationType === 'Shannon') { + return this.createShannonEntropyVisualization(this.calculateShannonEntropy(input)); + } else if (visualizationType === "Histogram (Bar)") { + return this.createByteFrequencyBarHistogram(this.calculateByteFrequency(input)); + } else if (visualizationType === "Histogram (Line)") { + return this.createByteFrequencyLineHistogram(this.calculateByteFrequency(input)); + } else if (visualizationType === "Curve") { + return this.createEntropyCurve(this.calculateScanningEntropy(input).entropyData); + } else if (visualizationType === "Image") { + return this.createEntropyImage(this.calculateScanningEntropy(input).entropyData); + } + } } export default Entropy; From b0b6de116d6e512644d33295dc542505a8f1b27d Mon Sep 17 00:00:00 2001 From: mshwed Date: Thu, 27 Jun 2019 14:11:26 -0400 Subject: [PATCH 35/42] Fixed linting issue --- src/core/operations/Entropy.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Entropy.mjs b/src/core/operations/Entropy.mjs index a205d757..850b8d1a 100644 --- a/src/core/operations/Entropy.mjs +++ b/src/core/operations/Entropy.mjs @@ -383,7 +383,7 @@ class Entropy extends Operation { run(input, args) { const visualizationType = args[0]; - if (visualizationType === 'Shannon') { + if (visualizationType === "Shannon") { return this.createShannonEntropyVisualization(this.calculateShannonEntropy(input)); } else if (visualizationType === "Histogram (Bar)") { return this.createByteFrequencyBarHistogram(this.calculateByteFrequency(input)); From c1ad2386ef165b9104a9cda951f24464b24325a2 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 28 Jun 2019 15:00:19 +0100 Subject: [PATCH 36/42] Improvements to Entropy operation. Converted to ArrayBuffers, improved efficiency with large files, added present method back in. --- package-lock.json | 239 ++++++++++++++++++++++---------- src/core/operations/Entropy.mjs | 136 +++++++++++------- 2 files changed, 245 insertions(+), 130 deletions(-) diff --git a/package-lock.json b/package-lock.json index e2936c2a..982a60ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5346,24 +5346,29 @@ "dependencies": { "abbrev": { "version": "1.1.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true, "optional": true }, "ansi-regex": { "version": "2.1.1", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "dev": true, "optional": true }, "are-we-there-yet": { "version": "1.1.5", - "bundled": true, + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", "dev": true, "optional": true, "requires": { @@ -5373,13 +5378,17 @@ }, "balanced-match": { "version": "1.0.0", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", - "bundled": true, + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5387,34 +5396,43 @@ }, "chownr": { "version": "1.1.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", + "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==", "dev": true, "optional": true }, "code-point-at": { "version": "1.1.0", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", - "bundled": true, + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "dev": true, "optional": true }, "debug": { "version": "4.1.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "dev": true, "optional": true, "requires": { @@ -5423,25 +5441,29 @@ }, "deep-extend": { "version": "0.6.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", "dev": true, "optional": true }, "delegates": { "version": "1.0.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", "dev": true, "optional": true }, "detect-libc": { "version": "1.0.3", - "bundled": true, + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", "dev": true, "optional": true }, "fs-minipass": { "version": "1.2.5", - "bundled": true, + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz", + "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==", "dev": true, "optional": true, "requires": { @@ -5450,13 +5472,15 @@ }, "fs.realpath": { "version": "1.0.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true, "optional": true }, "gauge": { "version": "2.7.4", - "bundled": true, + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", "dev": true, "optional": true, "requires": { @@ -5472,7 +5496,8 @@ }, "glob": { "version": "7.1.3", - "bundled": true, + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", "dev": true, "optional": true, "requires": { @@ -5486,13 +5511,15 @@ }, "has-unicode": { "version": "2.0.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", "dev": true, "optional": true }, "iconv-lite": { "version": "0.4.24", - "bundled": true, + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "dev": true, "optional": true, "requires": { @@ -5501,7 +5528,8 @@ }, "ignore-walk": { "version": "3.0.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.1.tgz", + "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==", "dev": true, "optional": true, "requires": { @@ -5510,7 +5538,8 @@ }, "inflight": { "version": "1.0.6", - "bundled": true, + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "dev": true, "optional": true, "requires": { @@ -5520,46 +5549,58 @@ }, "inherits": { "version": "2.0.3", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", - "bundled": true, + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "dev": true, "optional": true }, "is-fullwidth-code-point": { "version": "1.0.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } }, "isarray": { "version": "1.0.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", "dev": true, "optional": true }, "minimatch": { "version": "3.0.4", - "bundled": true, + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { "version": "0.0.8", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true, + "optional": true }, "minipass": { "version": "2.3.5", - "bundled": true, + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", + "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -5567,7 +5608,8 @@ }, "minizlib": { "version": "1.2.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz", + "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==", "dev": true, "optional": true, "requires": { @@ -5576,21 +5618,25 @@ }, "mkdirp": { "version": "0.5.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } }, "ms": { "version": "2.1.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", "dev": true, "optional": true }, "needle": { "version": "2.3.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/needle/-/needle-2.3.0.tgz", + "integrity": "sha512-QBZu7aAFR0522EyaXZM0FZ9GLpq6lvQ3uq8gteiDUp7wKdy0lSd2hPlgFwVuW1CBkfEs9PfDQsQzZghLs/psdg==", "dev": true, "optional": true, "requires": { @@ -5601,7 +5647,8 @@ }, "node-pre-gyp": { "version": "0.12.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.12.0.tgz", + "integrity": "sha512-4KghwV8vH5k+g2ylT+sLTjy5wmUOb9vPhnM8NHvRf9dHmnW/CndrFXy2aRPaPST6dugXSdHXfeaHQm77PIz/1A==", "dev": true, "optional": true, "requires": { @@ -5619,7 +5666,8 @@ }, "nopt": { "version": "4.0.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", + "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", "dev": true, "optional": true, "requires": { @@ -5629,13 +5677,15 @@ }, "npm-bundled": { "version": "1.0.6", - "bundled": true, + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.6.tgz", + "integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g==", "dev": true, "optional": true }, "npm-packlist": { "version": "1.4.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.1.tgz", + "integrity": "sha512-+TcdO7HJJ8peiiYhvPxsEDhF3PJFGUGRcFsGve3vxvxdcpO2Z4Z7rkosRM0kWj6LfbK/P0gu3dzk5RU1ffvFcw==", "dev": true, "optional": true, "requires": { @@ -5645,7 +5695,8 @@ }, "npmlog": { "version": "4.1.2", - "bundled": true, + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", "dev": true, "optional": true, "requires": { @@ -5657,38 +5708,46 @@ }, "number-is-nan": { "version": "1.0.1", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", "dev": true, "optional": true }, "once": { "version": "1.4.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "dev": true, + "optional": true, "requires": { "wrappy": "1" } }, "os-homedir": { "version": "1.0.2", - "bundled": true, + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true, "optional": true }, "os-tmpdir": { "version": "1.0.2", - "bundled": true, + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true, "optional": true }, "osenv": { "version": "0.1.5", - "bundled": true, + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", "dev": true, "optional": true, "requires": { @@ -5698,19 +5757,22 @@ }, "path-is-absolute": { "version": "1.0.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true, "optional": true }, "process-nextick-args": { "version": "2.0.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", "dev": true, "optional": true }, "rc": { "version": "1.2.8", - "bundled": true, + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", "dev": true, "optional": true, "requires": { @@ -5722,7 +5784,8 @@ "dependencies": { "minimist": { "version": "1.2.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true, "optional": true } @@ -5730,7 +5793,8 @@ }, "readable-stream": { "version": "2.3.6", - "bundled": true, + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "optional": true, "requires": { @@ -5745,7 +5809,8 @@ }, "rimraf": { "version": "2.6.3", - "bundled": true, + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", "dev": true, "optional": true, "requires": { @@ -5754,43 +5819,52 @@ }, "safe-buffer": { "version": "5.1.2", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", - "bundled": true, + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "dev": true, "optional": true }, "sax": { "version": "1.2.4", - "bundled": true, + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", "dev": true, "optional": true }, "semver": { "version": "5.7.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", + "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", "dev": true, "optional": true }, "set-blocking": { "version": "2.0.0", - "bundled": true, + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "dev": true, "optional": true }, "signal-exit": { "version": "3.0.2", - "bundled": true, + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "dev": true, "optional": true }, "string-width": { "version": "1.0.2", - "bundled": true, + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5799,7 +5873,8 @@ }, "string_decoder": { "version": "1.1.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "optional": true, "requires": { @@ -5808,21 +5883,25 @@ }, "strip-ansi": { "version": "3.0.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } }, "strip-json-comments": { "version": "2.0.1", - "bundled": true, + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", "dev": true, "optional": true }, "tar": { "version": "4.4.8", - "bundled": true, + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz", + "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==", "dev": true, "optional": true, "requires": { @@ -5837,13 +5916,15 @@ }, "util-deprecate": { "version": "1.0.2", - "bundled": true, + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", "dev": true, "optional": true }, "wide-align": { "version": "1.1.3", - "bundled": true, + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", "dev": true, "optional": true, "requires": { @@ -5852,13 +5933,17 @@ }, "wrappy": { "version": "1.0.2", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true, + "optional": true }, "yallist": { "version": "3.0.3", - "bundled": true, - "dev": true + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", + "dev": true, + "optional": true } } }, diff --git a/src/core/operations/Entropy.mjs b/src/core/operations/Entropy.mjs index 850b8d1a..10f23423 100644 --- a/src/core/operations/Entropy.mjs +++ b/src/core/operations/Entropy.mjs @@ -8,7 +8,6 @@ import * as d3temp from "d3"; import * as nodomtemp from "nodom"; import Operation from "../Operation"; -import Utils from "../Utils"; const d3 = d3temp.default ? d3temp.default : d3temp; const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp; @@ -25,16 +24,17 @@ class Entropy extends Operation { super(); this.name = "Entropy"; - this.module = "Default"; + this.module = "Charts"; this.description = "Shannon Entropy, in the context of information theory, is a measure of the rate at which information is produced by a source of data. It can be used, in a broad sense, to detect whether data is likely to be structured or unstructured. 8 is the maximum, representing highly unstructured, 'random' data. English language text usually falls somewhere between 3.5 and 5. Properly encrypted or compressed data should have an entropy of over 7.5."; this.infoURL = "https://wikipedia.org/wiki/Entropy_(information_theory)"; - this.inputType = "byteArray"; - this.outputType = "html"; + this.inputType = "ArrayBuffer"; + this.outputType = "json"; + this.presentType = "html"; this.args = [ { "name": "Visualisation", "type": "option", - "value": ["Shannon", "Histogram (Bar)", "Histogram (Line)", "Curve", "Image"] + "value": ["Shannon scale", "Histogram (Bar)", "Histogram (Line)", "Curve", "Image"] } ]; } @@ -42,19 +42,27 @@ class Entropy extends Operation { /** * Calculates the frequency of bytes in the input. * - * @param {byteArray} input - * @returns {frequency} + * @param {Uint8Array} input + * @returns {number} */ calculateShannonEntropy(input) { const prob = [], - uniques = input.unique(), - str = Utils.byteArrayToChars(input); + occurrences = new Array(256).fill(0); + // Count occurrences of each byte in the input let i; - for (i = 0; i < uniques.length; i++) { - prob.push(str.count(Utils.chr(uniques[i])) / input.length); + for (i = 0; i < input.length; i++) { + occurrences[input[i]]++; } + // Store probability list + for (i = 0; i < occurrences.length; i++) { + if (occurrences[i] > 0) { + prob.push(occurrences[i] / input.length); + } + } + + // Calculate Shannon entropy let entropy = 0, p; @@ -67,17 +75,16 @@ class Entropy extends Operation { } /** + * Calculates the scanning entropy of the input * - * @param inputBytes - * @returns {entropyData} + * @param {Uint8Array} inputBytes + * @returns {Object} */ - calculateScanningEntropy(inputBytes, binWidth) { + calculateScanningEntropy(inputBytes) { const entropyData = []; + const binWidth = inputBytes.length < 256 ? 8 : 256; - if (inputBytes.length < 256) binWidth = 8; - else binWidth = 256; - - for (let bytePos = 0; bytePos < inputBytes.length; bytePos+=binWidth) { + for (let bytePos = 0; bytePos < inputBytes.length; bytePos += binWidth) { const block = inputBytes.slice(bytePos, bytePos+binWidth); entropyData.push(this.calculateShannonEntropy(block)); } @@ -96,7 +103,6 @@ class Entropy extends Operation { * @param {object} margins * @param {string} xTitle * @param {string} yTitle - * @returns {undefined} */ createAxes(svg, xScale, yScale, svgHeight, svgWidth, margins, title, xTitle, yTitle) { // Axes @@ -138,31 +144,31 @@ class Entropy extends Operation { /** * Calculates the frequency of bytes in the input. * - * @param {byteArray} inputBytes - * @returns {frequency} + * @param {Uint8Array} inputBytes + * @returns {number[]} */ calculateByteFrequency(inputBytes) { - const byteFrequency = []; - for (let i = 0; i < 256; i++) { - if (inputBytes.length > 0) { - let count = 0; - for (const byte of inputBytes) { - if (byte === i) count++; - } - byteFrequency.push(count / inputBytes.length); - } else { - byteFrequency.push(0); - } + const freq = new Array(256).fill(0); + if (inputBytes.length === 0) return freq; + + // Count occurrences of each byte in the input + let i; + for (i = 0; i < inputBytes.length; i++) { + freq[inputBytes[i]]++; } - return byteFrequency; + for (i = 0; i < freq.length; i++) { + freq[i] = freq[i] / inputBytes.length; + } + + return freq; } /** * Calculates the frequency of bytes in the input. * - * @param {byteArray} byteFrequency - * @returns {frequency} + * @param {number[]} byteFrequency + * @returns {HTML} */ createByteFrequencyLineHistogram(byteFrequency) { const margins = { top: 30, right: 20, bottom: 50, left: 30 }; @@ -205,7 +211,7 @@ class Entropy extends Operation { /** * Creates a byte frequency histogram * - * @param {byteArray} byteFrequency + * @param {number[]} byteFrequency * @returns {HTML} */ createByteFrequencyBarHistogram(byteFrequency) { @@ -248,8 +254,7 @@ class Entropy extends Operation { /** * Creates a byte frequency histogram * - * @param {byteArray} input - * @param {number} blockSize + * @param {number[]} entropyData * @returns {HTML} */ createEntropyCurve(entropyData) { @@ -294,8 +299,7 @@ class Entropy extends Operation { /** * Creates an image representation of the entropy * - * @param {byteArray} input - * @param {number} blockSize + * @param {number[]} entropyData * @returns {HTML} */ createEntropyImage(entropyData) { @@ -341,7 +345,7 @@ class Entropy extends Operation { * Displays the entropy as a scale bar for web apps. * * @param {number} entropy - * @returns {html} + * @returns {HTML} */ createShannonEntropyVisualization(entropy) { return `Shannon entropy: ${entropy} @@ -376,23 +380,49 @@ class Entropy extends Operation { } /** - * @param {byteArray} input + * @param {ArrayBuffer} input * @param {Object[]} args - * @returns {html} + * @returns {json} */ run(input, args) { const visualizationType = args[0]; + input = new Uint8Array(input); - if (visualizationType === "Shannon") { - return this.createShannonEntropyVisualization(this.calculateShannonEntropy(input)); - } else if (visualizationType === "Histogram (Bar)") { - return this.createByteFrequencyBarHistogram(this.calculateByteFrequency(input)); - } else if (visualizationType === "Histogram (Line)") { - return this.createByteFrequencyLineHistogram(this.calculateByteFrequency(input)); - } else if (visualizationType === "Curve") { - return this.createEntropyCurve(this.calculateScanningEntropy(input).entropyData); - } else if (visualizationType === "Image") { - return this.createEntropyImage(this.calculateScanningEntropy(input).entropyData); + switch (visualizationType) { + case "Histogram (Bar)": + case "Histogram (Line)": + return this.calculateByteFrequency(input); + case "Curve": + case "Image": + return this.calculateScanningEntropy(input).entropyData; + case "Shannon scale": + default: + return this.calculateShannonEntropy(input); + } + } + + /** + * Displays the entropy in a visualisation for web apps. + * + * @param {json} entropyData + * @param {Object[]} args + * @returns {html} + */ + present(entropyData, args) { + const visualizationType = args[0]; + + switch (visualizationType) { + case "Histogram (Bar)": + return this.createByteFrequencyBarHistogram(entropyData); + case "Histogram (Line)": + return this.createByteFrequencyLineHistogram(entropyData); + case "Curve": + return this.createEntropyCurve(entropyData); + case "Image": + return this.createEntropyImage(entropyData); + case "Shannon scale": + default: + return this.createShannonEntropyVisualization(entropyData); } } } From d637ac7633d0ef54a179aa8b692ad1db6aa91e1b Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 28 Jun 2019 15:03:49 +0100 Subject: [PATCH 37/42] Updated CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f530c69..b63ff85e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.34.0] - 2019-06-28 +- Various new visualisations added to the 'Entropy' operation [@MShwed] | [#535] +- Efficiency improvements made to the 'Entropy' operation for large file support [@n1474335] + ### [8.33.0] - 2019-06-27 - 'Bzip2 Compress' operation added and 'Bzip2 Decompress' operation greatly improved [@artemisbot] | [#531] @@ -136,6 +140,7 @@ All major and minor version changes will be documented in this file. Details of +[8.34.0]: https://github.com/gchq/CyberChef/releases/tag/v8.34.0 [8.33.0]: https://github.com/gchq/CyberChef/releases/tag/v8.33.0 [8.32.0]: https://github.com/gchq/CyberChef/releases/tag/v8.32.0 [8.31.0]: https://github.com/gchq/CyberChef/releases/tag/v8.31.0 @@ -198,6 +203,7 @@ All major and minor version changes will be documented in this file. Details of [@anthony-arnold]: https://github.com/anthony-arnold [@masq]: https://github.com/masq [@Ge0rg3]: https://github.com/Ge0rg3 +[@MShwed]: https://github.com/MShwed [#95]: https://github.com/gchq/CyberChef/pull/299 [#173]: https://github.com/gchq/CyberChef/pull/173 @@ -242,4 +248,5 @@ All major and minor version changes will be documented in this file. Details of [#525]: https://github.com/gchq/CyberChef/pull/525 [#531]: https://github.com/gchq/CyberChef/pull/531 [#533]: https://github.com/gchq/CyberChef/pull/533 +[#535]: https://github.com/gchq/CyberChef/pull/535 [#571]: https://github.com/gchq/CyberChef/pull/571 From a5ea7f7d58047cc0672c94279ed4efdab5b39845 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 28 Jun 2019 15:03:55 +0100 Subject: [PATCH 38/42] 8.34.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 982a60ec..b3e679b7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.33.1", + "version": "8.34.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 6635e9bd..92336768 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.33.1", + "version": "8.34.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 6f6786d79e98259b81722624cb959ccee74e7208 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 28 Jun 2019 17:09:00 +0100 Subject: [PATCH 39/42] Baking controls now scale to fit the pane correctly --- src/web/App.mjs | 6 +++--- src/web/RecipeWaiter.mjs | 17 +++++++++++++++++ src/web/html/index.html | 2 +- src/web/stylesheets/layout/_controls.css | 8 ++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/web/App.mjs b/src/web/App.mjs index 8019d483..868684de 100755 --- a/src/web/App.mjs +++ b/src/web/App.mjs @@ -244,7 +244,7 @@ class App { /** * Sets up the adjustable splitter to allow the user to resize areas of the page. * - * @param {boolean} [minimise=false] - Set this flag if attempting to minimuse frames to 0 width + * @param {boolean} [minimise=false] - Set this flag if attempting to minimise frames to 0 width */ initialiseSplitter(minimise=false) { if (this.columnSplitter) this.columnSplitter.destroy(); @@ -252,9 +252,9 @@ class App { this.columnSplitter = Split(["#operations", "#recipe", "#IO"], { sizes: [20, 30, 50], - minSize: minimise ? [0, 0, 0] : [240, 370, 450], + minSize: minimise ? [0, 0, 0] : [240, 310, 450], gutterSize: 4, - expandToMin: false, + expandToMin: true, onDrag: function() { this.manager.recipe.adjustWidth(); }.bind(this) diff --git a/src/web/RecipeWaiter.mjs b/src/web/RecipeWaiter.mjs index b8b523dc..b408beb0 100755 --- a/src/web/RecipeWaiter.mjs +++ b/src/web/RecipeWaiter.mjs @@ -617,6 +617,23 @@ class RecipeWaiter { ingredientRule.style.gridTemplateColumns = "auto auto auto auto"; ingredientChildRule.style.gridColumn = "1 / span 4"; } + + // Hide Chef icon on Bake button if the page is compressed + const bakeIcon = document.querySelector("#bake img"); + + if (recList.clientWidth < 370) { + // Hide Chef icon on Bake button + bakeIcon.style.display = "none"; + } else { + bakeIcon.style.display = "inline-block"; + } + + // Scale controls to fit pane width + const controls = document.getElementById("controls"); + const controlsContent = document.getElementById("controls-content"); + const scale = (controls.clientWidth - 1) / controlsContent.scrollWidth; + + controlsContent.style.transform = `translate(-50%, -50%) scale(${scale})`; } } diff --git a/src/web/html/index.html b/src/web/html/index.html index 700cc24c..350c4a37 100755 --- a/src/web/html/index.html +++ b/src/web/html/index.html @@ -191,7 +191,7 @@
    -
    +
    diff --git a/src/web/stylesheets/layout/_controls.css b/src/web/stylesheets/layout/_controls.css index 380cd284..8f4cdf0b 100755 --- a/src/web/stylesheets/layout/_controls.css +++ b/src/web/stylesheets/layout/_controls.css @@ -21,6 +21,14 @@ background-color: var(--secondary-background-colour); } +#controls-content { + position: relative; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + transform-origin: center left; +} + #auto-bake-label { display: inline-block; width: 100px; From f9c24f252862bb84352a4770a9cc5d7639e18378 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 28 Jun 2019 17:09:04 +0100 Subject: [PATCH 40/42] 8.34.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b3e679b7..e75caa11 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.34.0", + "version": "8.34.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 92336768..856a456c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.34.0", + "version": "8.34.1", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From f22211ce8c804aea3826059be66937eec23a2a0e Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 28 Jun 2019 17:13:54 +0100 Subject: [PATCH 41/42] Backslashes are now escaped correctly by 'Unescape string'. CLoses #582 --- src/core/Utils.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/Utils.mjs b/src/core/Utils.mjs index 273eca1e..fed67553 100755 --- a/src/core/Utils.mjs +++ b/src/core/Utils.mjs @@ -201,9 +201,11 @@ class Utils { * Utils.parseEscapedChars("\\n"); */ static parseEscapedChars(str) { - return str.replace(/(\\)?\\([bfnrtv'"]|[0-3][0-7]{2}|[0-7]{1,2}|x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]{1,6}\})/g, function(m, a, b) { + return str.replace(/(\\)?\\([bfnrtv'"]|[0-3][0-7]{2}|[0-7]{1,2}|x[\da-fA-F]{2}|u[\da-fA-F]{4}|u\{[\da-fA-F]{1,6}\}|\\)/g, function(m, a, b) { if (a === "\\") return "\\"+b; switch (b[0]) { + case "\\": + return "\\"; case "0": case "1": case "2": From d148cae8140dda9b6996532b66ec98f32ad8fe2f Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 28 Jun 2019 17:13:58 +0100 Subject: [PATCH 42/42] 8.34.2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index e75caa11..a5fb311e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.34.1", + "version": "8.34.2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 856a456c..765b0cfb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.34.1", + "version": "8.34.2", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef",