From 25ca8d85a6ddeb185b46ada43c0183972f5f73ba Mon Sep 17 00:00:00 2001 From: n1073645 Date: Thu, 21 Nov 2019 11:14:56 +0000 Subject: [PATCH 01/18] Added extractor for OLE2 and modified the PLIST one. --- src/core/lib/FileSignatures.mjs | 110 +++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 3 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index dc7ced4d..e151a844 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -2914,15 +2914,119 @@ export function extractSQLITE(bytes, offset) { export function extractPListXML(bytes, offset) { const stream = new Stream(bytes.slice(offset)); - // Find closing tag () - stream.continueUntil([0x3c, 0x2f, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x3e]); - stream.moveForwardsBy(8); + let braceCount = 0; + + // Continue to the first ( 0 && stream.hasMore()) { + if (stream.readInt(1) === 0x3c) { + + // If we hit an . + if (stream.getBytes(7).join("") === [0x2f, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x3e].join("")) { + braceCount--; + } else { + stream.moveBackwardsBy(7); + } + } + } stream.consumeIf(0x0a); return stream.carve(); } +/** + * OLE2 extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractOLE2(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + const entries = [[[0x52, 0x00, 0x6f, 0x00, 0x6f, 0x00, 0x74, 0x00, 0x20, 0x00, 0x45, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x72, 0x00, 0x79], 19, "Root Entry"], + [[0x57, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6b, 0x00, 0x62, 0x00, 0x6f, 0x00, 0x6f, 0x00, 0x6b], 15, "Workbook"], + [[0x43, 0x00, 0x75, 0x00, 0x72, 0x00, 0x72, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x20, 0x00, 0x55, 0x00, 0x73, 0x00, 0x65, 0x00, 0x72], 23, "Current User"], + [[0x50, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x65, 0x00, 0x72, 0x00, 0x50, 0x00, 0x6f, 0x00, 0x69, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x20, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74], 37, "PowerPoint Document"], + [[0x57, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x64, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74], 23, "WordDocument"], + [[0x44, 0x00, 0x61, 0x00, 0x74, 0x00, 0x61], 7, "Data"], + [[0x50, 0x00, 0x69, 0x00, 0x63, 0x00, 0x74, 0x00, 0x75, 0x00, 0x72, 0x00, 0x65, 0x00, 0x73], 15, "Pictures"], + [[0x31, 0x00, 0x54, 0x00, 0x61, 0x00, 0x62, 0x00, 0x6c, 0x00, 0x65], 11, "1Table"], + [[0x05, 0x00, 0x53, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x72, 0x00, 0x79, 0x00, 0x49, 0x00, 0x6e, 0x00, 0x66, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x74, 0x00, 0x69, 0x00, 0x6f, 0x00, 0x6e], 37, "SummaryInformation"], + [[0x05, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x53, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x72, 0x00, 0x79, 0x00, 0x49, 0x00, 0x6e, 0x00, 0x66, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x74, 0x00, 0x69, 0x00, 0x6f, 0x00, 0x6e], 53, "DocumentSummaryInformation"], + [[0x43, 0x00, 0x6f, 0x00, 0x6d, 0x00, 0x70, 0x00, 0x4f, 0x00, 0x62, 0x00, 0x6a], 13, "Comp Obj"], + [[0x01, 0x00], 2, "Entry"]]; + let endianness = "le"; + + // Move to endianess field. + stream.moveForwardsBy(28); + if (stream.readInt(2, endianness) === 0xfffe) + endianness = "le"; + + // Calculate the size of the normal sectors. + const sizeOfSector = 2 ** stream.readInt(2, endianness); + + // Move to root directory offset field. + stream.moveTo(48); + + // Read root directory offset. + const rootStuff = stream.readInt(4, endianness); + + // Calculate root directory offset. + let total = 512 + (rootStuff * sizeOfSector); + stream.moveTo(total); + let found = true; + + // While valid directory entries. + while (found) { + found = false; + + // Attempt to determine what directory entry it is. + for (const element of entries) { + if (stream.getBytes(element[1]).join("") === element[0].join("")) { + stream.moveBackwardsBy(element[1]); + found = true; + + // Move forwards by the size of the comp obj. + if (element[2] === "Comp Obj") { + total += (128*6); + stream.moveTo(total); + } else if (element[2] === "Entry") { + + // If there is an entry move backwards by 126 to then move forwards by 128. Hence a total displacement of 2. + stream.moveBackwardsBy(126); + } + break; + } + stream.moveBackwardsBy(element[1]); + } + + // If we have found a valid entry, move forwards by 128. + if (found) { + total += 128; + stream.moveForwardsBy(128); + } + } + + // Round up to a multiple of 512. + total = Math.ceil(total / 512) * 512; + + stream.moveTo(total); + return stream.carve(); +} + + /** * GZIP extractor. * From 7386c145ef8d22b8a1e7334c006d2ee58eee43f2 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Thu, 21 Nov 2019 11:23:28 +0000 Subject: [PATCH 02/18] Comments for OLE2 extractor. --- src/core/lib/FileSignatures.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index e151a844..656df190 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -2994,6 +2994,8 @@ export function extractOLE2(bytes, offset) { // Attempt to determine what directory entry it is. for (const element of entries) { + + // If the byte pattern matches. if (stream.getBytes(element[1]).join("") === element[0].join("")) { stream.moveBackwardsBy(element[1]); found = true; From 071c1bdea6af3131e98faf86f5a00bf8418ebe8b Mon Sep 17 00:00:00 2001 From: n1073645 Date: Thu, 21 Nov 2019 11:29:45 +0000 Subject: [PATCH 03/18] Comments for OLE2 extractor. --- src/core/lib/FileSignatures.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 656df190..1683a950 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -3016,6 +3016,8 @@ export function extractOLE2(bytes, offset) { // If we have found a valid entry, move forwards by 128. if (found) { + + // Every entry is at least 128 in size, some are bigger which is dealt with by the above if statement. total += 128; stream.moveForwardsBy(128); } From 725b0d42f89e15f1ca3727cafb094167d4c53b94 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Thu, 21 Nov 2019 11:34:11 +0000 Subject: [PATCH 04/18] Comments to OLE2 extractor --- src/core/lib/FileSignatures.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 1683a950..d997f823 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -3002,6 +3002,8 @@ export function extractOLE2(bytes, offset) { // Move forwards by the size of the comp obj. if (element[2] === "Comp Obj") { + + // The size of the Comp Obj entry - 128. Since we add 128 later. total += (128*6); stream.moveTo(total); } else if (element[2] === "Entry") { From 09e93b4639c407dc019fa786cfbd883159b42683 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Mon, 25 Nov 2019 11:26:31 +0000 Subject: [PATCH 05/18] Added ICO extractor --- src/core/lib/FileSignatures.mjs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index dc7ced4d..41f870af 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -280,7 +280,7 @@ export const FILE_SIGNATURES = { 9: 0x0, 10: [0x0, 0x1] }, - extractor: null + extractor: extractICO }, { name: "Radiance High Dynamic Range image", @@ -2772,6 +2772,32 @@ export function extractBMP(bytes, offset) { } +/** + * ICO extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + */ +export function extractICO(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move to number of file there are. + stream.moveTo(4); + + // Read the number of files stored in the ICO + const numberFiles = stream.readInt(2, "le"); + + // Move forward to the last file header. + stream.moveForwardsBy(8 + ((numberFiles-1) * 16)); + const fileSize = stream.readInt(4, "le"); + const fileOffset = stream.readInt(4, "le"); + + // Move to the end of the last file. + stream.moveTo(fileOffset + fileSize); + return stream.carve(); +} + + /** * WAV extractor. * From 1118ff598d6ecf93792a439609e6c5a4224652d9 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Mon, 25 Nov 2019 13:43:31 +0000 Subject: [PATCH 06/18] From Base85 and From Braille signatures added for magic --- src/core/operations/FromBase85.mjs | 17 +++++++++++++++++ src/core/operations/FromBraille.mjs | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/src/core/operations/FromBase85.mjs b/src/core/operations/FromBase85.mjs index c874d5dc..001f4f69 100644 --- a/src/core/operations/FromBase85.mjs +++ b/src/core/operations/FromBase85.mjs @@ -33,6 +33,23 @@ class FromBase85 extends Operation { value: ALPHABET_OPTIONS }, ]; + this.patterns = [ + { + match: "^\\s*(<~)?([!-u]{4})+([!-u]{1,3})??(~>)?\\s*$", + flags: "", + args: ["!-u", true] + }, + { + match: "^\\s*(<~)?([0-9A-Z.-:+=^!/*?&<>()[]{}@%$#]{4})+([0-9A-Z.-:+=^!/*?&<>()[]{}@%$#]{1,3})??(~>)?\\s*$", + flags: "i", + args: ["0-9a-zA-Z.-:+=^!/*?&<>()[]{}@%$#", true] + }, + { + match: "^\\s*(<~)?([0-9A-Z.-:+=^!/*?&_<>()[]{}@%$#;`|~]{4})+([0-9A-Z.-:+=^!/*?&_<>()[]{}@%$#;`|~]{1,3})??(~>)?\\s*$", + flags: "i", + args: ["0-9A-Za-z!#$%&()*+-;<=>?@^_`{|~}", true] + } + ]; } /** diff --git a/src/core/operations/FromBraille.mjs b/src/core/operations/FromBraille.mjs index adbcff91..60e590a3 100644 --- a/src/core/operations/FromBraille.mjs +++ b/src/core/operations/FromBraille.mjs @@ -25,6 +25,13 @@ class FromBraille extends Operation { this.inputType = "string"; this.outputType = "string"; this.args = []; + this.patterns = [ + { + match: "^\\s*[⠀⠁⠂⠃⠄⠅⠆⠇⠈⠉⠊⠋⠌⠍⠎⠏⠐⠑⠒⠓⠔⠕⠖⠗⠘⠙⠚⠛⠜⠝⠞⠟⠠⠡⠢⠣⠤⠥⠦⠧⠨⠩⠪⠫⠬⠭⠮⠯⠰⠱⠲⠳⠴⠵⠶⠷⠸⠹⠺⠻⠼⠽⠾⠿]+\\s*$", + flags: "i", + args: [true] + } + ]; } /** From 47ccafcbb29cd34dafd2c157ceda326144a6e9c2 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Thu, 5 Dec 2019 09:47:32 +0000 Subject: [PATCH 07/18] Linting and tidy up --- src/core/operations/FromBase85.mjs | 17 ----------------- src/core/operations/FromBraille.mjs | 7 ------- 2 files changed, 24 deletions(-) diff --git a/src/core/operations/FromBase85.mjs b/src/core/operations/FromBase85.mjs index 001f4f69..c874d5dc 100644 --- a/src/core/operations/FromBase85.mjs +++ b/src/core/operations/FromBase85.mjs @@ -33,23 +33,6 @@ class FromBase85 extends Operation { value: ALPHABET_OPTIONS }, ]; - this.patterns = [ - { - match: "^\\s*(<~)?([!-u]{4})+([!-u]{1,3})??(~>)?\\s*$", - flags: "", - args: ["!-u", true] - }, - { - match: "^\\s*(<~)?([0-9A-Z.-:+=^!/*?&<>()[]{}@%$#]{4})+([0-9A-Z.-:+=^!/*?&<>()[]{}@%$#]{1,3})??(~>)?\\s*$", - flags: "i", - args: ["0-9a-zA-Z.-:+=^!/*?&<>()[]{}@%$#", true] - }, - { - match: "^\\s*(<~)?([0-9A-Z.-:+=^!/*?&_<>()[]{}@%$#;`|~]{4})+([0-9A-Z.-:+=^!/*?&_<>()[]{}@%$#;`|~]{1,3})??(~>)?\\s*$", - flags: "i", - args: ["0-9A-Za-z!#$%&()*+-;<=>?@^_`{|~}", true] - } - ]; } /** diff --git a/src/core/operations/FromBraille.mjs b/src/core/operations/FromBraille.mjs index 60e590a3..adbcff91 100644 --- a/src/core/operations/FromBraille.mjs +++ b/src/core/operations/FromBraille.mjs @@ -25,13 +25,6 @@ class FromBraille extends Operation { this.inputType = "string"; this.outputType = "string"; this.args = []; - this.patterns = [ - { - match: "^\\s*[⠀⠁⠂⠃⠄⠅⠆⠇⠈⠉⠊⠋⠌⠍⠎⠏⠐⠑⠒⠓⠔⠕⠖⠗⠘⠙⠚⠛⠜⠝⠞⠟⠠⠡⠢⠣⠤⠥⠦⠧⠨⠩⠪⠫⠬⠭⠮⠯⠰⠱⠲⠳⠴⠵⠶⠷⠸⠹⠺⠻⠼⠽⠾⠿]+\\s*$", - flags: "i", - args: [true] - } - ]; } /** From 928178716aeb81c02b58b474d7d6059746525f95 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Mon, 16 Dec 2019 14:46:06 +0000 Subject: [PATCH 08/18] Operation elements now have decreasing z-index properties, meaning dropdowns do not get hidden. Fixes #925 --- src/web/waiters/RecipeWaiter.mjs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/web/waiters/RecipeWaiter.mjs b/src/web/waiters/RecipeWaiter.mjs index ba0e7b11..4f8290f4 100755 --- a/src/web/waiters/RecipeWaiter.mjs +++ b/src/web/waiters/RecipeWaiter.mjs @@ -51,6 +51,7 @@ class RecipeWaiter { } }.bind(this), onSort: function(evt) { + this.updateZIndices(); if (evt.from.id === "rec-list") { document.dispatchEvent(this.manager.statechange); } @@ -149,6 +150,19 @@ class RecipeWaiter { } + /** + * Sets the z-index property on each operation to make sure that operations higher in the list + * have a higher index, meaning dropdowns are not hidden underneath subsequent operations. + */ + updateZIndices() { + const operations = document.getElementById("rec-list").children; + for (let i = 0; i < operations.length; i++) { + const operation = operations[i]; + operation.style.zIndex = 100 + operations.length - i; + } + } + + /** * Handler for favourite dragover events. * If the element being dragged is an operation, displays a visual cue so that the user knows it can @@ -466,6 +480,7 @@ class RecipeWaiter { log.debug(`'${e.target.querySelector(".op-title").textContent}' added to recipe`); this.triggerArgEvents(e.target); + this.updateZIndices(); window.dispatchEvent(this.manager.statechange); } From 3a1a6a94d26a96ee5d02c23845a7511813992ee5 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Mon, 16 Dec 2019 17:05:06 +0000 Subject: [PATCH 09/18] Sets the gzip comment bitfield --- src/core/operations/Gunzip.mjs | 10 ++++++---- src/core/operations/Gzip.mjs | 16 ++++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/core/operations/Gunzip.mjs b/src/core/operations/Gunzip.mjs index 07b1d6c2..ace795be 100644 --- a/src/core/operations/Gunzip.mjs +++ b/src/core/operations/Gunzip.mjs @@ -5,9 +5,11 @@ */ import Operation from "../Operation.mjs"; -import zlibAndGzip from "zlibjs/bin/zlib_and_gzip.min.js"; +// import zlibAndGzip from "zlibjs/bin/zlib_and_gzip.min.js"; +import gunzip from "zlibjs/bin/gunzip.min.js"; -const Zlib = zlibAndGzip.Zlib; +// const Zlib = zlibAndGzip.Zlib; +const Zlib = gunzip.Zlib; /** * Gunzip operation @@ -42,8 +44,8 @@ class Gunzip extends Operation { * @returns {File} */ run(input, args) { - const gunzip = new Zlib.Gunzip(new Uint8Array(input)); - return new Uint8Array(gunzip.decompress()).buffer; + const gzipObj = new Zlib.Gunzip(new Uint8Array(input)); + return new Uint8Array(gzipObj.decompress()).buffer; } } diff --git a/src/core/operations/Gzip.mjs b/src/core/operations/Gzip.mjs index 5f9fa474..b0ed9b53 100644 --- a/src/core/operations/Gzip.mjs +++ b/src/core/operations/Gzip.mjs @@ -6,9 +6,10 @@ import Operation from "../Operation.mjs"; import {COMPRESSION_TYPE, ZLIB_COMPRESSION_TYPE_LOOKUP} from "../lib/Zlib.mjs"; -import zlibAndGzip from "zlibjs/bin/zlib_and_gzip.min.js"; +// import zlibAndGzip from "zlibjs/bin/zlib_and_gzip.min.js"; +import gzip from "zlibjs/bin/gzip.min.js"; -const Zlib = zlibAndGzip.Zlib; +const Zlib = gzip.Zlib; /** * Gzip operation @@ -73,12 +74,15 @@ class Gzip extends Operation { options.filename = filename; } if (comment.length) { - options.flags.fcommenct = true; + options.flags.comment = true; options.comment = comment; } - - const gzip = new Zlib.Gzip(new Uint8Array(input), options); - return new Uint8Array(gzip.compress()).buffer; + const gzipObj = new Zlib.Gzip(new Uint8Array(input), options); + const compressed = new Uint8Array(gzipObj.compress()); + if (options.flags.comment) { + compressed[3] |= 0x10; + } + return compressed.buffer; } } From 5fd2512a9b353dd2dce1d50408c7d4ad997384d4 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Tue, 17 Dec 2019 12:15:11 +0000 Subject: [PATCH 10/18] Gzip tests added --- src/core/operations/Gzip.mjs | 3 +- tests/operations/index.mjs | 1 + tests/operations/tests/Gzip.mjs | 89 +++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 tests/operations/tests/Gzip.mjs diff --git a/src/core/operations/Gzip.mjs b/src/core/operations/Gzip.mjs index b0ed9b53..093ae6a4 100644 --- a/src/core/operations/Gzip.mjs +++ b/src/core/operations/Gzip.mjs @@ -6,7 +6,6 @@ import Operation from "../Operation.mjs"; import {COMPRESSION_TYPE, ZLIB_COMPRESSION_TYPE_LOOKUP} from "../lib/Zlib.mjs"; -// import zlibAndGzip from "zlibjs/bin/zlib_and_gzip.min.js"; import gzip from "zlibjs/bin/gzip.min.js"; const Zlib = gzip.Zlib; @@ -79,7 +78,7 @@ class Gzip extends Operation { } const gzipObj = new Zlib.Gzip(new Uint8Array(input), options); const compressed = new Uint8Array(gzipObj.compress()); - if (options.flags.comment) { + if (options.flags.comment && !(compressed[3] & 0x10)) { compressed[3] |= 0x10; } return compressed.buffer; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index c54fa7ef..09ebf0fc 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -41,6 +41,7 @@ import "./tests/DateTime.mjs"; import "./tests/ExtractEmailAddresses.mjs"; import "./tests/Fork.mjs"; import "./tests/FromDecimal.mjs"; +import "./tests/Gzip.mjs"; import "./tests/Hash.mjs"; import "./tests/HaversineDistance.mjs"; import "./tests/Hexdump.mjs"; diff --git a/tests/operations/tests/Gzip.mjs b/tests/operations/tests/Gzip.mjs new file mode 100644 index 00000000..2b055ae8 --- /dev/null +++ b/tests/operations/tests/Gzip.mjs @@ -0,0 +1,89 @@ +/** + * Gzip tests. + * + * @author n1073645 [n1073645@gmail.com] + * + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "Gzip: No comment, no checksum and no filename", + input: "The quick brown fox jumped over the slow dog", + expectedOutput: "0dc9dd0180200804e0556ea8262848fb3dc588c6a7e76faa8aeedb726036c68d951f76bf9a0af8aae1f97d9c0c084b02509cbf8c2c000000", + recipeConfig: [ + { + op: "Gzip", + args: ["Dynamic Huffman Coding", "", "", false] + }, + { + op: "Drop bytes", + args: [0, 10, false] + }, + { + op: "To Hex", + args: ["None"] + } + ] + }, + { + name: "Gzip: No comment, no checksum and has a filename", + input: "The quick brown fox jumped over the slow dog", + expectedOutput: "636f6d6d656e74000dc9dd0180200804e0556ea8262848fb3dc588c6a7e76faa8aeedb726036c68d951f76bf9a0af8aae1f97d9c0c084b02509cbf8c2c000000", + recipeConfig: [ + { + op: "Gzip", + args: ["Dynamic Huffman Coding", "comment", "", false] + }, + { + op: "Drop bytes", + args: [0, 10, false] + }, + { + op: "To Hex", + args: ["None"] + } + ] + }, + { + name: "Gzip: Has a comment, no checksum and no filename", + input: "The quick brown fox jumped over the slow dog", + expectedOutput: "636f6d6d656e74000dc9dd0180200804e0556ea8262848fb3dc588c6a7e76faa8aeedb726036c68d951f76bf9a0af8aae1f97d9c0c084b02509cbf8c2c000000", + recipeConfig: [ + { + op: "Gzip", + args: ["Dynamic Huffman Coding", "", "comment", false] + }, + { + op: "Drop bytes", + args: [0, 10, false] + }, + { + op: "To Hex", + args: ["None"] + } + ] + }, + { + name: "Gzip: Has a comment, no checksum and has a filename", + input: "The quick brown fox jumped over the slow dog", + expectedOutput: "66696c656e616d6500636f6d6d656e74000dc9dd0180200804e0556ea8262848fb3dc588c6a7e76faa8aeedb726036c68d951f76bf9a0af8aae1f97d9c0c084b02509cbf8c2c000000", + recipeConfig: [ + { + op: "Gzip", + args: ["Dynamic Huffman Coding", "filename", "comment", false] + }, + { + op: "Drop bytes", + args: [0, 10, false] + }, + { + op: "To Hex", + args: ["None"] + } + ] + }, +]); From 72ba579e1e2200d8ec7930d3525ab74aad71cff1 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Tue, 17 Dec 2019 12:17:13 +0000 Subject: [PATCH 11/18] Remove unnecessary comments. --- src/core/operations/Gunzip.mjs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/operations/Gunzip.mjs b/src/core/operations/Gunzip.mjs index ace795be..ef487b06 100644 --- a/src/core/operations/Gunzip.mjs +++ b/src/core/operations/Gunzip.mjs @@ -5,10 +5,8 @@ */ import Operation from "../Operation.mjs"; -// import zlibAndGzip from "zlibjs/bin/zlib_and_gzip.min.js"; import gunzip from "zlibjs/bin/gunzip.min.js"; -// const Zlib = zlibAndGzip.Zlib; const Zlib = gunzip.Zlib; /** From 71078d9332d68633de4c3d736be365f6956c9ef4 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Tue, 17 Dec 2019 12:28:09 +0000 Subject: [PATCH 12/18] Added tests for gunzip. --- tests/operations/index.mjs | 1 + tests/operations/tests/Gunzip.mjs | 58 +++++++++++++++++++++++++++++++ tests/operations/tests/Gzip.mjs | 2 +- 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tests/operations/tests/Gunzip.mjs diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 09ebf0fc..51676386 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -42,6 +42,7 @@ import "./tests/ExtractEmailAddresses.mjs"; import "./tests/Fork.mjs"; import "./tests/FromDecimal.mjs"; import "./tests/Gzip.mjs"; +import "./tests/Gunzip.mjs"; import "./tests/Hash.mjs"; import "./tests/HaversineDistance.mjs"; import "./tests/Hexdump.mjs"; diff --git a/tests/operations/tests/Gunzip.mjs b/tests/operations/tests/Gunzip.mjs new file mode 100644 index 00000000..f6e05c2f --- /dev/null +++ b/tests/operations/tests/Gunzip.mjs @@ -0,0 +1,58 @@ +/** + * Gunzip Tests. + * + * @author n1073645 [n1073645@gmail.com] + * + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "Gunzip: No comment, no checksum and no filename", + input: "1f8b0800f7c8f85d00ff0dc9dd0180200804e0556ea8262848fb3dc588c6a7e76faa8aeedb726036c68d951f76bf9a0af8aae1f97d9c0c084b02509cbf8c2c000000", + expectedOutput: "The quick brown fox jumped over the slow dog", + recipeConfig: [ + { + op: "From Hex", + args: ["None"] + }, + { + op: "Gunzip", + args: [] + } + ] + }, + { + name: "Gunzip: No comment, no checksum and filename", + input: "1f8b080843c9f85d00ff66696c656e616d65000dc9dd0180200804e0556ea8262848fb3dc588c6a7e76faa8aeedb726036c68d951f76bf9a0af8aae1f97d9c0c084b02509cbf8c2c000000", + expectedOutput: "The quick brown fox jumped over the slow dog", + recipeConfig: [ + { + op: "From Hex", + args: ["None"] + }, + { + op: "Gunzip", + args: [] + } + ] + }, + { + name: "Gunzip: Has a comment, no checksum and has a filename", + input: "1f8b08186fc9f85d00ff66696c656e616d6500636f6d6d656e74000dc9dd0180200804e0556ea8262848fb3dc588c6a7e76faa8aeedb726036c68d951f76bf9a0af8aae1f97d9c0c084b02509cbf8c2c000000", + expectedOutput: "The quick brown fox jumped over the slow dog", + recipeConfig: [ + { + op: "From Hex", + args: ["None"] + }, + { + op: "Gunzip", + args: [] + } + ] + } +]); \ No newline at end of file diff --git a/tests/operations/tests/Gzip.mjs b/tests/operations/tests/Gzip.mjs index 2b055ae8..c9b2b8ca 100644 --- a/tests/operations/tests/Gzip.mjs +++ b/tests/operations/tests/Gzip.mjs @@ -1,5 +1,5 @@ /** - * Gzip tests. + * Gzip Tests. * * @author n1073645 [n1073645@gmail.com] * From 4100a22c7f7edbcea7cfb8f9ac346f4ff0246585 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Tue, 17 Dec 2019 12:30:32 +0000 Subject: [PATCH 13/18] Linting on tests --- tests/operations/tests/Gunzip.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/operations/tests/Gunzip.mjs b/tests/operations/tests/Gunzip.mjs index f6e05c2f..70b67a31 100644 --- a/tests/operations/tests/Gunzip.mjs +++ b/tests/operations/tests/Gunzip.mjs @@ -55,4 +55,4 @@ TestRegister.addTests([ } ] } -]); \ No newline at end of file +]); From 9112bd4936453a413331dba9af9bef125fec275a Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 20 Dec 2019 15:00:10 +0000 Subject: [PATCH 14/18] Tidied up OLE2 extractor --- src/core/lib/FileSignatures.mjs | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 4385ecea..fc6476d2 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -3116,24 +3116,26 @@ export function extractPListXML(bytes, offset) { */ export function extractOLE2(bytes, offset) { const stream = new Stream(bytes.slice(offset)); - const entries = [[[0x52, 0x00, 0x6f, 0x00, 0x6f, 0x00, 0x74, 0x00, 0x20, 0x00, 0x45, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x72, 0x00, 0x79], 19, "Root Entry"], - [[0x57, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6b, 0x00, 0x62, 0x00, 0x6f, 0x00, 0x6f, 0x00, 0x6b], 15, "Workbook"], - [[0x43, 0x00, 0x75, 0x00, 0x72, 0x00, 0x72, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x20, 0x00, 0x55, 0x00, 0x73, 0x00, 0x65, 0x00, 0x72], 23, "Current User"], - [[0x50, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x65, 0x00, 0x72, 0x00, 0x50, 0x00, 0x6f, 0x00, 0x69, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x20, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74], 37, "PowerPoint Document"], - [[0x57, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x64, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74], 23, "WordDocument"], - [[0x44, 0x00, 0x61, 0x00, 0x74, 0x00, 0x61], 7, "Data"], - [[0x50, 0x00, 0x69, 0x00, 0x63, 0x00, 0x74, 0x00, 0x75, 0x00, 0x72, 0x00, 0x65, 0x00, 0x73], 15, "Pictures"], - [[0x31, 0x00, 0x54, 0x00, 0x61, 0x00, 0x62, 0x00, 0x6c, 0x00, 0x65], 11, "1Table"], - [[0x05, 0x00, 0x53, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x72, 0x00, 0x79, 0x00, 0x49, 0x00, 0x6e, 0x00, 0x66, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x74, 0x00, 0x69, 0x00, 0x6f, 0x00, 0x6e], 37, "SummaryInformation"], - [[0x05, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x53, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x72, 0x00, 0x79, 0x00, 0x49, 0x00, 0x6e, 0x00, 0x66, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x74, 0x00, 0x69, 0x00, 0x6f, 0x00, 0x6e], 53, "DocumentSummaryInformation"], - [[0x43, 0x00, 0x6f, 0x00, 0x6d, 0x00, 0x70, 0x00, 0x4f, 0x00, 0x62, 0x00, 0x6a], 13, "Comp Obj"], - [[0x01, 0x00], 2, "Entry"]]; + const entries = [ + [[0x52, 0x00, 0x6f, 0x00, 0x6f, 0x00, 0x74, 0x00, 0x20, 0x00, 0x45, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x72, 0x00, 0x79], 19, "Root Entry"], + [[0x57, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6b, 0x00, 0x62, 0x00, 0x6f, 0x00, 0x6f, 0x00, 0x6b], 15, "Workbook"], + [[0x43, 0x00, 0x75, 0x00, 0x72, 0x00, 0x72, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x20, 0x00, 0x55, 0x00, 0x73, 0x00, 0x65, 0x00, 0x72], 23, "Current User"], + [[0x50, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x65, 0x00, 0x72, 0x00, 0x50, 0x00, 0x6f, 0x00, 0x69, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x20, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74], 37, "PowerPoint Document"], + [[0x57, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x64, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74], 23, "WordDocument"], + [[0x44, 0x00, 0x61, 0x00, 0x74, 0x00, 0x61], 7, "Data"], + [[0x50, 0x00, 0x69, 0x00, 0x63, 0x00, 0x74, 0x00, 0x75, 0x00, 0x72, 0x00, 0x65, 0x00, 0x73], 15, "Pictures"], + [[0x31, 0x00, 0x54, 0x00, 0x61, 0x00, 0x62, 0x00, 0x6c, 0x00, 0x65], 11, "1Table"], + [[0x05, 0x00, 0x53, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x72, 0x00, 0x79, 0x00, 0x49, 0x00, 0x6e, 0x00, 0x66, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x74, 0x00, 0x69, 0x00, 0x6f, 0x00, 0x6e], 37, "SummaryInformation"], + [[0x05, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x53, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x72, 0x00, 0x79, 0x00, 0x49, 0x00, 0x6e, 0x00, 0x66, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x74, 0x00, 0x69, 0x00, 0x6f, 0x00, 0x6e], 53, "DocumentSummaryInformation"], + [[0x43, 0x00, 0x6f, 0x00, 0x6d, 0x00, 0x70, 0x00, 0x4f, 0x00, 0x62, 0x00, 0x6a], 13, "Comp Obj"], + [[0x01, 0x00], 2, "Entry"] + ]; let endianness = "le"; // Move to endianess field. stream.moveForwardsBy(28); if (stream.readInt(2, endianness) === 0xfffe) - endianness = "le"; + endianness = "be"; // Calculate the size of the normal sectors. const sizeOfSector = 2 ** stream.readInt(2, endianness); @@ -3147,9 +3149,9 @@ export function extractOLE2(bytes, offset) { // Calculate root directory offset. let total = 512 + (rootStuff * sizeOfSector); stream.moveTo(total); - let found = true; // While valid directory entries. + let found = true; while (found) { found = false; @@ -3165,7 +3167,7 @@ export function extractOLE2(bytes, offset) { if (element[2] === "Comp Obj") { // The size of the Comp Obj entry - 128. Since we add 128 later. - total += (128*6); + total += 128 * 6; stream.moveTo(total); } else if (element[2] === "Entry") { From 99ccd06f23904d1018d1189358e9e6f058d4aa11 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 20 Dec 2019 15:02:01 +0000 Subject: [PATCH 15/18] 9.11.15 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index ff373e31..f38dc584 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.14", + "version": "9.11.15", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index e33738dc..b86fcebe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.14", + "version": "9.11.15", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From b8afbf7458671dc9cc124d9219a7af4f9f5308db Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 20 Dec 2019 15:04:27 +0000 Subject: [PATCH 16/18] Tidied up ICO extractor --- src/core/lib/FileSignatures.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 0e318423..52c3d28f 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -2942,7 +2942,7 @@ export function extractBMP(bytes, offset) { export function extractICO(bytes, offset) { const stream = new Stream(bytes.slice(offset)); - // Move to number of file there are. + // Move to number of files there are. stream.moveTo(4); // Read the number of files stored in the ICO From 0c6eac3b21d942f597b2f546065a194bbb82fab9 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 20 Dec 2019 15:04:49 +0000 Subject: [PATCH 17/18] 9.11.16 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index f38dc584..2ff86957 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.15", + "version": "9.11.16", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index b86fcebe..47841c1c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.15", + "version": "9.11.16", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From ab83caa77b7cca0716ecc9f952a023de30974f69 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 20 Dec 2019 15:21:13 +0000 Subject: [PATCH 18/18] 9.11.17 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2ff86957..82f591d5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.16", + "version": "9.11.17", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 47841c1c..c38fde80 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.16", + "version": "9.11.17", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef",