From 7c72871c0200d8d0b51580f8ef6d1a0dbd291110 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Thu, 14 Nov 2019 17:17:25 +0000 Subject: [PATCH 01/14] Added Tar and Mach-O extractors --- src/core/lib/FileSignatures.mjs | 177 +++++++++++++++++++++++++++++--- 1 file changed, 164 insertions(+), 13 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index dc7ced4d..8877881d 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1282,17 +1282,25 @@ export const FILE_SIGNATURES = { extension: "dylib", mime: "application/octet-stream", description: "", - signature: { - 0: 0xca, - 1: 0xfe, - 2: 0xba, - 3: 0xbe, - 4: 0x00, - 5: 0x00, - 6: 0x00, - 7: [0x01, 0x02, 0x03] - }, - extractor: null + signature: [ + { + 0: 0xca, + 1: 0xfe, + 2: 0xba, + 3: 0xbe, + 4: 0x00, + 5: 0x00, + 6: 0x00, + 7: [0x01, 0x02, 0x03] + }, + { + 0: 0xce, + 1: 0xfa, + 2: 0xed, + 3: 0xfe + } + ], + extractor: extractMACHO }, { name: "MacOS Mach-O 64-bit object", @@ -1305,7 +1313,7 @@ export const FILE_SIGNATURES = { 2: 0xed, 3: 0xfe }, - extractor: null + extractor: extractMACHO }, { name: "Adobe Flash", @@ -1404,7 +1412,7 @@ export const FILE_SIGNATURES = { 260: 0x61, 261: 0x72 }, - extractor: null + extractor: extractTAR }, { name: "Roshal Archive", @@ -2720,6 +2728,149 @@ export function extractZIP(bytes, offset) { } +/** + * MACHO extractor + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractMACHO(bytes, offset) { + + + /** + * Checks to see if the file is 64-bit. + * + * @param {string} magic + * @returns {bool} + */ + function isMagic64(magic) { + return magic === MHCIGAM64 || magic === MHCIGAM64; + } + + + /** + * Checks the endianness of the file. + * + * @param {string} magic + * @returns {bool} + */ + function shouldSwapBytes(magic) { + return magic === MHCIGAM || magic === MHCIGAM64; + } + + + /** + * Jumps through segment information and calculates the sum of the segement sizes. + * + * @param {Stream} stream + * @param {number} offset + * @param {string} isSwap + * @param {number} ncmds + * @returns {number} + */ + function dumpSegmentCommands(stream, offset, isSwap, ncmds) { + let total = 0; + for (let i = 0; i < ncmds; i++) { + + // Move to start of segment. + stream.moveTo(offset); + const cmd = stream.readInt(4, isSwap); + if (cmd === LCSEGEMENT64) { + + // Move to size of segment field. + stream.moveTo(offset + 48); + + // Extract size of segement. + total += stream.readInt(8, isSwap); + stream.moveTo(offset + 4); + + // Move to offset of next segment. + offset += stream.readInt(4, isSwap); + } else if (cmd === LCSEGEMENT) { + stream.moveTo(offset + 36); + + // Extract size of segement. + total += stream.readInt(4, isSwap); + stream.moveTo(offset + 4); + offset += stream.readInt(4, isSwap); + } + } + return total; + } + + + /** + * Reads the number of command segments. + * + * @param {Stream} stream + * @param {bool} is64 + * @param {string} isSwap + * @returns {number} + */ + function dumpMachHeader(stream, is64, isSwap) { + let loadCommandsOffset = 28; + if (is64) + loadCommandsOffset += 4; + + // Move to number of commands field. + stream.moveTo(16); + const ncmds = stream.readInt(4, isSwap); + return dumpSegmentCommands(stream, loadCommandsOffset, isSwap, ncmds); + } + + + const MHCIGAM64 = "207250237254"; + const MHCIGAM = "206250237254"; + const LCSEGEMENT64 = 0x19; + const LCSEGEMENT = 0x1; + const stream = new Stream(bytes.slice(offset)); + const magic = stream.getBytes(4).join(""); + const is64 = isMagic64(magic); + const isSwap = shouldSwapBytes(magic) ? "le" : "be"; + + // Move to the end of the final segment. + stream.moveTo(dumpMachHeader(stream, is64, isSwap)); + return stream.carve(); +} + + +/** + * TAR extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractTAR(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + while (stream.hasMore()) { + + // Move to ustar identifier. + stream.moveForwardsBy(0x101); + if (stream.getBytes(5).join("") !== [0x75, 0x73, 0x74, 0x61, 0x72].join("")) + break; + + // Move back to file size field. + stream.moveBackwardsBy(0x8a); + let fsize = 0; + + // Read file size field. + stream.getBytes(11).forEach((element, index) => { + fsize += (element - 48).toString(); + }); + + // Round number up from octet to nearest 512. + fsize = (Math.ceil(parseInt(fsize, 8) / 512) * 512); + + // Move forwards to the end of that file. + stream.moveForwardsBy(fsize + 0x179); + } + stream.consumeWhile(0x00); + return stream.carve(); +} + + /** * PNG extractor. * From 7c25e295156006b788eeac2122d92d992ae2bec9 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Fri, 15 Nov 2019 09:21:46 +0000 Subject: [PATCH 02/14] Rectified magic bytes for MACHO extractor --- src/core/lib/FileSignatures.mjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 8877881d..2ef390cd 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -2745,7 +2745,7 @@ export function extractMACHO(bytes, offset) { * @returns {bool} */ function isMagic64(magic) { - return magic === MHCIGAM64 || magic === MHCIGAM64; + return magic === MHCIGAM64 || magic === MHMAGIC64; } @@ -2821,16 +2821,16 @@ export function extractMACHO(bytes, offset) { const MHCIGAM64 = "207250237254"; + const MHMAGIC64 = "254237250207"; const MHCIGAM = "206250237254"; const LCSEGEMENT64 = 0x19; const LCSEGEMENT = 0x1; + const stream = new Stream(bytes.slice(offset)); const magic = stream.getBytes(4).join(""); - const is64 = isMagic64(magic); - const isSwap = shouldSwapBytes(magic) ? "le" : "be"; // Move to the end of the final segment. - stream.moveTo(dumpMachHeader(stream, is64, isSwap)); + stream.moveTo(dumpMachHeader(stream, isMagic64(magic), shouldSwapBytes(magic) ? "le" : "be")); return stream.carve(); } From acf5c733c2fa74ac59cb8a4f3352ff0dc56443c8 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Fri, 15 Nov 2019 09:26:49 +0000 Subject: [PATCH 03/14] Tidied up local and global variables for Mach-o --- src/core/lib/FileSignatures.mjs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 2ef390cd..cf8a5373 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -2737,6 +2737,11 @@ export function extractZIP(bytes, offset) { */ export function extractMACHO(bytes, offset) { + // Magic bytes. + const MHCIGAM64 = "207250237254"; + const MHMAGIC64 = "254237250207"; + const MHCIGAM = "206250237254"; + /** * Checks to see if the file is 64-bit. @@ -2771,6 +2776,9 @@ export function extractMACHO(bytes, offset) { */ function dumpSegmentCommands(stream, offset, isSwap, ncmds) { let total = 0; + const LCSEGEMENT64 = 0x19; + const LCSEGEMENT = 0x1; + for (let i = 0; i < ncmds; i++) { // Move to start of segment. @@ -2820,12 +2828,6 @@ export function extractMACHO(bytes, offset) { } - const MHCIGAM64 = "207250237254"; - const MHMAGIC64 = "254237250207"; - const MHCIGAM = "206250237254"; - const LCSEGEMENT64 = 0x19; - const LCSEGEMENT = 0x1; - const stream = new Stream(bytes.slice(offset)); const magic = stream.getBytes(4).join(""); From e1cb62848c0a22e0cdb4d8200421f862612e12e2 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Fri, 15 Nov 2019 15:35:37 +0000 Subject: [PATCH 04/14] Made TAR extractor and MACHO signature more robust --- src/core/lib/FileSignatures.mjs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index cf8a5373..8c5a6557 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1297,7 +1297,12 @@ export const FILE_SIGNATURES = { 0: 0xce, 1: 0xfa, 2: 0xed, - 3: 0xfe + 3: 0xfe, + 4: 0x07, + 5: 0x00, + 6: 0x00, + 7: 0x00, + 8: 0x03 } ], extractor: extractMACHO @@ -2850,8 +2855,12 @@ export function extractTAR(bytes, offset) { // Move to ustar identifier. stream.moveForwardsBy(0x101); - if (stream.getBytes(5).join("") !== [0x75, 0x73, 0x74, 0x61, 0x72].join("")) + if (stream.getBytes(5).join("") !== [0x75, 0x73, 0x74, 0x61, 0x72].join("")) { + + // This is needed since if it were not here it relies on there being at least 0x106 padding of 0s at the end of the TAR + stream.moveBackwardsBy(0x101); break; + } // Move back to file size field. stream.moveBackwardsBy(0x8a); From 0a7b78b7ee7cc851799f8462f616c10f964987ca Mon Sep 17 00:00:00 2001 From: n1073645 Date: Fri, 15 Nov 2019 15:46:13 +0000 Subject: [PATCH 05/14] Made TAR extractor and MACHO signature more robust --- src/core/lib/FileSignatures.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 8c5a6557..0442b3b0 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -2858,7 +2858,7 @@ export function extractTAR(bytes, offset) { if (stream.getBytes(5).join("") !== [0x75, 0x73, 0x74, 0x61, 0x72].join("")) { // This is needed since if it were not here it relies on there being at least 0x106 padding of 0s at the end of the TAR - stream.moveBackwardsBy(0x101); + stream.moveBackwardsBy(0x106); break; } From c1a22ef639ad92d3b035e0a922edea1de5dadc83 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Fri, 15 Nov 2019 16:01:33 +0000 Subject: [PATCH 06/14] Made TAR extractor and MACHO signature more robust --- src/core/lib/FileSignatures.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 0442b3b0..2fa630e1 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1302,7 +1302,7 @@ export const FILE_SIGNATURES = { 5: 0x00, 6: 0x00, 7: 0x00, - 8: 0x03 + 8: [0x01, 0x02, 0x03] } ], extractor: extractMACHO From 2e7ce477d70fd9b6c93744c5f9119babb7bada43 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Mon, 18 Nov 2019 08:40:57 +0000 Subject: [PATCH 07/14] Tidied up ExtractTAR --- src/core/lib/FileSignatures.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 2fa630e1..939672f2 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -2857,7 +2857,7 @@ export function extractTAR(bytes, offset) { stream.moveForwardsBy(0x101); if (stream.getBytes(5).join("") !== [0x75, 0x73, 0x74, 0x61, 0x72].join("")) { - // This is needed since if it were not here it relies on there being at least 0x106 padding of 0s at the end of the TAR + // Needed since if it were not here it relies on there being at least 0x106 padding of 0s at the end of the TAR. stream.moveBackwardsBy(0x106); break; } From 7a4ebbf47e11bea8a00b2521f13779c3868197a6 Mon Sep 17 00:00:00 2001 From: n1073645 Date: Mon, 18 Nov 2019 08:42:46 +0000 Subject: [PATCH 08/14] Tidied up ExtractTAR --- src/core/lib/FileSignatures.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 939672f2..2e85baf5 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -2857,7 +2857,7 @@ export function extractTAR(bytes, offset) { stream.moveForwardsBy(0x101); if (stream.getBytes(5).join("") !== [0x75, 0x73, 0x74, 0x61, 0x72].join("")) { - // Needed since if it were not here it relies on there being at least 0x106 padding of 0s at the end of the TAR. + // Needed since we cannot rely on there being at least 0x106 padding of 0s at the end of the TAR(even though there usually is). stream.moveBackwardsBy(0x106); break; } From d16bbe1e7e2b749eb387bdb0ebd67560c399ac09 Mon Sep 17 00:00:00 2001 From: fjh1997 <549308442@qq.com> Date: Sun, 24 Nov 2019 22:03:37 +0800 Subject: [PATCH 09/14] Fixed typo in IPV6 alphabet According to python3 base64.b85decode module https://github.com/python/cpython/blob/3.8/Lib/base64.py --- src/core/lib/Base85.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/Base85.mjs b/src/core/lib/Base85.mjs index 18ba5ef8..8da729e2 100644 --- a/src/core/lib/Base85.mjs +++ b/src/core/lib/Base85.mjs @@ -20,7 +20,7 @@ export const ALPHABET_OPTIONS = [ }, { name: "IPv6", - value: "0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|~}", + value: "0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~", } ]; From 8e5aa2c393a4b3d4812ac069867c5c27ceb3817f Mon Sep 17 00:00:00 2001 From: Mirclus Date: Mon, 25 Nov 2019 20:08:30 +0100 Subject: [PATCH 10/14] DNS over HTTP: Fix "validate" argument The argument sets the "cd" parameter on the request. For both included providers, this flag disables validation ([1], [2]), so doing the exact opposite of the described action. This changes the label to the correct name and also flips the default value to keep the old behavior. [1] Google [2] Cloudflare --- src/core/operations/DNSOverHTTPS.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/operations/DNSOverHTTPS.mjs b/src/core/operations/DNSOverHTTPS.mjs index ca779815..4dde2f13 100644 --- a/src/core/operations/DNSOverHTTPS.mjs +++ b/src/core/operations/DNSOverHTTPS.mjs @@ -63,9 +63,9 @@ class DNSOverHTTPS extends Operation { value: false }, { - name: "Validate DNSSEC", + name: "Disable DNSSEC validation", type: "boolean", - value: true + value: false } ]; } From 13e9a4f0da9950ce9fc4339b7d39fb95ad8fd112 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 27 Nov 2019 12:36:18 +0000 Subject: [PATCH 11/14] 9.11.8 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6c765b4e..50532e20 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.7", + "version": "9.11.8", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 7bdea20b..c3b59f03 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.7", + "version": "9.11.8", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From b459c15d7431eeeea75c977b70ac4188fec45ef9 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 27 Nov 2019 12:47:16 +0000 Subject: [PATCH 12/14] 9.11.9 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 50532e20..b820863c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.8", + "version": "9.11.9", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index c3b59f03..15b7253f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.8", + "version": "9.11.9", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 2750284eea47b846cb47d159e874079398eeaf96 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 27 Nov 2019 12:56:10 +0000 Subject: [PATCH 13/14] Improved comment in Tar extractor --- src/core/lib/FileSignatures.mjs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 2e85baf5..17b17f23 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -2856,8 +2856,7 @@ export function extractTAR(bytes, offset) { // Move to ustar identifier. stream.moveForwardsBy(0x101); if (stream.getBytes(5).join("") !== [0x75, 0x73, 0x74, 0x61, 0x72].join("")) { - - // Needed since we cannot rely on there being at least 0x106 padding of 0s at the end of the TAR(even though there usually is). + // Reverse back to the end of the last section. stream.moveBackwardsBy(0x106); break; } From 44c2b71e6c63d4dbe27304efaefb1ca18c0902e2 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 27 Nov 2019 12:56:30 +0000 Subject: [PATCH 14/14] 9.11.10 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b820863c..190ba0ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.9", + "version": "9.11.10", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 15b7253f..3060deca 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.11.9", + "version": "9.11.10", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef",