From a8fbd5164e5ce92255f517177c6a40606f79465b Mon Sep 17 00:00:00 2001 From: d98762625 Date: Fri, 6 Sep 2019 12:21:53 +0100 Subject: [PATCH 01/18] Update NodeRecipe so args is optional in bake config --- package-lock.json | 41 ++++++++++-------------------------- src/node/NodeRecipe.mjs | 7 ++++-- tests/node/tests/nodeApi.mjs | 33 +++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 32 deletions(-) diff --git a/package-lock.json b/package-lock.json index 50e7fb46..8e7d46bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5839,8 +5839,7 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "aproba": { "version": "1.2.0", @@ -5861,14 +5860,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5883,20 +5880,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -6013,8 +6007,7 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -6026,7 +6019,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -6041,7 +6033,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -6049,14 +6040,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -6075,7 +6064,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -6156,8 +6144,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -6169,7 +6156,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -6255,8 +6241,7 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "safer-buffer": { "version": "2.1.2", @@ -6292,7 +6277,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -6312,7 +6296,6 @@ "version": "3.0.1", "bundled": true, "dev": true, - "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -6356,14 +6339,12 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true } } }, diff --git a/src/node/NodeRecipe.mjs b/src/node/NodeRecipe.mjs index 2759073e..70e3670b 100644 --- a/src/node/NodeRecipe.mjs +++ b/src/node/NodeRecipe.mjs @@ -44,10 +44,13 @@ class NodeRecipe { throw new TypeError("Inputted function not a Chef operation."); } // CASE: op with configuration - } else if (ing.op && ing.args) { + } else if (ing.op) { // Return op and args pair for opList item. const sanitisedOp = this._validateIngredient(ing.op); - return {op: sanitisedOp, args: ing.args}; + if (ing.args) { + return {op: sanitisedOp, args: ing.args}; + } + return sanitisedOp; } else { throw new TypeError("Recipe can only contain function names or functions"); } diff --git a/tests/node/tests/nodeApi.mjs b/tests/node/tests/nodeApi.mjs index 954a19ce..a4e907b8 100644 --- a/tests/node/tests/nodeApi.mjs +++ b/tests/node/tests/nodeApi.mjs @@ -259,6 +259,13 @@ TestRegister.addApiTests([ assert.strictEqual(result.toString(), "73:6f:6d:65:20:69:6e:70:75:74"); }), + it("chef.bake: should take single JSON object desribing op with optional args", () => { + const result = chef.bake("some input", { + op: chef.toHex, + }); + assert.strictEqual(result.toString(), "73 6f 6d 65 20 69 6e 70 75 74"); + }), + it("chef.bake: should take single JSON object describing op and args ARRAY", () => { const result = chef.bake("some input", { op: chef.toHex, @@ -295,6 +302,21 @@ TestRegister.addApiTests([ assert.strictEqual(result.toString(), "67;63;72;66;146;72;66;144;72;66;65;72;62;60;72;66;71;72;66;145;72;67;60;72;67;65;72;67;64"); }), + it("chef.bake: should take multiple ops in JSON object form, some without args", () => { + const result = chef.bake("some input", [ + { + op: chef.toHex, + }, + { + op: "to octal", + args: { + delimiter: "Semi-colon", + } + } + ]); + assert.strictEqual(result.toString(), "67;63;40;66;146;40;66;144;40;66;65;40;62;60;40;66;71;40;66;145;40;67;60;40;67;65;40;67;64"); + }), + it("chef.bake: should handle op with multiple args", () => { const result = chef.bake("some input", { op: "to morse code", @@ -324,6 +346,17 @@ TestRegister.addApiTests([ assert.strictEqual(result.toString(), "begin_something_anananaaaaak_da_aaak_da_aaaaananaaaaaaan_da_aaaaaaanan_da_aaak_end_something"); }), + it("chef.bake: should accept Clean JSON format from Chef website - args optional", () => { + const result = chef.bake("some input", [ + { "op": "To Morse Code" }, + { "op": "Hex to PEM", + "args": ["SOMETHING"] }, + { "op": "To Snake case", + "args": [false] } + ]); + assert.strictEqual(result.toString(), "begin_something_aaaaaaaaaaaaaa_end_something"); + }), + it("Excluded operations: throw a sensible error when you try and call one", () => { try { chef.fork(); From 111546ad1af7a4edd52750822347c3fcd85b3e40 Mon Sep 17 00:00:00 2001 From: d98762625 Date: Fri, 6 Sep 2019 12:26:24 +0100 Subject: [PATCH 02/18] update function comments --- src/node/NodeRecipe.mjs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/node/NodeRecipe.mjs b/src/node/NodeRecipe.mjs index 70e3670b..b623f611 100644 --- a/src/node/NodeRecipe.mjs +++ b/src/node/NodeRecipe.mjs @@ -43,9 +43,8 @@ class NodeRecipe { } else { throw new TypeError("Inputted function not a Chef operation."); } - // CASE: op with configuration + // CASE: op, maybe with configuration } else if (ing.op) { - // Return op and args pair for opList item. const sanitisedOp = this._validateIngredient(ing.op); if (ing.args) { return {op: sanitisedOp, args: ing.args}; From 928f1c3e4ba3a3ee9ea5aa0b6fd0bed7849c2df3 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 25 Sep 2019 15:39:04 +0100 Subject: [PATCH 03/18] Remove excess slashes from Tesseract paths --- src/core/operations/OpticalCharacterRecognition.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/operations/OpticalCharacterRecognition.mjs b/src/core/operations/OpticalCharacterRecognition.mjs index 5a20f123..1c26e55f 100644 --- a/src/core/operations/OpticalCharacterRecognition.mjs +++ b/src/core/operations/OpticalCharacterRecognition.mjs @@ -62,9 +62,9 @@ class OpticalCharacterRecognition extends Operation { try { const image = `data:${type};base64,${toBase64(input)}`; const worker = new TesseractWorker({ - workerPath: `${assetDir}/tesseract/worker.min.js`, - langPath: `${assetDir}/tesseract/lang-data/`, - corePath: `${assetDir}/tesseract/tesseract-core.wasm.js`, + workerPath: `${assetDir}tesseract/worker.min.js`, + langPath: `${assetDir}tesseract/lang-data`, + corePath: `${assetDir}tesseract/tesseract-core.wasm.js`, }); const result = await worker.recognize(image) .progress(progress => { From 1e8dee9935954664239fb910a711d90e8cb6c561 Mon Sep 17 00:00:00 2001 From: Andy Wang Date: Thu, 26 Sep 2019 21:32:08 +0100 Subject: [PATCH 04/18] update node-forge version --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5602635d..b56ea1d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9480,9 +9480,9 @@ "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" }, "node-forge": { - "version": "0.8.5", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.8.5.tgz", - "integrity": "sha512-vFMQIWt+J/7FLNyKouZ9TazT74PRV3wgv9UT4cRjC8BffxFbKXkgIWR42URCPSnHm/QDz6BOlb2Q0U4+VQT67Q==" + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", + "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==" }, "node-gyp": { "version": "3.8.0", diff --git a/package.json b/package.json index bc59edf2..058d070a 100644 --- a/package.json +++ b/package.json @@ -131,7 +131,7 @@ "moment": "^2.24.0", "moment-timezone": "^0.5.26", "ngeohash": "^0.6.3", - "node-forge": "^0.8.5", + "node-forge": "^0.9.1", "node-md6": "^0.1.0", "nodom": "^2.2.0", "notepack.io": "^2.2.0", From a59de80d189017bab8eb0aaa89be17e145b517bb Mon Sep 17 00:00:00 2001 From: Andy Wang Date: Thu, 26 Sep 2019 21:41:35 +0100 Subject: [PATCH 05/18] Update node API tests --- tests/node/tests/operations.mjs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/node/tests/operations.mjs b/tests/node/tests/operations.mjs index ac846d93..75f37ab1 100644 --- a/tests/node/tests/operations.mjs +++ b/tests/node/tests/operations.mjs @@ -70,13 +70,13 @@ TestRegister.addApiTests([ }), it("AES decrypt: toggleString and option", () => { - const result = AESDecrypt("812c34ae6af353244a63c6ce23b7c34286b60be28ea4645523d4494700e7", { + const result = AESDecrypt("4a123af235a507bbc9d5871721d61b98504d569a9a5a7847e2d78315fec7", { key: { string: "some longer key1", option: "utf8", }, iv: { - string: "some iv", + string: "some iv some iv1", option: "utf8", }, mode: "OFB", @@ -916,8 +916,13 @@ smothering ampersand abreast it("Triple DES encrypt / decrypt", () => { assert.strictEqual( chef.tripleDESDecrypt( - chef.tripleDESEncrypt("Destroy Money", {key: {string: "30 31 2f 30 34 2f 31 39 39 39 20 32 32 3a 33 33 3a 30 3130 31 2f 30 34", option: "Hex"}}), - {key: {string: "30 31 2f 30 34 2f 31 39 39 39 20 32 32 3a 33 33 3a 30 3130 31 2f 30 34", option: "Hex"}}).toString(), + chef.tripleDESEncrypt("Destroy Money", { + key: {string: "30 31 2f 30 34 2f 31 39 39 39 20 32 32 3a 33 33 3a 30 3130 31 2f 30 34", option: "Hex"}, + iv: {string: "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0000 00 00 00 00", option: "Hex"}}), + { + key: {string: "30 31 2f 30 34 2f 31 39 39 39 20 32 32 3a 33 33 3a 30 3130 31 2f 30 34", option: "Hex"}, + iv: {string: "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0000 00 00 00 00", option: "Hex"} + }).toString(), "Destroy Money"); }), From b06acd99ec5074340c80729278dd2bf103b33279 Mon Sep 17 00:00:00 2001 From: Andy Wang Date: Thu, 26 Sep 2019 21:52:34 +0100 Subject: [PATCH 06/18] Adapt AES "no IV" tests --- tests/operations/tests/Crypt.mjs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/operations/tests/Crypt.mjs b/tests/operations/tests/Crypt.mjs index 1db8093f..f66aeb4b 100644 --- a/tests/operations/tests/Crypt.mjs +++ b/tests/operations/tests/Crypt.mjs @@ -76,7 +76,7 @@ The following algorithms will be used based on the size of the key: ], }, { - name: "AES Encrypt: AES-128-CBC, no IV, ASCII", + name: "AES Encrypt: AES-128-CBC with IV0, ASCII", input: "The quick brown fox jumps over the lazy dog.", expectedOutput: "2ef6c3fdb1314b5c2c326a2087fe1a82d5e73bf605ec8431d73e847187fc1c8fbbe969c177df1ecdf8c13f2f505f9498", recipeConfig: [ @@ -84,14 +84,14 @@ The following algorithms will be used based on the size of the key: "op": "AES Encrypt", "args": [ {"option": "Hex", "string": "00112233445566778899aabbccddeeff"}, - {"option": "Hex", "string": ""}, + {"option": "Hex", "string": "00000000000000000000000000000000"}, "CBC", "Raw", "Hex" ] } ], }, { - name: "AES Encrypt: AES-128-CTR, no IV, ASCII", + name: "AES Encrypt: AES-128-CTR with IV0, ASCII", input: "The quick brown fox jumps over the lazy dog.", expectedOutput: "a98c9e8e3b7c894384d740e4f0f4ed0be2bbb1e0e13a255812c3c6b0a629e4ad759c075b2469c6f4fb2c0cf9", recipeConfig: [ @@ -99,14 +99,14 @@ The following algorithms will be used based on the size of the key: "op": "AES Encrypt", "args": [ {"option": "Hex", "string": "00112233445566778899aabbccddeeff"}, - {"option": "Hex", "string": ""}, + {"option": "Hex", "string": "00000000000000000000000000000000"}, "CTR", "Raw", "Hex" ] } ], }, { - name: "AES Encrypt: AES-128-CBC with IV, ASCII", + name: "AES Encrypt: AES-128-CBC with IV1, ASCII", input: "The quick brown fox jumps over the lazy dog.", expectedOutput: "4fa077d50cc71a57393e7b542c4e3aea0fb75383b97083f2f568ffc13c0e7a47502ec6d9f25744a061a3a5e55fe95e8d", recipeConfig: [ @@ -681,7 +681,7 @@ The following algorithms will be used based on the size of the key: ], }, { - name: "AES Decrypt: AES-128-CBC, no IV, ASCII", + name: "AES Decrypt: AES-128-CBC with IV0, ASCII", input: "2ef6c3fdb1314b5c2c326a2087fe1a82d5e73bf605ec8431d73e847187fc1c8fbbe969c177df1ecdf8c13f2f505f9498", expectedOutput: "The quick brown fox jumps over the lazy dog.", recipeConfig: [ @@ -689,7 +689,7 @@ The following algorithms will be used based on the size of the key: "op": "AES Decrypt", "args": [ {"option": "Hex", "string": "00112233445566778899aabbccddeeff"}, - {"option": "Hex", "string": ""}, + {"option": "Hex", "string": "00000000000000000000000000000000"}, "CBC", "Hex", "Raw", {"option": "Hex", "string": ""} ] @@ -697,7 +697,7 @@ The following algorithms will be used based on the size of the key: ], }, { - name: "AES Decrypt: AES-128-CTR, no IV, ASCII", + name: "AES Decrypt: AES-128-CTR with IV0, ASCII", input: "a98c9e8e3b7c894384d740e4f0f4ed0be2bbb1e0e13a255812c3c6b0a629e4ad759c075b2469c6f4fb2c0cf9", expectedOutput: "The quick brown fox jumps over the lazy dog.", recipeConfig: [ @@ -705,7 +705,7 @@ The following algorithms will be used based on the size of the key: "op": "AES Decrypt", "args": [ {"option": "Hex", "string": "00112233445566778899aabbccddeeff"}, - {"option": "Hex", "string": ""}, + {"option": "Hex", "string": "00000000000000000000000000000000"}, "CTR", "Hex", "Raw", {"option": "Hex", "string": ""} ] From 880df212d5cb96f0f2f467b63aa61e96023adfc0 Mon Sep 17 00:00:00 2001 From: Andy Wang Date: Thu, 26 Sep 2019 22:00:13 +0100 Subject: [PATCH 07/18] Correct wrong DES and 3DES tests --- tests/operations/tests/Crypt.mjs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/operations/tests/Crypt.mjs b/tests/operations/tests/Crypt.mjs index f66aeb4b..48225d77 100644 --- a/tests/operations/tests/Crypt.mjs +++ b/tests/operations/tests/Crypt.mjs @@ -537,9 +537,10 @@ Triple DES uses a key length of 24 bytes (192 bits).`, ], }, { + // play.golang.org/p/4Qm2hfLGsqc name: "DES Encrypt: DES-CTR, Binary", input: "7a0e643132750e96d805d11e9e48e281fa39a41039286423cc1c045e5442b40bf1c3f2822bded3f9c8ef11cb25da64dda9c7ab87c246bd305385150c98f31465c2a6180fe81d31ea289b916504d5a12e1de26cb10adba84a0cb0c86f94bc14bc554f3018", - expectedOutput: "09015087e15b0937ab0ae5a84d66e520893690a6ea066382bf1330e8876cb3aa82ccc634f8f0d458bbe0257df6f4637cdac89f311168ba91208a21ba4bdd13c4b1a92cb93b33364b5b94a5d3d7fba68f6eed5807d9f5afeb7fbffcd94792131d264004ae", + expectedOutput: "09015087e15b0937c462fd5974af0c4b5880de136a5680453c99f4500628cbeca769623515d836985110b93eacfea7fa4a7b2b3cb4f67acbb5f7e8ddb5a5d445da74bf6572b0a874befa3888c81110776388e400afd8dc908dcc0c018c7753355f8a1c9f", recipeConfig: [ { "op": "DES Encrypt", @@ -630,9 +631,10 @@ DES uses a key length of 8 bytes (64 bits).`, ], }, { + // play.golang.org/p/RElT6pVeNz2 name: "Triple DES Encrypt: DES-EDE3-CTR, Binary", input: "7a0e643132750e96d805d11e9e48e281fa39a41039286423cc1c045e5442b40bf1c3f2822bded3f9c8ef11cb25da64dda9c7ab87c246bd305385150c98f31465c2a6180fe81d31ea289b916504d5a12e1de26cb10adba84a0cb0c86f94bc14bc554f3018", - expectedOutput: "874d32cd7bdae52c254687e2d7e7093b077af2ec70878f99315f52a21ded5fb10c80a47e6271384335ac47376c758f675484fd7b8be9568aaec643f0d15cffdf3fe54ef3a1b2da50d5d8c7994d7a4a94e0a13a4d437443f0f1f39e93dd13ff06a80c66e4", + expectedOutput: "874d32cd7bdae52cd8630d3ab2bf373e7110e13713caa6a8bfed9d9dd802d0ebe93128ac0d0f05abcc56237b75fb69207dba11e68ddc4b0118a4c75e7248bbd80aaba4dd4436642546ec6ca7fa7526f3b0018ed5194c409dc2c1484530b968af554984f3", recipeConfig: [ { "op": "Triple DES Encrypt", @@ -1160,9 +1162,10 @@ Triple DES uses a key length of 24 bytes (192 bits).`, ], }, { + // play.golang.org/p/FpvqncmPk7R name: "DES Decrypt: DES-CTR, Binary", input: "09015087e15b0937ab0ae5a84d66e520893690a6ea066382bf1330e8876cb3aa82ccc634f8f0d458bbe0257df6f4637cdac89f311168ba91208a21ba4bdd13c4b1a92cb93b33364b5b94a5d3d7fba68f6eed5807d9f5afeb7fbffcd94792131d264004ae", - expectedOutput: "7a0e643132750e96d805d11e9e48e281fa39a41039286423cc1c045e5442b40bf1c3f2822bded3f9c8ef11cb25da64dda9c7ab87c246bd305385150c98f31465c2a6180fe81d31ea289b916504d5a12e1de26cb10adba84a0cb0c86f94bc14bc554f3018", + expectedOutput: "7a0e643132750e96b76dc9efa7810bea2b8feaa5b97887e44f96c0e6d506cc4dd4665683c6f63139221f8d887fd0a05b39741f8a67d87d6ac6f8dc6b668bd3e4a97b8bd3a19eafd5cdf50c3e1b3f17d61087d0b67cf6db31fec338b75f5954942c852829", recipeConfig: [ { "op": "DES Decrypt", @@ -1253,9 +1256,10 @@ DES uses a key length of 8 bytes (64 bits).`, ], }, { + // play.golang.org/p/iBacN9kX_RO name: "Triple DES Decrypt: DES-EDE3-CTR, Binary", input: "874d32cd7bdae52c254687e2d7e7093b077af2ec70878f99315f52a21ded5fb10c80a47e6271384335ac47376c758f675484fd7b8be9568aaec643f0d15cffdf3fe54ef3a1b2da50d5d8c7994d7a4a94e0a13a4d437443f0f1f39e93dd13ff06a80c66e4", - expectedOutput: "7a0e643132750e96d805d11e9e48e281fa39a41039286423cc1c045e5442b40bf1c3f2822bded3f9c8ef11cb25da64dda9c7ab87c246bd305385150c98f31465c2a6180fe81d31ea289b916504d5a12e1de26cb10adba84a0cb0c86f94bc14bc554f3018", + expectedOutput: "7a0e643132750e9625205bc6fb10dc848c53b7cb5a654d1242aecb6191ad3b5114727e5044a0ee11311575873c54829a80f9471ac473a0bbe5e791a23be75062f7e8f2210d998f9fbbaf3a5bb3dacd494d42d82950e3ab273f821eb979168315a80ad20f", recipeConfig: [ { "op": "Triple DES Decrypt", From 70f705afbc51f19536500ef4216287a5a2f7f6ae Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 27 Sep 2019 11:28:57 +0100 Subject: [PATCH 08/18] Fixed typo in 'Convert Data Units' preventing Kilobits from working. Closes #649 --- src/core/operations/ConvertDataUnits.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/ConvertDataUnits.mjs b/src/core/operations/ConvertDataUnits.mjs index baed9004..0335e852 100644 --- a/src/core/operations/ConvertDataUnits.mjs +++ b/src/core/operations/ConvertDataUnits.mjs @@ -54,7 +54,7 @@ class ConvertDataUnits extends Operation { const DATA_UNITS = [ "Bits (b)", "Nibbles", "Octets", "Bytes (B)", "[Binary bits (2^n)]", "Kibibits (Kib)", "Mebibits (Mib)", "Gibibits (Gib)", "Tebibits (Tib)", "Pebibits (Pib)", "Exbibits (Eib)", "Zebibits (Zib)", "Yobibits (Yib)", "[/Binary bits (2^n)]", - "[Decimal bits (10^n)]", "Decabits", "Hectobits", "Kilobits (kb)", "Megabits (Mb)", "Gigabits (Gb)", "Terabits (Tb)", "Petabits (Pb)", "Exabits (Eb)", "Zettabits (Zb)", "Yottabits (Yb)", "[/Decimal bits (10^n)]", + "[Decimal bits (10^n)]", "Decabits", "Hectobits", "Kilobits (Kb)", "Megabits (Mb)", "Gigabits (Gb)", "Terabits (Tb)", "Petabits (Pb)", "Exabits (Eb)", "Zettabits (Zb)", "Yottabits (Yb)", "[/Decimal bits (10^n)]", "[Binary bytes (8 x 2^n)]", "Kibibytes (KiB)", "Mebibytes (MiB)", "Gibibytes (GiB)", "Tebibytes (TiB)", "Pebibytes (PiB)", "Exbibytes (EiB)", "Zebibytes (ZiB)", "Yobibytes (YiB)", "[/Binary bytes (8 x 2^n)]", "[Decimal bytes (8 x 10^n)]", "Kilobytes (KB)", "Megabytes (MB)", "Gigabytes (GB)", "Terabytes (TB)", "Petabytes (PB)", "Exabytes (EB)", "Zettabytes (ZB)", "Yottabytes (YB)", "[/Decimal bytes (8 x 10^n)]" ]; From db232f4ff281a96bf7199329443c47f27dc889fa Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 27 Sep 2019 11:29:01 +0100 Subject: [PATCH 09/18] 9.7.2 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5602635d..ad2173c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.1", + "version": "9.7.2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index bc59edf2..57a2eb1a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.1", + "version": "9.7.2", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From f87d3bd1cb6781c961e3445bbe7e6cc48b114d8d Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Oct 2019 12:10:43 +0100 Subject: [PATCH 10/18] 9.7.3 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3b2cc1ab..4ab88021 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.2", + "version": "9.7.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index ac853448..76680a84 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.2", + "version": "9.7.3", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 777d814e702aa7fb4ee2eb3c3ad68c110f42baa7 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Oct 2019 12:14:57 +0100 Subject: [PATCH 11/18] Updated package-lock.json --- package-lock.json | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1e1ca9c9..4ab88021 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6026,7 +6026,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -6047,12 +6048,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -6067,17 +6070,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -6194,7 +6200,8 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -6206,6 +6213,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -6220,6 +6228,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -6227,12 +6236,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -6251,6 +6262,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -6331,7 +6343,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -6343,6 +6356,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -6428,7 +6442,8 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -6464,6 +6479,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -6483,6 +6499,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -6526,12 +6543,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, From 3d086beed27f615adddfc0e3ea520adb8ed5eddf Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Oct 2019 12:15:07 +0100 Subject: [PATCH 12/18] 9.7.4 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4ab88021..558972ea 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.3", + "version": "9.7.4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 76680a84..7cd3d598 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.3", + "version": "9.7.4", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 22028b074a1757810b243b2298390b6604207afc Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Oct 2019 16:54:19 +0100 Subject: [PATCH 13/18] Added support for many more file types to file signature operations --- src/core/lib/FileSignatures.mjs | 605 ++++++++++++++++++++++++- src/core/operations/DetectFileType.mjs | 8 +- 2 files changed, 600 insertions(+), 13 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 613e82b2..afb4b6e0 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -241,6 +241,28 @@ export const FILE_SIGNATURES = { ], extractor: null }, + { + name: "The GIMP image", + extension: "xcf", + mime: "image/x-xcf", + description: "", + signature: { + 0: 0x67, // gimp xcf + 1: 0x69, + 2: 0x6d, + 3: 0x70, + 4: 0x20, + 5: 0x78, + 6: 0x63, + 7: 0x66, + 8: 0x20, + 9: [0x66, 0x76], + 10: [0x69, 0x30], + 11: [0x6c, 0x30], + 12: [0x65, 0x31, 0x32, 0x33] + }, + extractor: null + }, { name: "Icon image", extension: "ico", @@ -363,10 +385,40 @@ export const FILE_SIGNATURES = { 3: 0x00 }, extractor: null + }, + { + name: "Joint Photographic Experts Group image (under Base64)", + extension: "B64", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x2f, + 1: 0x39, + 2: 0x6a, + 3: 0x2f, + 4: 0x34 + }, + extractor: null + }, + { + name: "Portable Network Graphics image (under Base64)", + extension: "B64", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x69, + 1: 0x56, + 2: 0x42, + 3: 0x4f, + 4: 0x52, + 5: 0x77, + 6: 0x30 + }, + extractor: null } ], "Video": [ - { // Place before webm + { name: "Matroska Multimedia Container", extension: "mkv", mime: "video/x-matroska", @@ -392,7 +444,28 @@ export const FILE_SIGNATURES = { 0: 0x1a, 1: 0x45, 2: 0xdf, - 3: 0xa3 + 3: 0xa3, + 4: 0x01, + 5: 0x00, + 6: 0x00, + 7: 0x00 + }, + extractor: null + }, + { // Place before MPEG-4 + name: "Flash MP4 video", + extension: "f4v", + mime: "video/mp4", + description: "", + signature: { + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70, + 8: [0x66, 0x46], + 9: 0x34, + 10: [0x76, 0x56], + 11: 0x20 }, extractor: null }, @@ -766,6 +839,41 @@ export const FILE_SIGNATURES = { }, extractor: extractPDF }, + { + name: "Portable Document Format (under Base64)", + extension: "B64", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x41, + 1: 0x4a, + 2: 0x56, + 3: 0x42, + 4: 0x45, + 5: 0x52, + 6: 0x69 + }, + extractor: null + }, + { // Place before PostScript + name: "Adobe PostScript", + extension: "ps,eps,ai,pfa", + mime: "application/postscript", + description: "", + signature: { + 0: 0x25, + 1: 0x21, + 2: 0x50, + 3: 0x53, + 4: 0x2d, + 5: 0x41, + 6: 0x64, + 7: 0x6f, + 8: 0x62, + 9: 0x65 + }, + extractor: null + }, { name: "PostScript", extension: "ps", @@ -792,7 +900,7 @@ export const FILE_SIGNATURES = { extractor: extractRTF }, { - name: "Microsoft Office documents/OLE2", + name: "Microsoft Office document/OLE2", extension: "ole2,doc,xls,dot,ppt,xla,ppa,pps,pot,msi,sdw,db,vsd,msg", mime: "application/msword,application/vnd.ms-excel,application/vnd.ms-powerpoint", description: "Microsoft Office documents", @@ -809,7 +917,24 @@ export const FILE_SIGNATURES = { extractor: null }, { - name: "Microsoft Office 2007+ documents", + name: "Microsoft Office document/OLE2 (under Base64)", + extension: "B64", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x30, + 1: 0x4d, + 2: 0x38, + 3: 0x52, + 4: 0x34, + 5: 0x4b, + 6: 0x47, + 7: 0x78 + }, + extractor: null + }, + { + name: "Microsoft Office 2007+ document", extension: "docx,xlsx,pptx", mime: "application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/vnd.openxmlformats-officedocument.presentationml.presentation", description: "", @@ -828,6 +953,131 @@ export const FILE_SIGNATURES = { }, extractor: extractZIP }, + { + name: "Microsoft Access database", + extension: "mdb,mda,mde,mdt,fdb,psa", + mime: "application/msaccess", + description: "", + signature: { + 0: 0x00, + 1: 0x01, + 2: 0x00, + 3: 0x00, + 4: 0x53, // Standard Jet + 5: 0x74, + 6: 0x61, + 7: 0x6e, + 8: 0x64, + 9: 0x61, + 10: 0x72, + 11: 0x64, + 12: 0x20, + 13: 0x4a, + 14: 0x65, + 15: 0x74 + }, + extractor: null + }, + { + name: "Microsoft Access 2007+ database", + extension: "accdb,accde,accda,accdu", + mime: "application/msaccess", + description: "", + signature: { + 0: 0x00, + 1: 0x01, + 2: 0x00, + 3: 0x00, + 4: 0x53, // Standard ACE DB + 5: 0x74, + 6: 0x61, + 7: 0x6e, + 8: 0x64, + 9: 0x61, + 10: 0x72, + 11: 0x64, + 12: 0x20, + 13: 0x41, + 14: 0x43, + 15: 0x45, + 16: 0x20 + }, + extractor: null + }, + { + name: "Microsoft OneNote document", + extension: "one", + mime: "application/onenote", + description: "", + signature: { + 0: 0xe4, + 1: 0x52, + 2: 0x5c, + 3: 0x7b, + 4: 0x8c, + 5: 0xd8, + 6: 0xa7, + 7: 0x4d, + 8: 0xae, + 9: 0xb1, + 10: 0x53, + 11: 0x78, + 12: 0xd0, + 13: 0x29, + 14: 0x96, + 15: 0xd3 + }, + extractor: null + }, + { + name: "Outlook Express database", + extension: "dbx", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0xcf, + 1: 0xad, + 2: 0x12, + 3: 0xfe, + 4: [0x30, 0xc5, 0xc6, 0xc7], + 11: 0x11 + }, + extractor: null + }, + { + name: "Personal Storage Table (Outlook)", + extension: "pst,ost,fdb,pab", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x21, // !BDN + 1: 0x42, + 2: 0x44, + 3: 0x4e + }, + extractor: null + }, + { + name: "Microsoft Exchange Database", + extension: "edb", + mime: "application/octet-stream", + description: "", + signature: { + 4: 0xef, + 5: 0xcd, + 6: 0xab, + 7: 0x89, + 8: [0x20, 0x23], + 9: 0x06, + 10: 0x00, + 11: 0x00, + 12: [0x00, 0x01], + 13: 0x00, + 14: 0x00, + 15: 0x00 + }, + extractor: null + }, { name: "EPUB e-book", extension: "epub", @@ -897,6 +1147,36 @@ export const FILE_SIGNATURES = { }, extractor: extractELF }, + { + name: "MacOS Mach-O object file", + extension: "dylib", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0xca, + 1: 0xfe, + 2: 0xba, + 3: 0xbe, + 4: 0x00, + 5: 0x00, + 6: 0x00, + 7: [0x01, 0x02, 0x03] + }, + extractor: null + }, + { + name: "MacOS Mach-O 64-bit object file", + extension: "dylib", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0xcf, + 1: 0xfa, + 2: 0xed, + 3: 0xfe + }, + extractor: null + }, { name: "Adobe Flash", extension: "swf", @@ -905,7 +1185,7 @@ export const FILE_SIGNATURES = { signature: { 0: [0x43, 0x46], 1: 0x57, - 2: 0x53 + 2: 0x53, }, extractor: null }, @@ -967,13 +1247,28 @@ export const FILE_SIGNATURES = { }, extractor: extractZIP }, + { + name: "PKZIP archive (under Base64)", + extension: "B64", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x55, + 1: 0x45, + 2: 0x73, + 3: 0x44, + 4: 0x42, + 5: 0x42 + }, + extractor: null + }, { name: "TAR archive", extension: "tar", mime: "application/x-tar", description: "", signature: { - 257: 0x75, + 257: 0x75, // ustar 258: 0x73, 259: 0x74, 260: 0x61, @@ -1111,7 +1406,45 @@ export const FILE_SIGNATURES = { signature: { 0: 0x4b, 1: 0x44, - 2: 0x4d + 2: 0x4d, + 3: 0x56, + 5: 0x00, + 6: 0x00, + 7: 0x00 + }, + extractor: null + }, + { + name: "Virtual Hard Drive", + extension: "vhd", + mime: "application/x-vhd", + description: "", + signature: { + 0: 0x63, // conectix + 1: 0x6f, + 2: 0x6e, + 3: 0x65, + 4: 0x63, + 5: 0x74, + 6: 0x69, + 7: 0x78 + }, + extractor: null + }, + { + name: "Macintosh disk image", + extension: "dmf,dmg", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x78, + 1: 0x01, + 2: 0x73, + 3: 0x0d, + 4: 0x62, + 5: 0x62, + 6: 0x60, + 7: 0x60 }, extractor: null }, @@ -1218,7 +1551,37 @@ export const FILE_SIGNATURES = { 9: 0x2d }, extractor: null - } + }, + { + name: "Microsoft Cabinet file", + extension: "cab", + mime: "vnd.ms-cab-compressed", + description: "", + signature: { + 0: 0x4d, + 1: 0x53, + 2: 0x43, + 3: 0x46, + 4: 0x00, + 5: 0x00, + 6: 0x00, + 7: 0x00 + }, + extractor: null + }, + { + name: "Jar Archive", + extension: "jar", + mime: "application/java-archive", + description: "", + signature: { + 0: 0x5f, + 1: 0x27, + 2: 0xa8, + 3: 0x89 + }, + extractor: null + }, ], "Miscellaneous": [ { @@ -1398,7 +1761,213 @@ export const FILE_SIGNATURES = { } ], extractor: null - } + }, + { + name: "Cryptocurrency wallet", + extension: "wallet", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x00, + 1: 0x00, + 2: 0x00, + 3: 0x00, + 4: 0x01, + 5: 0x00, + 6: 0x00, + 7: 0x00, + 8: 0x00, + 9: 0x00, + 10: 0x00, + 11: 0x00, + 12: 0x62, + 13: 0x31, + 14: 0x05, + 15: 0x00 + }, + extractor: null + }, + { + name: "Registry fragment", + extension: "hbin", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x68, // hbin + 1: 0x62, + 2: 0x69, + 3: 0x6e, + 4: 0x00 + }, + extractor: null + }, + { + name: "Registry script", + extension: "rgs", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x48, // HKCR + 1: 0x4b, + 2: 0x43, + 3: 0x52, + 4: 0x0d, + 5: 0x0a, + 6: 0x5c, + 7: 0x7b + }, + extractor: null + }, + { + name: "WinNT Registry Hive", + extension: "registry", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x72, + 1: 0x65, + 2: 0x67, + 3: 0x66 + }, + extractor: null + }, + { + name: "Windows Event Log", + extension: "evt", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x30, + 1: 0x00, + 2: 0x00, + 3: 0x00, + 4: 0x4c, + 5: 0x66, + 6: 0x4c, + 7: 0x65 + }, + extractor: null + }, + { + name: "Windows Event Log", + extension: "evtx", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x45, // ElfFile + 1: 0x6c, + 2: 0x66, + 3: 0x46, + 4: 0x69, + 5: 0x6c, + 6: 0x65 + }, + extractor: null + }, + { + name: "Windows Pagedump", + extension: "dmp", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0x50, // PAGEDU(MP|64) + 1: 0x41, + 2: 0x47, + 3: 0x45, + 4: 0x44, + 5: 0x55, + 6: [0x4d, 0x36], + 7: [0x50, 0x34] + }, + extractor: null + }, + { + name: "Windows Prefetch", + extension: "pf", + mime: "application/x-pf", + description: "", + signature: { + 0: [0x11, 0x17, 0x1a], + 1: 0x0, + 2: 0x0, + 3: 0x0, + 4: 0x53, + 5: 0x43, + 6: 0x43, + 7: 0x41 + }, + extractor: null + }, + { + name: "Windows Prefetch (Win 10)", + extension: "pf", + mime: "application/x-pf", + description: "", + signature: { + 0: 0x4d, + 1: 0x41, + 2: 0x4d, + 3: 0x04, + 7: 0x0 + }, + extractor: null + }, + { + name: "PList (XML)", + extension: "plist", + mime: "application/xml", + description: "", + signature: { + 39: 0x3c, // \n) + stream.continueUntil([0x3c, 0x2f, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x3e, 0x0a]); + stream.moveForwardsBy(9); + + return stream.carve(); +} + + /** * GZIP extractor. * diff --git a/src/core/operations/DetectFileType.mjs b/src/core/operations/DetectFileType.mjs index 7ddef0f9..4ffaa5ff 100644 --- a/src/core/operations/DetectFileType.mjs +++ b/src/core/operations/DetectFileType.mjs @@ -23,10 +23,10 @@ class DetectFileType extends Operation { this.module = "Default"; this.description = "Attempts to guess the MIME (Multipurpose Internet Mail Extensions) type of the data based on 'magic bytes'.

Currently supports the following file types: " + Object.keys(FILE_SIGNATURES).map(cat => - FILE_SIGNATURES[cat].map(sig => - sig.extension.split(",")[0] - ).join(", ") - ).join(", ") + "."; + [].concat.apply([], FILE_SIGNATURES[cat].map(sig => + sig.extension.split(",") + )).unique().join(", ") + ).unique().join(", ") + "."; this.infoURL = "https://wikipedia.org/wiki/List_of_file_signatures"; this.inputType = "ArrayBuffer"; this.outputType = "string"; From 10c810147676906bc01f230f0e2474a6e2ba6fc4 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Oct 2019 16:54:26 +0100 Subject: [PATCH 14/18] 9.7.5 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 558972ea..f91ee6ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.4", + "version": "9.7.5", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 7cd3d598..191c7ac6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.4", + "version": "9.7.5", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 087cc6b8fd21443908cd3614743bd468203d92f8 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Oct 2019 17:11:31 +0100 Subject: [PATCH 15/18] Fixed webm signature --- src/core/lib/FileSignatures.mjs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index afb4b6e0..5c7d0e4f 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -418,7 +418,7 @@ export const FILE_SIGNATURES = { } ], "Video": [ - { + { // Place before webm name: "Matroska Multimedia Container", extension: "mkv", mime: "video/x-matroska", @@ -444,11 +444,7 @@ export const FILE_SIGNATURES = { 0: 0x1a, 1: 0x45, 2: 0xdf, - 3: 0xa3, - 4: 0x01, - 5: 0x00, - 6: 0x00, - 7: 0x00 + 3: 0xa3 }, extractor: null }, From 6810f38808663215867d82ebe5f213ec17af97fb Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Oct 2019 17:11:36 +0100 Subject: [PATCH 16/18] 9.7.6 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index f91ee6ce..5ad2e504 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.5", + "version": "9.7.6", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 191c7ac6..e82908f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.5", + "version": "9.7.6", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From da901e20d908a320b4c81dd3ee8f07da2fef5a6d Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 4 Oct 2019 17:52:09 +0100 Subject: [PATCH 17/18] Added several more file signatures. The background magic button now highlights when a file type has been detected. --- src/core/lib/FileSignatures.mjs | 395 ++++++++++++++++++++++++- src/core/lib/Magic.mjs | 1 + src/core/operations/DetectFileType.mjs | 13 +- src/web/waiters/OutputWaiter.mjs | 31 +- 4 files changed, 411 insertions(+), 29 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 5c7d0e4f..27d2e676 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -415,7 +415,59 @@ export const FILE_SIGNATURES = { 6: 0x30 }, extractor: null - } + }, + { + name: "AutoCAD Drawing", + extension: "dwg,123d", + mime: "application/acad", + description: "", + signature: { + 0: 0x41, + 1: 0x43, + 2: 0x31, + 3: 0x30, + 4: [0x30, 0x31], + 5: [0x30, 0x31, 0x32, 0x33, 0x34, 0x35], + 6: 0x00 + }, + extractor: null + }, + { + name: "AutoCAD Drawing", + extension: "dwg,dwt", + mime: "application/acad", + description: "", + signature: [ + { + 0: 0x41, + 1: 0x43, + 2: 0x31, + 3: 0x30, + 4: 0x31, + 5: 0x38, + 6: 0x00 + }, + { + 0: 0x41, + 1: 0x43, + 2: 0x31, + 3: 0x30, + 4: 0x32, + 5: 0x34, + 6: 0x00 + }, + { + 0: 0x41, + 1: 0x43, + 2: 0x31, + 3: 0x30, + 4: 0x32, + 5: 0x37, + 6: 0x00 + } + ], + extractor: null + }, ], "Video": [ { // Place before webm @@ -614,6 +666,59 @@ export const FILE_SIGNATURES = { }, extractor: extractFLV }, + { + name: "OGG Video", + extension: "ogv,ogm,opus,ogx", + mime: "video/ogg", + description: "", + signature: [ + { + 0: 0x4f, // OggS + 1: 0x67, + 2: 0x67, + 3: 0x53, + 4: 0x00, + 5: 0x02, + 28: 0x01, + 29: 0x76, // video + 30: 0x69, + 31: 0x64, + 32: 0x65, + 33: 0x6f + }, + { + 0: 0x4f, // OggS + 1: 0x67, + 2: 0x67, + 3: 0x53, + 4: 0x00, + 5: 0x02, + 28: 0x80, + 29: 0x74, // theora + 30: 0x68, + 31: 0x65, + 32: 0x6f, + 33: 0x72, + 34: 0x61 + }, + { + 0: 0x4f, // OggS + 1: 0x67, + 2: 0x67, + 3: 0x53, + 4: 0x00, + 5: 0x02, + 28: 0x66, // fishead + 29: 0x69, + 30: 0x73, + 31: 0x68, + 32: 0x65, + 33: 0x61, + 34: 0x64 + } + ], + extractor: null + }, ], "Audio": [ { @@ -881,6 +986,19 @@ export const FILE_SIGNATURES = { }, extractor: null }, + { + name: "Encapsulated PostScript", + extension: "eps,ai", + mime: "application/eps", + description: "", + signature: { + 0: 0xc5, + 1: 0xd0, + 2: 0xd3, + 3: 0xc6 + }, + extractor: null + }, { name: "Rich Text Format", extension: "rtf", @@ -1074,6 +1192,22 @@ export const FILE_SIGNATURES = { }, extractor: null }, + { + name: "WordPerfect document", + extension: "wpd,wp,wp5,wp6,wpp,bk!,wcm", + mime: "application/wordperfect", + description: "", + signature: { + 0: 0xff, + 1: 0x57, + 2: 0x50, + 3: 0x43, + 7: [0x00, 0x01, 0x02], + 8: 0x01, + 9: 0x0a + }, + extractor: null + }, { name: "EPUB e-book", extension: "epub", @@ -1120,7 +1254,7 @@ export const FILE_SIGNATURES = { { name: "Windows Portable Executable", extension: "exe,dll,drv,vxd,sys,ocx,vbx,com,fon,scr", - mime: "application/x-msdownload", + mime: "application/vnd.microsoft.portable-executable", description: "", signature: { 0: 0x4d, @@ -1131,7 +1265,7 @@ export const FILE_SIGNATURES = { extractor: extractMZPE }, { - name: "Executable and Linkable Format file", + name: "Executable and Linkable Format", extension: "elf,bin,axf,o,prx,so", mime: "application/x-executable", description: "Executable and Linkable Format file. No standard file extension.", @@ -1144,7 +1278,7 @@ export const FILE_SIGNATURES = { extractor: extractELF }, { - name: "MacOS Mach-O object file", + name: "MacOS Mach-O object", extension: "dylib", mime: "application/octet-stream", description: "", @@ -1161,7 +1295,7 @@ export const FILE_SIGNATURES = { extractor: null }, { - name: "MacOS Mach-O 64-bit object file", + name: "MacOS Mach-O 64-bit object", extension: "dylib", mime: "application/octet-stream", description: "", @@ -1549,7 +1683,7 @@ export const FILE_SIGNATURES = { extractor: null }, { - name: "Microsoft Cabinet file", + name: "Microsoft Cabinet", extension: "cab", mime: "vnd.ms-cab-compressed", description: "", @@ -1578,10 +1712,27 @@ export const FILE_SIGNATURES = { }, extractor: null }, + { + name: "lzop compressed", + extension: "lzop,lzo", + mime: "application/x-lzop", + description: "", + signature: { + 0: 0x89, + 1: 0x4c, // LZO + 2: 0x5a, + 3: 0x4f, + 4: 0x00, + 5: 0x0d, + 6: 0x0a, + 7: 0x1a + }, + extractor: null + }, ], "Miscellaneous": [ { - name: "UTF-8 text file", + name: "UTF-8 text", extension: "txt", mime: "text/plain", description: "UTF-8 encoded Unicode byte order mark, commonly but not exclusively seen in text files.", @@ -1592,8 +1743,8 @@ export const FILE_SIGNATURES = { }, extractor: null }, - { // Place before UTF-16 LE file - name: "UTF-32 LE file", + { // Place before UTF-16 LE text + name: "UTF-32 LE text", extension: "utf32le", mime: "charset/utf32le", description: "Little-endian UTF-32 encoded Unicode byte order mark.", @@ -1606,7 +1757,7 @@ export const FILE_SIGNATURES = { extractor: null }, { - name: "UTF-16 LE file", + name: "UTF-16 LE text", extension: "utf16le", mime: "charset/utf16le", description: "Little-endian UTF-16 encoded Unicode byte order mark.", @@ -1964,6 +2115,223 @@ export const FILE_SIGNATURES = { }, extractor: null }, + { + name: "TCP Packet", + extension: "tcp", + mime: "application/tcp", + description: "", + signature: { + 12: 0x08, + 13: 0x00, + 14: 0x45, + 15: 0x00, + 21: 0x00, + 22: b => b >= 0x01 && b <= 0x80, + 23: 0x06 + }, + extractor: null + }, + { + name: "UDP Packet", + extension: "udp", + mime: "application/udp", + description: "", + signature: { + 12: 0x08, + 13: 0x00, + 14: 0x45, + 15: 0x00, + 16: [0x00, 0x01, 0x02, 0x03, 0x04, 0x05], + 22: b => b >= 0x01 && b <= 0x80, + 23: 0x11 + }, + extractor: null + }, + { + name: "Compiled HTML", + extension: "chm,chw,chi", + mime: "application/vnd.ms-htmlhelp", + description: "", + signature: { + 0: 0x49, // ITSF + 1: 0x54, + 2: 0x53, + 3: 0x46, + 4: 0x03, + 5: 0x00, + 6: 0x00, + 7: 0x00 + }, + extractor: null + }, + { + name: "Windows Password", + extension: "pwl", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0xe3, + 1: 0x82, + 2: 0x85, + 3: 0x96 + }, + extractor: null + }, + { + name: "Bitlocker recovery key", + extension: "bitlocker", + mime: "application/octet-stream", + description: "", + signature: { + 0: 0xff, + 1: 0xfe, + 2: 0x42, + 3: 0x00, + 4: 0x69, + 5: 0x00, + 6: 0x74, + 7: 0x00, + 8: 0x4c, + 9: 0x00, + 10: 0x6f, + 11: 0x00, + 12: 0x63, + 13: 0x00, + 14: 0x6b, + 15: 0x00, + 16: 0x65, + 17: 0x00, + 18: 0x72, + 19: 0x00, + 20: 0x20, + 21: 0x00 + }, + extractor: null + }, + { + name: "Certificate", + extension: "cer,cat,p7b,p7c,p7m,p7s,swz,rsa,crl,crt,der", + mime: "application/pkix-cert", + description: "", + signature: { + 0: 0x30, + 1: 0x82, + 4: [0x06, 0x0a, 0x30] + }, + extractor: null + }, + { + name: "Certificate", + extension: "cat,swz,p7m", + mime: "application/vnd.ms-pki.seccat", + description: "", + signature: { + 0: 0x30, + 1: 0x83, + 2: b => b !== 0x00, + 5: 0x06, + 6: 0x09 + }, + extractor: null + }, + { + name: "PGP pubring", + extension: "pkr,gpg", + mime: "application/pgp-keys", + description: "", + signature: { + 0: 0x99, + 1: 0x01, + 2: [0x0d, 0xa2], + 3: 0x04 + }, + extractor: null + }, + { + name: "PGP secring", + extension: "skr", + mime: "application/pgp-keys", + description: "", + signature: [ + { + 0: 0x95, + 1: 0x01, + 2: 0xcf, + 3: 0x04 + }, + { + 0: 0x95, + 1: 0x03, + 2: 0xc6, + 3: 0x04 + } + ], + extractor: null + }, + { + name: "PGP Safe", + extension: "pgd", + mime: "application/pgp-keys", + description: "", + signature: { + 0: 0x50, // PGPdMAIN + 1: 0x47, + 2: 0x50, + 3: 0x64, + 4: 0x4d, + 5: 0x41, + 6: 0x49, + 7: 0x4e, + 8: 0x60, + 9: 0x01, + 10: 0x00 + }, + extractor: null + }, + { + name: "Task Scheduler", + extension: "job", + mime: "application/octet-stream", + description: "", + signature: { + 0: [0x00, 0x01, 0x02, 0x03], + 1: [0x05, 0x06], + 2: 0x01, + 3: 0x00, + 20: 0x46, + 21: 0x00 + }, + extractor: null + }, + { + name: "Windows Shortcut", + extension: "lnk", + mime: "application/x-ms-shortcut", + description: "", + signature: { + 0: 0x4c, + 1: 0x00, + 2: 0x00, + 3: 0x00, + 4: 0x01, + 5: 0x14, + 6: 0x02, + 7: 0x00, + 8: 0x00, + 9: 0x00, + 10: 0x00, + 11: 0x00, + 12: 0xc0, + 13: 0x00, + 14: 0x00, + 15: 0x00, + 16: 0x00, + 17: 0x00, + 18: 0x00, + 19: 0x46 + }, + extractor: null + } ] }; @@ -2307,9 +2675,10 @@ export function extractRTF(bytes, offset) { export function extractPListXML(bytes, offset) { const stream = new Stream(bytes.slice(offset)); - // Find closing tag (\n) - stream.continueUntil([0x3c, 0x2f, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x3e, 0x0a]); - stream.moveForwardsBy(9); + // Find closing tag () + stream.continueUntil([0x3c, 0x2f, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x3e]); + stream.moveForwardsBy(8); + stream.consumeIf(0x0a); return stream.carve(); } diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index f6802e97..68a36bfa 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -97,6 +97,7 @@ class Magic { if (!fileType.length) return null; return { + name: fileType[0].name, ext: fileType[0].extension, mime: fileType[0].mime, desc: fileType[0].description diff --git a/src/core/operations/DetectFileType.mjs b/src/core/operations/DetectFileType.mjs index 4ffaa5ff..c8cdb822 100644 --- a/src/core/operations/DetectFileType.mjs +++ b/src/core/operations/DetectFileType.mjs @@ -8,6 +8,13 @@ import Operation from "../Operation.mjs"; import {detectFileType} from "../lib/FileType.mjs"; import {FILE_SIGNATURES} from "../lib/FileSignatures.mjs"; +// Concat all supported extensions into a single flat list +const exts = [].concat.apply([], Object.keys(FILE_SIGNATURES).map(cat => + [].concat.apply([], FILE_SIGNATURES[cat].map(sig => + sig.extension.split(",") + )) +)).unique().sort().join(", "); + /** * Detect File Type operation */ @@ -22,11 +29,7 @@ class DetectFileType extends Operation { this.name = "Detect File Type"; this.module = "Default"; this.description = "Attempts to guess the MIME (Multipurpose Internet Mail Extensions) type of the data based on 'magic bytes'.

Currently supports the following file types: " + - Object.keys(FILE_SIGNATURES).map(cat => - [].concat.apply([], FILE_SIGNATURES[cat].map(sig => - sig.extension.split(",") - )).unique().join(", ") - ).unique().join(", ") + "."; + exts + "."; this.infoURL = "https://wikipedia.org/wiki/List_of_file_signatures"; this.inputType = "ArrayBuffer"; this.outputType = "string"; diff --git a/src/web/waiters/OutputWaiter.mjs b/src/web/waiters/OutputWaiter.mjs index cea2b514..833922b8 100755 --- a/src/web/waiters/OutputWaiter.mjs +++ b/src/web/waiters/OutputWaiter.mjs @@ -1046,15 +1046,25 @@ class OutputWaiter { * @param {Object[]} options */ backgroundMagicResult(options) { - if (!options.length || - !options[0].recipe.length) - return; + if (!options.length) return; const currentRecipeConfig = this.app.getRecipeConfig(); - const newRecipeConfig = currentRecipeConfig.concat(options[0].recipe); - const opSequence = options[0].recipe.map(o => o.op).join(", "); + let msg = "", + newRecipeConfig; - this.showMagicButton(opSequence, options[0].data, newRecipeConfig); + if (options[0].recipe.length) { + const opSequence = options[0].recipe.map(o => o.op).join(", "); + newRecipeConfig = currentRecipeConfig.concat(options[0].recipe); + msg = `${opSequence} will produce "${Utils.escapeHtml(Utils.truncate(options[0].data), 30)}"`; + } else if (options[0].fileType && options[0].fileType.name) { + const ft = options[0].fileType; + newRecipeConfig = currentRecipeConfig.concat([{op: "Detect File Type", args: []}]); + msg = `${ft.name} file detected`; + } else { + return; + } + + this.showMagicButton(msg, newRecipeConfig); } /** @@ -1072,15 +1082,14 @@ class OutputWaiter { } /** - * Displays the Magic button with a title and adds a link to a complete recipe. + * Displays the Magic button with a title and adds a link to a recipe. * - * @param {string} opSequence - * @param {string} result + * @param {string} msg * @param {Object[]} recipeConfig */ - showMagicButton(opSequence, result, recipeConfig) { + showMagicButton(msg, recipeConfig) { const magicButton = document.getElementById("magic"); - magicButton.setAttribute("data-original-title", `${opSequence} will produce "${Utils.escapeHtml(Utils.truncate(result), 30)}"`); + magicButton.setAttribute("data-original-title", msg); magicButton.setAttribute("data-recipe", JSON.stringify(recipeConfig), null, ""); magicButton.classList.remove("hidden"); magicButton.classList.add("pulse"); From 06708949a15dd7266f3f965324858b6f611fba39 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 4 Oct 2019 17:52:15 +0100 Subject: [PATCH 18/18] 9.7.7 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5ad2e504..11c80ca0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.6", + "version": "9.7.7", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index e82908f0..e9c33484 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "9.7.6", + "version": "9.7.7", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef",