From 71c743ff5a6c4f27c9088329bb41211c10845336 Mon Sep 17 00:00:00 2001 From: Cynser <42423063+Cynser@users.noreply.github.com> Date: Wed, 12 Dec 2018 17:34:45 +0000 Subject: [PATCH 01/25] Add Text Encoding Brute Force operation --- src/core/config/Categories.json | 1 + .../operations/TextEncodingBruteForce.mjs | 52 +++++++++++++++++++ test/index.mjs | 1 + .../operations/TextEncodingBruteForce.mjs | 24 +++++++++ 4 files changed, 78 insertions(+) create mode 100644 src/core/operations/TextEncodingBruteForce.mjs create mode 100644 test/tests/operations/TextEncodingBruteForce.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 4fac84c9..9db13647 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -49,6 +49,7 @@ "Change IP format", "Encode text", "Decode text", + "Text Encoding Brute Force", "Swap endianness", "To MessagePack", "From MessagePack", diff --git a/src/core/operations/TextEncodingBruteForce.mjs b/src/core/operations/TextEncodingBruteForce.mjs new file mode 100644 index 00000000..ef2d6500 --- /dev/null +++ b/src/core/operations/TextEncodingBruteForce.mjs @@ -0,0 +1,52 @@ +/** + * @author Cynser + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import Utils from "../Utils"; +import cptable from "../vendor/js-codepage/cptable.js"; +import {IO_FORMAT} from "../lib/ChrEnc"; + +/** + * Text Encoding Brute Force operation + */ +class TextEncodingBruteForce extends Operation { + + /** + * TextEncodingBruteForce constructor + */ + constructor() { + super(); + + this.name = "Text Encoding Brute Force"; + this.module = "CharEnc"; + this.description = "Enumerate all possible text encodings for input."; + this.infoURL = "https://wikipedia.org/wiki/Character_encoding"; + this.inputType = "string"; + this.outputType = "string"; + this.args = []; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const output = [], + charSets = Object.keys(IO_FORMAT); + + for (let i = 0; i < charSets.length; i++) { + let currentEncoding = Utils.printable(charSets[i] + ": ", false); + currentEncoding += cptable.utils.encode(IO_FORMAT[charSets[i]], input); + output.push(currentEncoding); + } + + return output.join("\n"); + } + +} + +export default TextEncodingBruteForce; diff --git a/test/index.mjs b/test/index.mjs index 9c11f6ae..89e09ea4 100644 --- a/test/index.mjs +++ b/test/index.mjs @@ -74,6 +74,7 @@ import "./tests/operations/SetIntersection"; import "./tests/operations/SetUnion"; import "./tests/operations/StrUtils"; import "./tests/operations/SymmetricDifference"; +import "./tests/operations/TextEncodingBruteForce"; import "./tests/operations/ToGeohash.mjs"; import "./tests/operations/TranslateDateTimeFormat"; import "./tests/operations/Magic"; diff --git a/test/tests/operations/TextEncodingBruteForce.mjs b/test/tests/operations/TextEncodingBruteForce.mjs new file mode 100644 index 00000000..5316eb16 --- /dev/null +++ b/test/tests/operations/TextEncodingBruteForce.mjs @@ -0,0 +1,24 @@ +/** + * Text Encoding Brute Force tests. + * + * @author Cynser + * + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ +import TestRegister from "../../TestRegister"; + +TestRegister.addTests([ + { + name: "Text Encoding Brute Force", + input: "Булкі праз ляніва сабаку.", + expectedMatch: /Windows\-1251 Cyrillic \(1251\): Булкі праз ляніва сабаку\./, + recipeConfig: [ + { + op: "Text Encoding Brute Force", + args: [], + }, + ], + } +]); + From 3f7059a2352fc397ec035233f6958877687ceae3 Mon Sep 17 00:00:00 2001 From: Cynser <42423063+Cynser@users.noreply.github.com> Date: Wed, 12 Dec 2018 17:49:11 +0000 Subject: [PATCH 02/25] Remove unnecessary escape character --- test/tests/operations/TextEncodingBruteForce.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests/operations/TextEncodingBruteForce.mjs b/test/tests/operations/TextEncodingBruteForce.mjs index 5316eb16..e79b24af 100644 --- a/test/tests/operations/TextEncodingBruteForce.mjs +++ b/test/tests/operations/TextEncodingBruteForce.mjs @@ -12,7 +12,7 @@ TestRegister.addTests([ { name: "Text Encoding Brute Force", input: "Булкі праз ляніва сабаку.", - expectedMatch: /Windows\-1251 Cyrillic \(1251\): Булкі праз ляніва сабаку\./, + expectedMatch: /Windows-1251 Cyrillic \(1251\): Булкі праз ляніва сабаку\./, recipeConfig: [ { op: "Text Encoding Brute Force", From dcff8971e8ebaf9e60d6d1900e64c851d70c11c2 Mon Sep 17 00:00:00 2001 From: Jarmo van Lenthe Date: Fri, 14 Dec 2018 22:29:51 +0100 Subject: [PATCH 03/25] Added simple A1Z26 'cipher' --- src/core/config/Categories.json | 2 + src/core/operations/A1Z26CipherDecode.mjs | 63 +++++++++++++++++++++++ src/core/operations/A1Z26CipherEncode.mjs | 61 ++++++++++++++++++++++ test/tests/operations/Ciphers.mjs | 33 ++++++++++++ 4 files changed, 159 insertions(+) create mode 100644 src/core/operations/A1Z26CipherDecode.mjs create mode 100644 src/core/operations/A1Z26CipherEncode.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 4fac84c9..e9fe3399 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -86,6 +86,8 @@ "Bifid Cipher Decode", "Affine Cipher Encode", "Affine Cipher Decode", + "A1Z26 Cipher Encode", + "A1Z26 Cipher Decode", "Atbash Cipher", "Substitute", "Derive PBKDF2 key", diff --git a/src/core/operations/A1Z26CipherDecode.mjs b/src/core/operations/A1Z26CipherDecode.mjs new file mode 100644 index 00000000..183bf047 --- /dev/null +++ b/src/core/operations/A1Z26CipherDecode.mjs @@ -0,0 +1,63 @@ +/** + * @author Jarmo van Lenthe [github.com/jarmovanlenthe] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import Utils from "../Utils"; +import {DELIM_OPTIONS} from "../lib/Delim"; +import OperationError from "../errors/OperationError"; + +/** + * A1Z26 Cipher Decode operation + */ +class A1Z26CipherDecode extends Operation { + + /** + * A1Z26CipherDecode constructor + */ + constructor() { + super(); + + this.name = "A1Z26 Cipher Decode"; + this.module = "Ciphers"; + this.description = "Converts alphabet order numbers into their corresponding alphabet character.

e.g. 1 becomes a and 2 becomes b."; + this.infoURL = ""; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + name: "Delimiter", + type: "option", + value: DELIM_OPTIONS + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const delim = Utils.charRep(args[0] || "Space"); + + if (input.length === 0) { + return []; + } + + let bites = input.split(delim), + latin1 = ""; + for (let i = 0; i < bites.length; i++) { + if (bites[i] < 1 || bites[i] > 26) { + throw new OperationError("Error: all numbers must be between 1 and 26."); + } + latin1 += Utils.chr(parseInt(bites[i], 10) + 96); + } + return latin1; + } + +} + +export default A1Z26CipherDecode; diff --git a/src/core/operations/A1Z26CipherEncode.mjs b/src/core/operations/A1Z26CipherEncode.mjs new file mode 100644 index 00000000..1abcb900 --- /dev/null +++ b/src/core/operations/A1Z26CipherEncode.mjs @@ -0,0 +1,61 @@ +/** + * @author Jarmo van Lenthe [github.com/jarmovanlenthe] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import Utils from "../Utils"; +import {DELIM_OPTIONS} from "../lib/Delim"; + +/** + * A1Z26 Cipher Encode operation + */ +class A1Z26CipherEncode extends Operation { + + /** + * A1Z26CipherEncode constructor + */ + constructor() { + super(); + + this.name = "A1Z26 Cipher Encode"; + this.module = "Ciphers"; + this.description = "Converts alphabet characters into their corresponding alphabet order number.

e.g. a becomes 1 and b becomes 2.

Non-alphabet characters are dropped."; + this.infoURL = ""; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + name: "Delimiter", + type: "option", + value: DELIM_OPTIONS + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const delim = Utils.charRep(args[0] || "Space"); + let output = ""; + + const sanitizedinput = input.toLowerCase(), + charcode = Utils.strToCharcode(sanitizedinput); + + for (let i = 0; i < charcode.length; i++) { + let ordinal = charcode[i] - 96; + + if (ordinal > 0 && ordinal <= 26) { + output += ordinal.toString(10) + delim; + } + } + return output.slice(0, -delim.length); + } + +} + +export default A1Z26CipherEncode; diff --git a/test/tests/operations/Ciphers.mjs b/test/tests/operations/Ciphers.mjs index fdc98a3a..a46f50a9 100644 --- a/test/tests/operations/Ciphers.mjs +++ b/test/tests/operations/Ciphers.mjs @@ -110,6 +110,39 @@ TestRegister.addTests([ } ], }, + { + name: "A1Z26 Encode: normal", + input: "This is the test sentence.", + expectedOutput: "20 8 9 19 9 19 20 8 5 20 5 19 20 19 5 14 20 5 14 3 5", + recipeConfig: [ + { + op: "A1Z26 Cipher Encode", + args: ["Space"] + } + ], + }, + { + name: "A1Z26 Decode: normal", + input: "20 8 9 19 9 19 20 8 5 20 5 19 20 19 5 14 20 5 14 3 5", + expectedOutput: "thisisthetestsentence", + recipeConfig: [ + { + op: "A1Z26 Cipher Decode", + args: ["Space"] + } + ], + }, + { + name: "A1Z26 Decode: error", + input: "20 8 9 27", + expectedOutput: "Error: all numbers must be between 1 and 26.", + recipeConfig: [ + { + op: "A1Z26 Cipher Decode", + args: ["Space"] + } + ], + }, { name: "Atbash: no input", input: "", From 63593f1b6c37430d520a4ef2a28f4c9dc1a22f9a Mon Sep 17 00:00:00 2001 From: Oliver Grubin Date: Fri, 14 Dec 2018 21:32:44 +0000 Subject: [PATCH 04/25] Fix HMAC operation when hex key has bytes >= 0x80 (#437) Add test vectors from RFC4231 --- src/core/operations/HMAC.mjs | 2 +- test/tests/operations/Hash.mjs | 298 ++++++++++++++++++++++++++++++++- 2 files changed, 298 insertions(+), 2 deletions(-) diff --git a/src/core/operations/HMAC.mjs b/src/core/operations/HMAC.mjs index e013737e..6517c581 100644 --- a/src/core/operations/HMAC.mjs +++ b/src/core/operations/HMAC.mjs @@ -72,7 +72,7 @@ class HMAC extends Operation { msg = Utils.arrayBufferToStr(input, false), hasher = CryptoApi.getHasher(hashFunc); - const mac = CryptoApi.getHmac(CryptoApi.encoder.fromUtf(key), hasher); + const mac = CryptoApi.getHmac(key, hasher); mac.update(msg); return CryptoApi.encoder.toHex(mac.finalize()); } diff --git a/test/tests/operations/Hash.mjs b/test/tests/operations/Hash.mjs index 8e774329..ec4a6dac 100644 --- a/test/tests/operations/Hash.mjs +++ b/test/tests/operations/Hash.mjs @@ -405,7 +405,7 @@ TestRegister.addTests([ ] }, { - name: "HMAC SHA256", + name: "HMAC: SHA256", input: "Hello, World!", expectedOutput: "52589bd80ccfa4acbb3f9512dfaf4f700fa5195008aae0b77a9e47dcca75beac", recipeConfig: [ @@ -415,6 +415,302 @@ TestRegister.addTests([ } ] }, + { + name: "HMAC: RFC4231 Test Case 1 SHA-224", + input: "Hi There", + expectedOutput: "896fb1128abbdf196832107cd49df33f47b4b1169912ba4f53684b22", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"}, "SHA224"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 1 SHA-256", + input: "Hi There", + expectedOutput: "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"}, "SHA256"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 1 SHA-384", + input: "Hi There", + expectedOutput: "afd03944d84895626b0825f4ab46907f15f9dadbe4101ec682aa034c7cebc59cfaea9ea9076ede7f4af152e8b2fa9cb6", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"}, "SHA384"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 1 SHA-512", + input: "Hi There", + expectedOutput: "87aa7cdea5ef619d4ff0b4241a1d6cb02379f4e2ce4ec2787ad0b30545e17cdedaa833b7d6b8a702038b274eaea3f4e4be9d914eeb61f1702e696c203a126854", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"}, "SHA512"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 2 SHA-224", + input: "what do ya want for nothing?", + expectedOutput: "a30e01098bc6dbbf45690f3a7e9e6d0f8bbea2a39e6148008fd05e44", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "4a656665"}, "SHA224"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 2 SHA-256", + input: "what do ya want for nothing?", + expectedOutput: "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "4a656665"}, "SHA256"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 2 SHA-384", + input: "what do ya want for nothing?", + expectedOutput: "af45d2e376484031617f78d2b58a6b1b9c7ef464f5a01b47e42ec3736322445e8e2240ca5e69e2c78b3239ecfab21649", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "4a656665"}, "SHA384"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 2 SHA-512", + input: "what do ya want for nothing?", + expectedOutput: "164b7a7bfcf819e2e395fbe73b56e0a387bd64222e831fd610270cd7ea2505549758bf75c05a994a6d034f65f8f0e6fdcaeab1a34d4a6b4b636e070a38bce737", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "4a656665"}, "SHA512"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 3 SHA-224", + input: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + expectedOutput: "7fb3cb3588c6c1f6ffa9694d7d6ad2649365b0c1f65d69d1ec8333ea", + recipeConfig: [ + { + "op": "From Hex", + "args": ["None"] + }, + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA224"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 3 SHA-256", + input: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + expectedOutput: "773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe", + recipeConfig: [ + { + "op": "From Hex", + "args": ["None"] + }, + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA256"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 3 SHA-384", + input: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + expectedOutput: "88062608d3e6ad8a0aa2ace014c8a86f0aa635d947ac9febe83ef4e55966144b2a5ab39dc13814b94e3ab6e101a34f27", + recipeConfig: [ + { + "op": "From Hex", + "args": ["None"] + }, + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA384"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 3 SHA-512", + input: "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + expectedOutput: "fa73b0089d56a284efb0f0756c890be9b1b5dbdd8ee81a3655f83e33b2279d39bf3e848279a722c806b485a47e67c807b946a337bee8942674278859e13292fb", + recipeConfig: [ + { + "op": "From Hex", + "args": ["None"] + }, + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA512"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 4 SHA-224", + input: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", + expectedOutput: "6c11506874013cac6a2abc1bb382627cec6a90d86efc012de7afec5a", + recipeConfig: [ + { + "op": "From Hex", + "args": ["None"] + }, + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "0102030405060708090a0b0c0d0e0f10111213141516171819"}, "SHA224"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 4 SHA-256", + input: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", + expectedOutput: "82558a389a443c0ea4cc819899f2083a85f0faa3e578f8077a2e3ff46729665b", + recipeConfig: [ + { + "op": "From Hex", + "args": ["None"] + }, + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "0102030405060708090a0b0c0d0e0f10111213141516171819"}, "SHA256"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 4 SHA-384", + input: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", + expectedOutput: "3e8a69b7783c25851933ab6290af6ca77a9981480850009cc5577c6e1f573b4e6801dd23c4a7d679ccf8a386c674cffb", + recipeConfig: [ + { + "op": "From Hex", + "args": ["None"] + }, + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "0102030405060708090a0b0c0d0e0f10111213141516171819"}, "SHA384"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 4 SHA-512", + input: "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd", + expectedOutput: "b0ba465637458c6990e5a8c5f61d4af7e576d97ff94b872de76f8050361ee3dba91ca5c11aa25eb4d679275cc5788063a5f19741120c4f2de2adebeb10a298dd", + recipeConfig: [ + { + "op": "From Hex", + "args": ["None"] + }, + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "0102030405060708090a0b0c0d0e0f10111213141516171819"}, "SHA512"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 6 SHA-224", + input: "Test Using Larger Than Block-Size Key - Hash Key First", + expectedOutput: "95e9a0db962095adaebe9b2d6f0dbce2d499f112f2d2b7273fa6870e", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA224"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 6 SHA-256", + input: "Test Using Larger Than Block-Size Key - Hash Key First", + expectedOutput: "60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA256"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 6 SHA-384", + input: "Test Using Larger Than Block-Size Key - Hash Key First", + expectedOutput: "4ece084485813e9088d2c63a041bc5b44f9ef1012a2b588f3cd11f05033ac4c60c2ef6ab4030fe8296248df163f44952", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA384"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 6 SHA-512", + input: "Test Using Larger Than Block-Size Key - Hash Key First", + expectedOutput: "80b24263c7c1a3ebb71493c1dd7be8b49b46d1f41b4aeec1121b013783f8f3526b56d037e05f2598bd0fd2215d6a1e5295e64f73f63f0aec8b915a985d786598", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA512"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 7 SHA-224", + input: "This is a test using a larger than block-size key and a larger than block-size data. The key needs to be hashed before being used by the HMAC algorithm.", + expectedOutput: "3a854166ac5d9f023f54d517d0b39dbd946770db9c2b95c9f6f565d1", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA224"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 7 SHA-256", + input: "This is a test using a larger than block-size key and a larger than block-size data. The key needs to be hashed before being used by the HMAC algorithm.", + expectedOutput: "9b09ffa71b942fcb27635fbcd5b0e944bfdc63644f0713938a7f51535c3a35e2", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA256"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 7 SHA-384", + input: "This is a test using a larger than block-size key and a larger than block-size data. The key needs to be hashed before being used by the HMAC algorithm.", + expectedOutput: "6617178e941f020d351e2f254e8fd32c602420feb0b8fb9adccebb82461e99c5a678cc31e799176d3860e6110c46523e", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA384"] + } + ] + }, + { + name: "HMAC: RFC4231 Test Case 7 SHA-512", + input: "This is a test using a larger than block-size key and a larger than block-size data. The key needs to be hashed before being used by the HMAC algorithm.", + expectedOutput: "e37b6a775dc87dbaa4dfa9f96e5e3ffddebd71f8867289865df5a32d20cdc944b6022cac3c4982b10d5eeb55c3e4de15134676fb6de0446065c97440fa8c6a58", + recipeConfig: [ + { + "op": "HMAC", + "args": [{"option": "Hex", "string": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "SHA512"] + } + ] + }, { name: "MD5: Complex bytes", input: "10dc10e32010de10d010dc10d810d910d010e12e", From b4a586c0b9cf635c41a6f36a27925d3086d90d83 Mon Sep 17 00:00:00 2001 From: Jarmo van Lenthe Date: Fri, 14 Dec 2018 22:35:43 +0100 Subject: [PATCH 05/25] Some lets to consts and removing of trailing spaces from grunt lint --- src/core/operations/A1Z26CipherDecode.mjs | 6 +++--- src/core/operations/A1Z26CipherEncode.mjs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/operations/A1Z26CipherDecode.mjs b/src/core/operations/A1Z26CipherDecode.mjs index 183bf047..2a9f9ce7 100644 --- a/src/core/operations/A1Z26CipherDecode.mjs +++ b/src/core/operations/A1Z26CipherDecode.mjs @@ -42,13 +42,13 @@ class A1Z26CipherDecode extends Operation { */ run(input, args) { const delim = Utils.charRep(args[0] || "Space"); - + if (input.length === 0) { return []; } - let bites = input.split(delim), - latin1 = ""; + const bites = input.split(delim); + let latin1 = ""; for (let i = 0; i < bites.length; i++) { if (bites[i] < 1 || bites[i] > 26) { throw new OperationError("Error: all numbers must be between 1 and 26."); diff --git a/src/core/operations/A1Z26CipherEncode.mjs b/src/core/operations/A1Z26CipherEncode.mjs index 1abcb900..d1202d83 100644 --- a/src/core/operations/A1Z26CipherEncode.mjs +++ b/src/core/operations/A1Z26CipherEncode.mjs @@ -42,12 +42,12 @@ class A1Z26CipherEncode extends Operation { run(input, args) { const delim = Utils.charRep(args[0] || "Space"); let output = ""; - + const sanitizedinput = input.toLowerCase(), charcode = Utils.strToCharcode(sanitizedinput); for (let i = 0; i < charcode.length; i++) { - let ordinal = charcode[i] - 96; + const ordinal = charcode[i] - 96; if (ordinal > 0 && ordinal <= 26) { output += ordinal.toString(10) + delim; From 31cbf8ccccc295d0ed6b472e890876d8c5da9ae7 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sat, 15 Dec 2018 00:16:23 +0000 Subject: [PATCH 06/25] 8.12.4 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4f1be0a3..3f1f599a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.12.3", + "version": "8.12.4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 4fac8034..338aced6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.12.3", + "version": "8.12.4", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 50f078cc45f221418519bb5e573d704e8a760c6d Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sat, 15 Dec 2018 00:26:15 +0000 Subject: [PATCH 07/25] Updated CHANGELOG --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87a81bdd..a65fc145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.13.0] - 2018-12-15 +- 'A1Z26 Cipher Encode' and 'A1Z26 Cipher Decode' operations added [@jarmovanlenthe] | [#441] + ### [8.12.0] - 2018-11-21 - 'Citrix CTX1 Encode' and 'Citrix CTX1 Decode' operations added [@bwhitn] | [#428] @@ -20,7 +23,7 @@ All major and minor version changes will be documented in this file. Details of - 'JWT Sign', 'JWT Verify' and 'JWT Decode' operations added [@GCHQ77703] | [#348] ### [8.6.0] - 2018-08-29 -- 'To Geohash' and 'From Geohash' operations added [@GCHQ77703] | [#344] +- 'To Geohash' and 'From Geohash' operations added [@GCHQ77703] | [#344] ### [8.5.0] - 2018-08-23 - 'To Braille' and 'From Braille' operations added [@n1474335] | [#255] @@ -66,6 +69,7 @@ All major and minor version changes will be documented in this file. Details of +[8.13.0]: https://github.com/gchq/CyberChef/releases/tag/v8.13.0 [8.12.0]: https://github.com/gchq/CyberChef/releases/tag/v8.12.0 [8.11.0]: https://github.com/gchq/CyberChef/releases/tag/v8.11.0 [8.10.0]: https://github.com/gchq/CyberChef/releases/tag/v8.10.0 @@ -96,6 +100,7 @@ All major and minor version changes will be documented in this file. Details of [@arnydo]: https://github.com/arnydo [@klaxon1]: https://github.com/klaxon1 [@bwhitn]: https://github.com/bwhitn +[@jarmovanlenthe]: https://github.com/jarmovanlenthe [#95]: https://github.com/gchq/CyberChef/pull/299 [#173]: https://github.com/gchq/CyberChef/pull/173 @@ -119,3 +124,4 @@ All major and minor version changes will be documented in this file. Details of [#387]: https://github.com/gchq/CyberChef/pull/387 [#394]: https://github.com/gchq/CyberChef/pull/394 [#428]: https://github.com/gchq/CyberChef/pull/428 +[#441]: https://github.com/gchq/CyberChef/pull/441 From 79b9b63982ba612bc426f50a54b6137af08831ec Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sat, 15 Dec 2018 00:26:41 +0000 Subject: [PATCH 08/25] 8.13.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3f1f599a..07446032 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.12.4", + "version": "8.13.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 338aced6..48751e21 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.12.4", + "version": "8.13.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 22454ae842769fa819cde142b609d08e9db6be25 Mon Sep 17 00:00:00 2001 From: tcode2k16 Date: Mon, 17 Dec 2018 12:37:00 +0800 Subject: [PATCH 09/25] Add "To Base62" and "From Base62" operations --- src/core/config/Categories.json | 2 + src/core/operations/FromBase62.mjs | 52 ++++++++++++++++++++ src/core/operations/ToBase62.mjs | 54 ++++++++++++++++++++ test/index.mjs | 1 + test/tests/operations/Base62.mjs | 79 ++++++++++++++++++++++++++++++ 5 files changed, 188 insertions(+) create mode 100644 src/core/operations/FromBase62.mjs create mode 100644 src/core/operations/ToBase62.mjs create mode 100644 test/tests/operations/Base62.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index e9fe3399..0a3a5f6e 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -18,6 +18,8 @@ "From Binary", "To Octal", "From Octal", + "To Base62", + "From Base62", "To Base64", "From Base64", "Show Base64 offsets", diff --git a/src/core/operations/FromBase62.mjs b/src/core/operations/FromBase62.mjs new file mode 100644 index 00000000..5d56b3b7 --- /dev/null +++ b/src/core/operations/FromBase62.mjs @@ -0,0 +1,52 @@ +/** + * @author tcode2k16 [tcode2k16@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import BigNumber from "bignumber.js"; +import Utils from "../Utils"; + + +/** + * From Base62 operation + */ +class FromBase62 extends Operation { + + /** + * FromBase62 constructor + */ + constructor() { + super(); + + this.name = "From Base62"; + this.module = "Default"; + this.description = "decode base62 string"; + this.infoURL = "https://en.wikipedia.org/wiki/List_of_numeral_systems"; + this.inputType = "string"; + this.outputType = "string"; + this.args = []; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + if (input.length < 1) return ""; + const ALPHABET = Utils.expandAlphRange("0-9A-Za-z").join(""); + const BN = BigNumber.clone({ ALPHABET }); + + const re = new RegExp("[^" + ALPHABET.replace(/[[\]\\\-^$]/g, "\\$&") + "]", "g"); + input = input.replace(re, ""); + + const number = new BN(input, 62); + + return Utils.byteArrayToUtf8(Utils.convertToByteArray(number.toString(16), "Hex")); + } + +} + +export default FromBase62; diff --git a/src/core/operations/ToBase62.mjs b/src/core/operations/ToBase62.mjs new file mode 100644 index 00000000..a52679ef --- /dev/null +++ b/src/core/operations/ToBase62.mjs @@ -0,0 +1,54 @@ +/** + * @author tcode2k16 [tcode2k16@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import BigNumber from "bignumber.js"; +import Utils from "../Utils"; +import {toHexFast} from "../lib/Hex"; + +/** + * To Base62 operation + */ +class ToBase62 extends Operation { + + /** + * ToBase62 constructor + */ + constructor() { + super(); + + this.name = "To Base62"; + this.module = "Default"; + this.description = "encode string to base62"; + this.infoURL = "https://en.wikipedia.org/wiki/List_of_numeral_systems"; + this.inputType = "string"; + this.outputType = "string"; + this.args = []; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + if (input.length < 1) return ""; + + const ALPHABET = Utils.expandAlphRange("0-9A-Za-z").join(""); + const BN = BigNumber.clone({ ALPHABET }); + + input = Utils.strToByteArray(input); + input = toHexFast(input); + input = input.toUpperCase(); + + const number = new BN(input, 16); + + return number.toString(62); + } + +} + +export default ToBase62; diff --git a/test/index.mjs b/test/index.mjs index 9c11f6ae..9b1caea1 100644 --- a/test/index.mjs +++ b/test/index.mjs @@ -28,6 +28,7 @@ import "./tests/operations/BCD"; import "./tests/operations/BSON"; import "./tests/operations/Base58"; import "./tests/operations/Base64"; +import "./tests/operations/Base62"; import "./tests/operations/BitwiseOp"; import "./tests/operations/ByteRepr"; import "./tests/operations/CartesianProduct"; diff --git a/test/tests/operations/Base62.mjs b/test/tests/operations/Base62.mjs new file mode 100644 index 00000000..e96e4e46 --- /dev/null +++ b/test/tests/operations/Base62.mjs @@ -0,0 +1,79 @@ +/** + * Base62 tests. + * + * @author tcode2k16 [tcode2k16@gmail.com] + * + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import TestRegister from "../../TestRegister"; + +TestRegister.addTests([ + { + name: "To Base62: nothing", + input: "", + expectedOutput: "", + recipeConfig: [ + { + op: "To Base62", + args: [], + }, + ], + }, + { + name: "To Base62: Hello, World!", + input: "Hello, World!", + expectedOutput: "1wJfrzvdbtXUOlUjUf", + recipeConfig: [ + { + op: "To Base62", + args: [], + }, + ], + }, + { + name: "To Base62: UTF-8", + input: "ნუ პანიკას", + expectedOutput: "BPDNbjoGvDCDzHbKT77eWg0vGQrJuWRXltuRVZ", + recipeConfig: [ + { + op: "To Base62", + args: [], + }, + ], + }, + { + name: "From Base62: nothing", + input: "", + expectedOutput: "", + recipeConfig: [ + { + op: "From Base62", + args: [], + }, + ], + }, + { + name: "From Base62: Hello, World!", + input: "1wJfrzvdbtXUOlUjUf", + expectedOutput: "Hello, World!", + recipeConfig: [ + { + op: "From Base62", + args: [], + }, + ], + }, + { + name: "From Base62: UTF-8", + input: "BPDNbjoGvDCDzHbKT77eWg0vGQrJuWRXltuRVZ", + expectedOutput: "ნუ პანიკას", + recipeConfig: [ + { + op: "From Base62", + args: [], + }, + ], + } +]); From dacb3ef6c3da374ef6aa721675a4b4a697d4fea0 Mon Sep 17 00:00:00 2001 From: Cynser <42423063+Cynser@users.noreply.github.com> Date: Mon, 17 Dec 2018 19:39:12 +0000 Subject: [PATCH 10/25] Added decode option --- .../operations/TextEncodingBruteForce.mjs | 27 +++++++++++++++---- .../operations/TextEncodingBruteForce.mjs | 15 +++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/core/operations/TextEncodingBruteForce.mjs b/src/core/operations/TextEncodingBruteForce.mjs index ef2d6500..9c606eaa 100644 --- a/src/core/operations/TextEncodingBruteForce.mjs +++ b/src/core/operations/TextEncodingBruteForce.mjs @@ -26,7 +26,13 @@ class TextEncodingBruteForce extends Operation { this.infoURL = "https://wikipedia.org/wiki/Character_encoding"; this.inputType = "string"; this.outputType = "string"; - this.args = []; + this.args = [ + { + name: "Mode", + type: "option", + value: ["Encode", "Decode"] + } + ]; } /** @@ -36,12 +42,23 @@ class TextEncodingBruteForce extends Operation { */ run(input, args) { const output = [], - charSets = Object.keys(IO_FORMAT); + charSets = Object.keys(IO_FORMAT), + mode = args[0]; for (let i = 0; i < charSets.length; i++) { - let currentEncoding = Utils.printable(charSets[i] + ": ", false); - currentEncoding += cptable.utils.encode(IO_FORMAT[charSets[i]], input); - output.push(currentEncoding); + let currentEncoding = charSets[i] + ": "; + + try { + if (mode === "Decode") { + currentEncoding += cptable.utils.decode(IO_FORMAT[charSets[i]], input); + } else { + currentEncoding += cptable.utils.encode(IO_FORMAT[charSets[i]], input); + } + } catch (err) { + currentEncoding += "Could not decode."; + } + + output.push(Utils.printable(currentEncoding, true)); } return output.join("\n"); diff --git a/test/tests/operations/TextEncodingBruteForce.mjs b/test/tests/operations/TextEncodingBruteForce.mjs index e79b24af..3b16453d 100644 --- a/test/tests/operations/TextEncodingBruteForce.mjs +++ b/test/tests/operations/TextEncodingBruteForce.mjs @@ -10,13 +10,24 @@ import TestRegister from "../../TestRegister"; TestRegister.addTests([ { - name: "Text Encoding Brute Force", + name: "Text Encoding Brute Force - Encode", input: "Булкі праз ляніва сабаку.", expectedMatch: /Windows-1251 Cyrillic \(1251\): Булкі праз ляніва сабаку\./, recipeConfig: [ { op: "Text Encoding Brute Force", - args: [], + args: ["Encode"], + }, + ], + }, + { + name: "Text Encoding Brute Force - Decode", + input: "Áóëê³ ïðàç ëÿí³âà ñàáàêó.", + expectedMatch: /Windows-1251 Cyrillic \(1251\): Булкі праз ляніва сабаку\./, + recipeConfig: [ + { + op: "Text Encoding Brute Force", + args: ["Decode"], }, ], } From 283d7f2159db862f0b17d2840251775f6c762c6b Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 10:40:18 +0000 Subject: [PATCH 11/25] Add Output Filter option to Magic operation --- src/core/lib/Magic.mjs | 9 ++++++--- src/core/operations/Magic.mjs | 9 +++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index b4d5a7b0..52908eb3 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -267,7 +267,7 @@ class Magic { * @param {boolean} [useful=false] - Whether the current recipe should be scored highly * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding */ - async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false) { + async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, filter=null) { if (depth < 0) return []; // Find any operations that can be run on this data @@ -305,7 +305,7 @@ class Magic { const magic = new Magic(output, this.opPatterns), speculativeResults = await magic.speculativeExecution( - depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful); + depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, filter); results = results.concat(speculativeResults); })); @@ -331,7 +331,10 @@ class Magic { r.fileType || // A file was found r.isUTF8 || // UTF-8 was found r.matchingOps.length // A matching op was found - ) + ) && + ( + filter == null || // Either no filter was passed, or + new RegExp(filter).test(r.data)) // the filter matches the result data ); // Return a sorted list of possible recipes along with their properties diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index b3e15e63..bb419bab 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -43,6 +43,11 @@ class Magic extends Operation { "name": "Extensive language support", "type": "boolean", "value": false + }, + { + "name": "Output Filter (Regex)", + "type": "string", + "value": "" } ]; } @@ -56,10 +61,10 @@ class Magic extends Operation { */ async run(state) { const ings = state.opList[state.progress].ingValues, - [depth, intensive, extLang] = ings, + [depth, intensive, extLang, filter] = ings, dish = state.dish, magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)), - options = await magic.speculativeExecution(depth, extLang, intensive); + options = await magic.speculativeExecution(depth, extLang, intensive, [], false, filter); // Record the current state for use when presenting this.state = state; From 367d79e82049ea6200f807f547129de72998b2a7 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 11:55:49 +0000 Subject: [PATCH 12/25] Fix filtering to work on all the data and not just the result snippet --- src/core/lib/Magic.mjs | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 52908eb3..50c632b8 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -276,17 +276,18 @@ class Magic { let results = []; // Record the properties of the current data - results.push({ - recipe: recipeConfig, - data: this.inputStr.slice(0, 100), - languageScores: this.detectLanguage(extLang), - fileType: this.detectFileType(), - isUTF8: this.isUTF8(), - entropy: this.calcEntropy(), - matchingOps: matchingOps, - useful: useful - }); - + if (filter == null || new RegExp(filter).test(this.inputStr)){ + results.push({ + recipe: recipeConfig, + data: this.inputStr.slice(0, 100), + languageScores: this.detectLanguage(extLang), + fileType: this.detectFileType(), + isUTF8: this.isUTF8(), + entropy: this.calcEntropy(), + matchingOps: matchingOps, + useful: useful + }); + } const prevOp = recipeConfig[recipeConfig.length - 1]; // Execute each of the matching operations, then recursively call the speculativeExecution() @@ -331,10 +332,7 @@ class Magic { r.fileType || // A file was found r.isUTF8 || // UTF-8 was found r.matchingOps.length // A matching op was found - ) && - ( - filter == null || // Either no filter was passed, or - new RegExp(filter).test(r.data)) // the filter matches the result data + ) ); // Return a sorted list of possible recipes along with their properties From d89d79116c827ce6dc6ecd36a2b990423c0d9874 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 18 Dec 2018 12:19:42 +0000 Subject: [PATCH 13/25] Cleaned up Base62 ops and updated CHANGELOG --- CHANGELOG.md | 7 +++++++ src/core/config/Categories.json | 4 ++-- src/core/operations/FromBase62.mjs | 22 ++++++++++++++-------- src/core/operations/ToBase62.mjs | 18 +++++++++++------- test/tests/operations/Base62.mjs | 12 ++++++------ 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a65fc145..15943e7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). + +### [8.14.0] - 2018-12-18 +- 'To Base62' and 'From Base62' operations added [@tcode2k16] | [#443] + ### [8.13.0] - 2018-12-15 - 'A1Z26 Cipher Encode' and 'A1Z26 Cipher Decode' operations added [@jarmovanlenthe] | [#441] @@ -69,6 +73,7 @@ All major and minor version changes will be documented in this file. Details of +[8.14.0]: https://github.com/gchq/CyberChef/releases/tag/v8.14.0 [8.13.0]: https://github.com/gchq/CyberChef/releases/tag/v8.13.0 [8.12.0]: https://github.com/gchq/CyberChef/releases/tag/v8.12.0 [8.11.0]: https://github.com/gchq/CyberChef/releases/tag/v8.11.0 @@ -101,6 +106,7 @@ All major and minor version changes will be documented in this file. Details of [@klaxon1]: https://github.com/klaxon1 [@bwhitn]: https://github.com/bwhitn [@jarmovanlenthe]: https://github.com/jarmovanlenthe +[@tcode2k16]: https://github.com/tcode2k16 [#95]: https://github.com/gchq/CyberChef/pull/299 [#173]: https://github.com/gchq/CyberChef/pull/173 @@ -125,3 +131,4 @@ All major and minor version changes will be documented in this file. Details of [#394]: https://github.com/gchq/CyberChef/pull/394 [#428]: https://github.com/gchq/CyberChef/pull/428 [#441]: https://github.com/gchq/CyberChef/pull/441 +[#443]: https://github.com/gchq/CyberChef/pull/443 diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 0a3a5f6e..d4e815ac 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -18,8 +18,6 @@ "From Binary", "To Octal", "From Octal", - "To Base62", - "From Base62", "To Base64", "From Base64", "Show Base64 offsets", @@ -27,6 +25,8 @@ "From Base32", "To Base58", "From Base58", + "To Base62", + "From Base62", "To Base85", "From Base85", "To Base", diff --git a/src/core/operations/FromBase62.mjs b/src/core/operations/FromBase62.mjs index 5d56b3b7..525f2e2f 100644 --- a/src/core/operations/FromBase62.mjs +++ b/src/core/operations/FromBase62.mjs @@ -22,21 +22,27 @@ class FromBase62 extends Operation { this.name = "From Base62"; this.module = "Default"; - this.description = "decode base62 string"; - this.infoURL = "https://en.wikipedia.org/wiki/List_of_numeral_systems"; + this.description = "Base62 is a notation for encoding arbitrary byte data using a restricted set of symbols that can be conveniently used by humans and processed by computers. The high number base results in shorter strings than with the decimal or hexadecimal system."; + this.infoURL = "https://wikipedia.org/wiki/List_of_numeral_systems"; this.inputType = "string"; - this.outputType = "string"; - this.args = []; + this.outputType = "byteArray"; + this.args = [ + { + name: "Alphabet", + type: "string", + value: "0-9A-Za-z" + } + ]; } /** * @param {string} input * @param {Object[]} args - * @returns {string} + * @returns {byteArray} */ run(input, args) { - if (input.length < 1) return ""; - const ALPHABET = Utils.expandAlphRange("0-9A-Za-z").join(""); + if (input.length < 1) return []; + const ALPHABET = Utils.expandAlphRange(args[0]).join(""); const BN = BigNumber.clone({ ALPHABET }); const re = new RegExp("[^" + ALPHABET.replace(/[[\]\\\-^$]/g, "\\$&") + "]", "g"); @@ -44,7 +50,7 @@ class FromBase62 extends Operation { const number = new BN(input, 62); - return Utils.byteArrayToUtf8(Utils.convertToByteArray(number.toString(16), "Hex")); + return Utils.convertToByteArray(number.toString(16), "Hex"); } } diff --git a/src/core/operations/ToBase62.mjs b/src/core/operations/ToBase62.mjs index a52679ef..3f615db2 100644 --- a/src/core/operations/ToBase62.mjs +++ b/src/core/operations/ToBase62.mjs @@ -22,11 +22,17 @@ class ToBase62 extends Operation { this.name = "To Base62"; this.module = "Default"; - this.description = "encode string to base62"; + this.description = "Base62 is a notation for encoding arbitrary byte data using a restricted set of symbols that can be conveniently used by humans and processed by computers. The high number base results in shorter strings than with the decimal or hexadecimal system."; this.infoURL = "https://en.wikipedia.org/wiki/List_of_numeral_systems"; - this.inputType = "string"; + this.inputType = "byteArray"; this.outputType = "string"; - this.args = []; + this.args = [ + { + name: "Alphabet", + type: "string", + value: "0-9A-Za-z" + } + ]; } /** @@ -37,12 +43,10 @@ class ToBase62 extends Operation { run(input, args) { if (input.length < 1) return ""; - const ALPHABET = Utils.expandAlphRange("0-9A-Za-z").join(""); + const ALPHABET = Utils.expandAlphRange(args[0]).join(""); const BN = BigNumber.clone({ ALPHABET }); - input = Utils.strToByteArray(input); - input = toHexFast(input); - input = input.toUpperCase(); + input = toHexFast(input).toUpperCase(); const number = new BN(input, 16); diff --git a/test/tests/operations/Base62.mjs b/test/tests/operations/Base62.mjs index e96e4e46..8bf41b36 100644 --- a/test/tests/operations/Base62.mjs +++ b/test/tests/operations/Base62.mjs @@ -17,7 +17,7 @@ TestRegister.addTests([ recipeConfig: [ { op: "To Base62", - args: [], + args: ["0-9A-Za-z"], }, ], }, @@ -28,7 +28,7 @@ TestRegister.addTests([ recipeConfig: [ { op: "To Base62", - args: [], + args: ["0-9A-Za-z"], }, ], }, @@ -39,7 +39,7 @@ TestRegister.addTests([ recipeConfig: [ { op: "To Base62", - args: [], + args: ["0-9A-Za-z"], }, ], }, @@ -50,7 +50,7 @@ TestRegister.addTests([ recipeConfig: [ { op: "From Base62", - args: [], + args: ["0-9A-Za-z"], }, ], }, @@ -61,7 +61,7 @@ TestRegister.addTests([ recipeConfig: [ { op: "From Base62", - args: [], + args: ["0-9A-Za-z"], }, ], }, @@ -72,7 +72,7 @@ TestRegister.addTests([ recipeConfig: [ { op: "From Base62", - args: [], + args: ["0-9A-Za-z"], }, ], } From 56f830240204db8a70990ec45445a33829e90901 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 18 Dec 2018 12:20:03 +0000 Subject: [PATCH 14/25] 8.14.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 07446032..eb66cd91 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.13.0", + "version": "8.14.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 48751e21..9565b4e0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.13.0", + "version": "8.14.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 076a1f97c29ab3857b5801bf1d1c0991a432dfc3 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 18 Dec 2018 13:50:10 +0000 Subject: [PATCH 15/25] Tidied up 'Text Encoding Brute Force' operations and updated CHANGELOG --- CHANGELOG.md | 6 +++ babel.config.js | 2 +- .../operations/TextEncodingBruteForce.mjs | 49 ++++++++++++++----- src/core/operations/ToBase62.mjs | 2 +- .../operations/TextEncodingBruteForce.mjs | 4 +- 5 files changed, 46 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15943e7e..6355831e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.15.0] - 2018-12-18 +- 'Text Encoding Brute Force' operation added [@Cynser] | [#439] + ### [8.14.0] - 2018-12-18 - 'To Base62' and 'From Base62' operations added [@tcode2k16] | [#443] @@ -73,6 +76,7 @@ All major and minor version changes will be documented in this file. Details of +[8.15.0]: https://github.com/gchq/CyberChef/releases/tag/v8.15.0 [8.14.0]: https://github.com/gchq/CyberChef/releases/tag/v8.14.0 [8.13.0]: https://github.com/gchq/CyberChef/releases/tag/v8.13.0 [8.12.0]: https://github.com/gchq/CyberChef/releases/tag/v8.12.0 @@ -107,6 +111,7 @@ All major and minor version changes will be documented in this file. Details of [@bwhitn]: https://github.com/bwhitn [@jarmovanlenthe]: https://github.com/jarmovanlenthe [@tcode2k16]: https://github.com/tcode2k16 +[@Cynser]: https://github.com/Cynser [#95]: https://github.com/gchq/CyberChef/pull/299 [#173]: https://github.com/gchq/CyberChef/pull/173 @@ -130,5 +135,6 @@ All major and minor version changes will be documented in this file. Details of [#387]: https://github.com/gchq/CyberChef/pull/387 [#394]: https://github.com/gchq/CyberChef/pull/394 [#428]: https://github.com/gchq/CyberChef/pull/428 +[#439]: https://github.com/gchq/CyberChef/pull/439 [#441]: https://github.com/gchq/CyberChef/pull/441 [#443]: https://github.com/gchq/CyberChef/pull/443 diff --git a/babel.config.js b/babel.config.js index 2362c42a..5459f6c8 100644 --- a/babel.config.js +++ b/babel.config.js @@ -1,7 +1,7 @@ module.exports = function(api) { api.cache.forever(); - return { + return { "presets": [ ["@babel/preset-env", { "targets": { diff --git a/src/core/operations/TextEncodingBruteForce.mjs b/src/core/operations/TextEncodingBruteForce.mjs index 9c606eaa..3919dcd9 100644 --- a/src/core/operations/TextEncodingBruteForce.mjs +++ b/src/core/operations/TextEncodingBruteForce.mjs @@ -1,5 +1,6 @@ /** * @author Cynser + * @author n1474335 [n1474335@gmail.com] * @copyright Crown Copyright 2018 * @license Apache-2.0 */ @@ -22,10 +23,18 @@ class TextEncodingBruteForce extends Operation { this.name = "Text Encoding Brute Force"; this.module = "CharEnc"; - this.description = "Enumerate all possible text encodings for input."; + this.description = [ + "Enumerates all supported text encodings for the input, allowing you to quickly spot the correct one.", + "

", + "Supported charsets are:", + "
    ", + Object.keys(IO_FORMAT).map(e => `
  • ${e}
    • `).join("\n"), + "
" + ].join("\n"); this.infoURL = "https://wikipedia.org/wiki/Character_encoding"; this.inputType = "string"; - this.outputType = "string"; + this.outputType = "json"; + this.presentType = "html"; this.args = [ { name: "Mode", @@ -38,30 +47,44 @@ class TextEncodingBruteForce extends Operation { /** * @param {string} input * @param {Object[]} args - * @returns {string} + * @returns {json} */ run(input, args) { - const output = [], - charSets = Object.keys(IO_FORMAT), + const output = {}, + charsets = Object.keys(IO_FORMAT), mode = args[0]; - for (let i = 0; i < charSets.length; i++) { - let currentEncoding = charSets[i] + ": "; - + charsets.forEach(charset => { try { if (mode === "Decode") { - currentEncoding += cptable.utils.decode(IO_FORMAT[charSets[i]], input); + output[charset] = cptable.utils.decode(IO_FORMAT[charset], input); } else { - currentEncoding += cptable.utils.encode(IO_FORMAT[charSets[i]], input); + output[charset] = Utils.arrayBufferToStr(cptable.utils.encode(IO_FORMAT[charset], input)); } } catch (err) { - currentEncoding += "Could not decode."; + output[charset] = "Could not decode."; } + }); - output.push(Utils.printable(currentEncoding, true)); + return output; + } + + /** + * Displays the encodings in an HTML table for web apps. + * + * @param {Object[]} encodings + * @returns {html} + */ + present(encodings) { + let table = ""; + + for (const enc in encodings) { + const value = Utils.printable(encodings[enc], true); + table += ``; } - return output.join("\n"); + table += "
EncodingValue
${enc}${value}
"; + return table; } } diff --git a/src/core/operations/ToBase62.mjs b/src/core/operations/ToBase62.mjs index 3f615db2..51f89ecd 100644 --- a/src/core/operations/ToBase62.mjs +++ b/src/core/operations/ToBase62.mjs @@ -23,7 +23,7 @@ class ToBase62 extends Operation { this.name = "To Base62"; this.module = "Default"; this.description = "Base62 is a notation for encoding arbitrary byte data using a restricted set of symbols that can be conveniently used by humans and processed by computers. The high number base results in shorter strings than with the decimal or hexadecimal system."; - this.infoURL = "https://en.wikipedia.org/wiki/List_of_numeral_systems"; + this.infoURL = "https://wikipedia.org/wiki/List_of_numeral_systems"; this.inputType = "byteArray"; this.outputType = "string"; this.args = [ diff --git a/test/tests/operations/TextEncodingBruteForce.mjs b/test/tests/operations/TextEncodingBruteForce.mjs index 3b16453d..22e8f7c5 100644 --- a/test/tests/operations/TextEncodingBruteForce.mjs +++ b/test/tests/operations/TextEncodingBruteForce.mjs @@ -12,7 +12,7 @@ TestRegister.addTests([ { name: "Text Encoding Brute Force - Encode", input: "Булкі праз ляніва сабаку.", - expectedMatch: /Windows-1251 Cyrillic \(1251\): Булкі праз ляніва сабаку\./, + expectedMatch: /Windows-1251 Cyrillic \(1251\).{1,10}Булкі праз ляніва сабаку\./, recipeConfig: [ { op: "Text Encoding Brute Force", @@ -23,7 +23,7 @@ TestRegister.addTests([ { name: "Text Encoding Brute Force - Decode", input: "Áóëê³ ïðàç ëÿí³âà ñàáàêó.", - expectedMatch: /Windows-1251 Cyrillic \(1251\): Булкі праз ляніва сабаку\./, + expectedMatch: /Windows-1251 Cyrillic \(1251\).{1,10}Булкі праз ляніва сабаку\./, recipeConfig: [ { op: "Text Encoding Brute Force", From 8ab56a29ac9473a8a68ad24d6c0ff33669170e25 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 18 Dec 2018 13:50:30 +0000 Subject: [PATCH 16/25] 8.15.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index eb66cd91..5e20c8bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.14.0", + "version": "8.15.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 9565b4e0..3505cfa2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.14.0", + "version": "8.15.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From c8eab5d2185cf89fbfd0b7b2065dc842de15905f Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 14:06:39 +0000 Subject: [PATCH 17/25] Commenting --- src/core/lib/Magic.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 50c632b8..eeda155a 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -276,6 +276,8 @@ class Magic { let results = []; // Record the properties of the current data + // Only if there either wasn't a filter provided, + // or the filter matches in the data if (filter == null || new RegExp(filter).test(this.inputStr)){ results.push({ recipe: recipeConfig, From b6bdcaa71f8f5b689c7a76e6840c351d9a983c6a Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 14:19:52 +0000 Subject: [PATCH 18/25] Rename output filter argument to Crib --- src/core/operations/Magic.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index bb419bab..4575acf6 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -45,7 +45,7 @@ class Magic extends Operation { "value": false }, { - "name": "Output Filter (Regex)", + "name": "Crib (known plaintext string or regex)", "type": "string", "value": "" } From 5d4c7244e197956e967dd5ae9626309ccf9cef2c Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 18 Dec 2018 16:52:09 +0000 Subject: [PATCH 19/25] Add note about crib to operation description --- src/core/operations/Magic.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index 4575acf6..8bfdb067 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -23,7 +23,7 @@ class Magic extends Operation { this.name = "Magic"; this.flowControl = true; this.module = "Default"; - this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar."; + this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.

Optionally enter a regular expression to match a string you expect to find to filter results (crib)"; this.infoURL = "https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic"; this.inputType = "ArrayBuffer"; this.outputType = "JSON"; From f600571c6d743e573be2b7966f01f67bcf6b63a1 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 19 Dec 2018 09:42:09 +0000 Subject: [PATCH 20/25] Fix to make the filter work when intensive mode was turned on. --- src/core/lib/Magic.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index eeda155a..640979cf 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -265,6 +265,7 @@ class Magic { * performance) * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point * @param {boolean} [useful=false] - Whether the current recipe should be scored highly + * @param {string} [filter=null] - The regex crib provided by the user, to filter the operation output * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding */ async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, filter=null) { @@ -320,7 +321,7 @@ class Magic { await Promise.all(bfEncodings.map(async enc => { const magic = new Magic(enc.data, this.opPatterns), bfResults = await magic.speculativeExecution( - depth-1, extLang, false, [...recipeConfig, enc.conf]); + depth-1, extLang, false, [...recipeConfig, enc.conf], useful, filter); results = results.concat(bfResults); })); From 866dd546c8bddc20a6678b0cd87d52f99714cda9 Mon Sep 17 00:00:00 2001 From: Anthony Arnold Date: Thu, 20 Dec 2018 00:05:10 +1000 Subject: [PATCH 21/25] Add the Play Media operation and place it under a new "Multimedia" category. Move the Render Image to this new category as well. --- src/core/config/Categories.json | 8 ++- src/core/operations/PlayMedia.mjs | 101 ++++++++++++++++++++++++++++++ test/index.mjs | 1 + test/tests/operations/Media.mjs | 43 +++++++++++++ 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 src/core/operations/PlayMedia.mjs create mode 100644 test/tests/operations/Media.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 3998d7db..690fea57 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -341,6 +341,13 @@ "From MessagePack" ] }, + { + "name": "Multimedia", + "ops": [ + "Render Image", + "Play Media" + ] + }, { "name": "Other", "ops": [ @@ -355,7 +362,6 @@ "Generate TOTP", "Generate HOTP", "Haversine distance", - "Render Image", "Remove EXIF", "Extract EXIF", "Numberwang", diff --git a/src/core/operations/PlayMedia.mjs b/src/core/operations/PlayMedia.mjs new file mode 100644 index 00000000..e6ac53e4 --- /dev/null +++ b/src/core/operations/PlayMedia.mjs @@ -0,0 +1,101 @@ +/** + * @author anthony-arnold [anthony.arnold@uqconnect.edu.au] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import { fromBase64, toBase64 } from "../lib/Base64"; +import { fromHex } from "../lib/Hex"; +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Utils from "../Utils"; +import Magic from "../lib/Magic"; + +/** + * PlayMedia operation + */ +class PlayMedia extends Operation { + + /** + * PlayMedia constructor + */ + constructor() { + super(); + + this.name = "Play Media"; + this.module = "Media"; + this.description = "Plays the input as sound or video depending on the type."; + this.infoURL = ""; + this.inputType = "string"; + this.outputType = "byteArray"; + this.presentType = "html"; + this.args = [ + { + "name": "Input format", + "type": "option", + "value": ["Raw", "Base64", "Hex"] + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {byteArray} The multimedia data as bytes. + */ + run(input, args) { + const inputFormat = args[0]; + + if (!input.length) return []; + + // Convert input to raw bytes + switch (inputFormat) { + case "Hex": + input = fromHex(input); + break; + case "Base64": + // Don't trust the Base64 entered by the user. + // Unwrap it first, then re-encode later. + input = fromBase64(input, undefined, "byteArray"); + break; + case "Raw": + default: + input = Utils.strToByteArray(input); + break; + } + + + // Determine file type + const type = Magic.magicFileType(input); + if (!(type && /^audio|video/.test(type.mime))) { + throw new OperationError("Invalid file type"); + } + + return input; + } + + /** + * Displays an audio or video element that may be able to play the media + * file. + * @param data {byteArray} Data containing an audio or video file. + * @returns {string} Markup to display a media player. + */ + async present(data) { + if (!data.length) return ""; + + const type = Magic.magicFileType(data); + const matches = /^audio|video/.exec(type.mime); + if (!matches) { + throw new OperationError("Invalid file type"); + } + const dataURI = `data:${type.mime};base64,${toBase64(data)}`; + const element = matches[0]; + + let html = `<${element} src='${dataURI}' type='${type.mime}' controls>`; + html += "

Unsupported media type.

"; + html += ``; + return html; + } +} + +export default PlayMedia; diff --git a/test/index.mjs b/test/index.mjs index a08d9b32..c944b765 100644 --- a/test/index.mjs +++ b/test/index.mjs @@ -80,6 +80,7 @@ import "./tests/operations/ToGeohash.mjs"; import "./tests/operations/TranslateDateTimeFormat"; import "./tests/operations/Magic"; import "./tests/operations/ParseTLV"; +import "./tests/operations/Media"; let allTestsPassing = true; const testStatusCounts = { diff --git a/test/tests/operations/Media.mjs b/test/tests/operations/Media.mjs new file mode 100644 index 00000000..4d285e8d --- /dev/null +++ b/test/tests/operations/Media.mjs @@ -0,0 +1,43 @@ +/** + * Media operation tests. + * @author anthony-arnold [anthony.arnold@uqconnect.edu.au] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ +import TestRegister from "../../TestRegister"; + +TestRegister.addTests([ + { + name: "Play Media: nothing", + input: "", + expectedOutput: "", + recipeConfig: [ + { op: "Play Media", args: ["Raw"] } + ] + }, + { + name: "Play Media: raw wav", + input: "52494646bcaf010057415645666d74201000000001000100401f0000401f0000010008006461746198af0100818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081808180818081818281807f82807f817e81808280827d8086817d80828184817f80807d847f7e7e8582837b81857e7e82867f7c7f857e7d838182837f83847b7c838082807c7d838481827e83827c7f848383807c7f7c80848085808180827e83827f7c82827c7f858183807c837b847f7f82827a85837a8287817f7882827f89837e7f877b7e808281817f7e807c827b7e7d8383838486838883837d7d7682797d847f7d868287867d83847c81807d7e818080828082828380807e7f827d8181807e828082858581808081807f7a7f81808183807d7c7f7f858180848484857f8082817e827b7e7f7f8385807f7f7d858183808081807f7f8183847f7e7d81828284857d7f7e8084868182837d7e7d7a7e7e8085837f80807b7f7b7a80827d7f817e8386838588868386868689868586838386868584807c7a78787875706e6c6a6c6d70757a7c7e7f80858b94a1a29a9996989a9e9ca09ea19caa8e554129243e4d475e7184a0ada2a4a0918179646167696f7b7a8390939ca4aab2af9c9691909aa2a3ae74321e162a4d5c506c88a5babca6a4a7a18c7a5e5c646f6a6b6d7f929a8f858182827e79849299a1abb2c7cc975b3826324447496788a5b8b6a9aaa9977b60494c586467707c949fa195938f8f847a717c88a0aab7b7d6a46045362f3c483d6089a1acb4a3a4a89570615652676f6d788f97a29c8e8992898381808393999da9bf8e515e504c4e5d4879929a90a1909e9c855c5b5f717679768b9b9f958c838c8d8c7f8782969ca797a963585f63425d60687e93818b99938c84786f7e7a7b7f86838a867f8486858899939697a7a6be6e465f6c4d465d678e948275958a867063688e8b828fa2a0988b7a7f7c737185898a8e90989c9b8c8c6e5f5a655159646f727f737382857c7f898a9aa4b1b6cac5d4957d695b403c4152", + expectedOutput: "", + recipeConfig: [ + { op: "From Hex", args: ["Space"] }, + { op: "Play Media", args: ["Raw"] } + ] + }, + { + name: "Play Media: hex ogg", + input: "4f676753000200000000000000003129000000000000642493e3011e01766f72626973000000000244ac0000000000008138010000000000b8014f676753000000000000000000003129000001000000a3565ae9102dffffffffffffffffffffffffffff2403766f726269731d000000586970682e4f7267206c6962566f726269732049203230303230373137000000000105766f726269732242435601000001009c739a318799629452892194de3968196394526929a55a4aa9a183166babbdf7de7befbdf7de7bef1d739431469552524aa99d739631471563524a89a5945642682184d662abbdf7de6befb5f6de7bef99424c29a41442084a281d538c29a494424a4a0825640e3ac61c538c52093dd65e6bccbdb6d87beda163ce39e61c534c4a6821740e3ae69c534c4a68a984524206a153d05289adf7de62ebb9a5da7bef81d0905500000100c040101ab20a00500000108aa1188a028486ac020032000004e0288ee3388ee23892623916101ab20a00000200100000c0900c4bb114cdd1244dd22ccf134dd3377dd3366d55d7755dd7755dd77520346415000001004040a719a61a20c28c6416080d590500200000004420c3140342435601000001000052243949a2e4a494520e836431492ae5a494521ec5e4514d3206a594524a29a594524a29a594520a8364394a2ae5a4945212a364314aaad4a494521ee5e4", + expectedOutput: "", + recipeConfig: [ + { op: "Play Media", args: ["Hex"] } + ] + }, + { + name: "Play Media: base64 webm", + input: "GkXfo6NChoEBQveBAULygQRC84EIQoKEd2VibeyCAABCh4EBQoWBARhTgGcQIQmHEU2bdLtNu4tTq4QVSalmU6yBQE27i1OrhBZUrmtTrIGsTbuNU6uEEU2bdFOsgyEJc027jFOrhBxTu2tTrIINQRVJqWbnc6SQRsadRaGFqSlNPQovdQBWvSrXsYMPQkBEiYRG/cAARGGIBBu7mlIesABNgKVodHRwOi8vc291cmNlZm9yZ2UubmV0L3Byb2plY3RzL3lhbWthV0GQU29yZW5zb24gU3F1ZWV6ZRZUrmtMj66414EBc8WHiBmgyaYxwoOBASPjg4QCYloAIzFPhD+AAACGhVZfVlA4JYaIg1ZQOOCIsIICgLqCAWiuTFLXgQJzxYgBiP65XI76uoOBAiMxT4Q/gAAAhohBX1ZPUkJJU2OiTBkCHjoBdm9yYmlzAAAAAAFErAAA/////wD6AAD/////uAEDdm9yYmlzKgAAAFhpcGguT3JnIGxpYlZvcmJpcyBJIDIwMTAwMzI1IChFdmVyeXdoZXJlKQAAAAABBXZvcmJpcx9CQ1YBAAABABhjVClGmVLSSokZc5QxRplikkqJpYQWQkidcxRTqTnXnGusubUghBAaU1ApBZlSjlJpGWOQKQWZUhBLSSV0EjonnWMQW0nB1phri0G2HIQNmlJMKcSUUopCCBlTjCnFlFJKQgcldA465hxTjkooQbicc6u1lpZji6l0kkrnJGRMQkgphZJKB6VTTkJINZbWUikdc1JSakHoIIQQQrYghA2C0JBVAAABAMBAEBqyCgBQAAAQiqEYigKEhqwCADIAAASgKI7iKI4jOZJjSRYQGrIKAAACABAAAMBwFEmRFMmxJEvSLEvTRFFVfdU2VVX2dV3XdV3XdSA0ZBUAAAEAQEinmaUaIMIMZBgIDVkFACAAAABGKMIQA0JDVgEAAAEAAGIoOYgmtOZ8c46DZjloKsXmdHAi1eZJbirm5pxzzjknm3PGOOecc4pyZjFoJrTmnHMSg2YpaCa05pxznsTmQWuqtOacc8Y5p4NxRhjnnHOatOZBajbW5pxzFrSmOWouxeaccyLl5kltLtXmnHPOOeecc84555xzqhenc3BOOOecc6L25lpuQhfnnHM+Gad7c0I455xzzjnnnHPOOeecc4LQkFUAABAAAEEYNoZxpyBIn6OBGEWIacikB92jwyRoDHIKqUejo5FS6iCUVMZJKZ0gNGQVAAAIAAAhhBRSSCGFFFJIIYUUUoghhhhiyCmnnIIKKqmkoooyyiyzzDLLLLPMMuuws8467DDEEEMMrbQSS0211VhjrbnnnGsO0lpprbXWSimllFJKKQgNWQUAgAAAEAgZZJBBRiGFFFKIIaaccsopqKACQkNWAQCAAAACAAAAPMlzREd0REd0REd0REd0RMdzPEeUREmUREm0TMvUTE8VVdWVXVvWZd32bWEXdt33dd/3dePXhWFZlmVZlmVZlmVZlmVZlmVZgtCQVQAACAAAgBBCCCGFFFJIIaUYY8wx56CTUEIgNGQVAAAIACAAAADAURzFcSRHciTJkixJkzRLszzN0zxN9ERRFE3TVEVXdEXdtEXZlE3XdE3ZdFVZtV1Ztm3Z1m1flm3f933f933f933f933f93UdCA1ZBQBIAADoSI6kSIqkSI7jOJIkAaEhqwAAGQAAAQAoiqM4juNIkiRJlqRJnuVZomZqpmd6qqgCoSGrAABAAAABAAAAAAAomuIppuIpouI5oiNKomVaoqZqriibsuu6ruu6ruu6ruu6ruu6ruu6ruu6ruu6ruu6ruu6ruu6ruu6LhAasgoAkAAA0JEcyZEcSZEUSZEcyQFCQ1YBADIAAAIAcAzHkBTJsSxL0zzN0zxN9ERP9ExPFV3RBUJDVgEAgAAAAgAAAAAAMCTDUixHczRJlFRLtVRNtVRLFVVPVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVNU3TNE0gNGQlAAAEAMBijcHlICElJeXeEMIQk54xJiG1XiEEkZLeMQYVg54yogxy3kLjEIMeCA1ZEQBEAQAAxiDHEHPIOUepkxI556h0lBrnHKWOUmcpxZhizSiV2FKsjXOOUketo5RiLC12lFKNqcYCAAACHAAAAiyEQkNWBABRAACEMUgppBRijDmnnEOMKeeYc4Yx5hxzjjnnoHRSKuecdE5KxBhzjjmnnHNSOieVc05KJ6EAAIAABwCAAAuh0JAVAUCcAIBBkjxP8jRRlDRPFEVTdF1RNF3X8jzV9ExTVT3RVFVTVW3ZVFVZljzPND3TVFXPNFXVVFVZNlVVlkVV1W3TdXXbdFXdlm3b911bFnZRVW3dVF3bN1XX9l3Z9n1Z1nVj8jxV9UzTdT3TdGXVdW1bdV1d90xTlk3XlWXTdW3blWVdd2XZ9zXTdF3TVWXZdF3ZdmVXt11Z9n3TdYXflWVfV2VZGHZd94Vb15XldF3dV2VXN1ZZ9n1b14Xh1nVhmTxPVT3TdF3PNF1XdV1fV13X1jXTlGXTdW3ZVF1ZdmXZ911X1nXPNGXZdF3bNl1Xll1Z9n1XlnXddF1fV2VZ+FVX9nVZ15Xh1m3hN13X91VZ9oVXlnXh1nVhuXVdGD5V9X1TdoXhdGXf14XfWW5dOJbRdX1hlW3hWGVZOX7hWJbd95VldF1fWG3ZGFZZFoZf+J3l9n3jeHVdGW7d58y67wzH76T7ytPVbWOZfd1ZZl93juEYOr/w46mqr5uuKwynLAu/7evGs/u+soyu6/uqLAu/KtvCseu+8/y+sCyj7PrCasvCsNq2Mdy+biy/cBzLa+vKMeu+UbZ1fF94CsPzdHVdeWZdx/Z1dONHOH7KAACAAQcAgAATykChISsCgDgBAI8kiaJkWaIoWZYoiqbouqJouq6kaaapaZ5pWppnmqZpqrIpmq4saZppWp5mmpqnmaZomq5rmqasiqYpy6ZqyrJpmrLsurJtu65s26JpyrJpmrJsmqYsu7Kr267s6rqkWaapeZ5pap5nmqZqyrJpmq6reZ5qep5oqp4oqqpqqqqtqqosW55nmproqaYniqpqqqatmqoqy6aq2rJpqrZsqqptu6rs+rJt67ppqrJtqqYtm6pq267s6rIs27ovaZppap5nmprnmaZpmrJsmqorW56nmp4oqqrmiaZqqqosm6aqypbnmaoniqrqiZ5rmqoqy6Zq2qppmrZsqqotm6Yqy65t+77ryrJuqqpsm6pq66ZqyrJsy77vyqruiqYpy6aq2rJpqrIt27Lvy7Ks+6JpyrJpqrJtqqouy7JtG7Ns+7pomrJtqqYtm6oq27It+7os27rvyq5vq6qs67It+7ru+q5w67owvLJs+6qs+ror27pv6zLb9n1E05RlUzVt21RVWXZl2fZl2/Z90TRtW1VVWzZN1bZlWfZ9WbZtYTRN2TZVVdZN1bRtWZZtYbZl4XZl2bdlW/Z115V1X9d949dl3ea6su3Lsq37qqv6tu77wnDrrvAKAAAYcAAACDChDBQashIAiAIAAIxhjDEIjVLOOQehUco55yBkzkEIIZXMOQghlJI5B6GUlDLnIJSSUgihlJRaCyGUlFJrBQAAFDgAAATYoCmxOEChISsBgFQAAIPjWJbnmaJq2rJjSZ4niqqpqrbtSJbniaJpqqptW54niqapqq7r65rniaJpqqrr6rpomqapqq7ruroumqKpqqrrurKum6aqqq4ru7Ls66aqqqrryq4s+8Kquq4ry7Jt68Kwqq7ryrJs27Zv3Lqu677v+8KRreu6LvzCMQxHAQDgCQ4AQAU2rI5wUjQWWGjISgAgAwCAMAYhgxBCBiGEkFJKIaWUEgAAMOAAABBgQhkoNGQlABADAAAQASGDEEIIIYQQQgghhBBCCCGEEELnnHPOOeecc84JANiPcACQejAxMYWFhqwEAFIBAABjlFKKMecgRIw5xhh0EkqKGHOOMQelpFQ5ByGEVFrLrXIOQggptVRb5pyU1mKMOcbMOSkpxVZzzqGU1GKsueaaOymt1ZprzbmW1mrNNedccy6txZprzjXn3HLMNeecc845xpxzzjnnnHMBADgNDgCgBzasjnBSNBZYaMhKACAVAIBARinGnHMOOoQUY845ByGESCHGnHMOQggVY845Bx2EECrGHHMOQgghZM45ByGEEELInIMOOgghhNBBByGEEEIopXMQQgghhBJKCCGEEEIIIYQOQgghhBBCCCGEEEIIoZQSQgghhFBCKCUUAABY4AAAEGDD6ggnRWOBhYasBACAAAAghyWolDNhkGPQY0OQctRMgxBTTnSmmJPaTMUUZA5EJ51EhlpQtpfMAgAAIAgACDABBAYICr4QAmIMAEAQIjNEQmEVLDAogwaHeQDwABEhEQAkJijSLi6gywAXdHHXgRCCEIQgFgdQQAIOTrjhiTc84QYn6BSVOggAAAAAAAMAeAAAOCiAiIjmKiwuMDI0Njg6PAIAAAAAAAYAPgAAjg8gIqK5CosLjAyNDY4OjwAAAAAAAAAAACAgIAAAAAAAEAAAACAgJYaIhlZvcmJpc+GGtYRHLEQAHFO7a0IAu4yzgQC3h/eBAfGCD0e7kbOCAli3i/eBAfGCD0dTeIEqu5Gzgg3At4v3gQHxgg9HU3iB8buSs4Ib0LeM94EB8YIPR1N4ggHmu5KzgiN4t4z3gQHxgg9HU3iCAnW7krOCK8C3jPeBAfGCD0dTeIIDBruSs4IzkLeM94EB8YIPR1N4ggOXu5KzgjY4t4z3gQHxgg9HU3iCA8W7krOCOpi3jPeBAfGCD0dTeIIEIbuSs4I+gLeM94EB8YIPR1N4ggRlu5KzgkDYt4z3gQHxgg9HU3iCBI67krOCSNC3jPeBAfGCD0dTeIIFIbuSs4JJwLeM94EB8YIPR1N4ggU2u5Kzgk3Qt4z3gQHxgg9HU3iCBYO7krOCUZC3jPeBAfGCD0dTeIIFxLuSs4JR4LeM94EB8YIPR1N4ggXKu5KzglXwt4z3gQHxgg9HU3iCBhe7krOCWYi3jPeBAfGCD0dTeIIGXLuSs4JhWLeM94EB8YIPR1N4ggblu5KzgmWQt4z3gQHxgg9HU3iCBy67krOCaDi3jPeBAfGCD0dTeIIHXLuSs4JosLeM94EB8YIPR1N4ggdku5KzgnUIt4z3gQHxgg9HU3iCCEK7krOCddC3jPeBAfGCD0dTeIIIUruSs4J2ILeM94EB8YIPR1N4gghdu5Kzgn5At4z3gQHxgg9HU3iCCPgfQ7Z1ECD6JOeBAKeCD0ejQbOBAACAEjQAnQEqgAJoATkPAEEcIhYWIhYSIAYAABhYE9d0hkrLkkLy9LukMlZckheXpd0hkrLkkLy9LukMlZckheXpd0hkrLkkLy9LukMlZckheXpd0hkrLkkLy9LukMlZckheXpd0hkrLkkLy9LukMlZckheXpQ==", + expectedOutput: "", + recipeConfig: [ + { op: "Play Media", args: ["Base64"] } + ] + } +]); From ed2c886359669ee090c5e795668510c9f8a19f14 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 19 Dec 2018 17:24:38 +0000 Subject: [PATCH 22/25] Magic crib now only applies after all branches have been explored. --- src/core/lib/Magic.mjs | 33 +++++++++++++++------------------ src/core/operations/Magic.mjs | 12 +++++++++--- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index 640979cf..00132d5d 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -265,10 +265,10 @@ class Magic { * performance) * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point * @param {boolean} [useful=false] - Whether the current recipe should be scored highly - * @param {string} [filter=null] - The regex crib provided by the user, to filter the operation output + * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation output * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding */ - async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, filter=null) { + async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, crib=null) { if (depth < 0) return []; // Find any operations that can be run on this data @@ -277,20 +277,17 @@ class Magic { let results = []; // Record the properties of the current data - // Only if there either wasn't a filter provided, - // or the filter matches in the data - if (filter == null || new RegExp(filter).test(this.inputStr)){ - results.push({ - recipe: recipeConfig, - data: this.inputStr.slice(0, 100), - languageScores: this.detectLanguage(extLang), - fileType: this.detectFileType(), - isUTF8: this.isUTF8(), - entropy: this.calcEntropy(), - matchingOps: matchingOps, - useful: useful - }); - } + results.push({ + recipe: recipeConfig, + data: this.inputStr.slice(0, 100), + languageScores: this.detectLanguage(extLang), + fileType: this.detectFileType(), + isUTF8: this.isUTF8(), + entropy: this.calcEntropy(), + matchingOps: matchingOps, + useful: useful, + matchesCrib: crib && crib.test(this.inputStr) + }); const prevOp = recipeConfig[recipeConfig.length - 1]; // Execute each of the matching operations, then recursively call the speculativeExecution() @@ -309,7 +306,7 @@ class Magic { const magic = new Magic(output, this.opPatterns), speculativeResults = await magic.speculativeExecution( - depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, filter); + depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib); results = results.concat(speculativeResults); })); @@ -321,7 +318,7 @@ class Magic { await Promise.all(bfEncodings.map(async enc => { const magic = new Magic(enc.data, this.opPatterns), bfResults = await magic.speculativeExecution( - depth-1, extLang, false, [...recipeConfig, enc.conf], useful, filter); + depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib); results = results.concat(bfResults); })); diff --git a/src/core/operations/Magic.mjs b/src/core/operations/Magic.mjs index 8bfdb067..1555ebad 100644 --- a/src/core/operations/Magic.mjs +++ b/src/core/operations/Magic.mjs @@ -23,7 +23,7 @@ class Magic extends Operation { this.name = "Magic"; this.flowControl = true; this.module = "Default"; - this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.

Optionally enter a regular expression to match a string you expect to find to filter results (crib)"; + this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.

Options
Depth: If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.

Intensive mode: When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.

Extensive language support: At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.

Optionally enter a regular expression to match a string you expect to find to filter results (crib)."; this.infoURL = "https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic"; this.inputType = "ArrayBuffer"; this.outputType = "JSON"; @@ -61,10 +61,16 @@ class Magic extends Operation { */ async run(state) { const ings = state.opList[state.progress].ingValues, - [depth, intensive, extLang, filter] = ings, + [depth, intensive, extLang, crib] = ings, dish = state.dish, magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)), - options = await magic.speculativeExecution(depth, extLang, intensive, [], false, filter); + cribRegex = (crib && crib.length) ? new RegExp(crib, "i") : null; + let options = await magic.speculativeExecution(depth, extLang, intensive, [], false, cribRegex); + + // Filter down to results which matched the crib + if (cribRegex) { + options = options.filter(option => option.matchesCrib); + } // Record the current state for use when presenting this.state = state; From f367c1f78bbcecd9f841efa31622627758bbe041 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 19 Dec 2018 17:25:05 +0000 Subject: [PATCH 23/25] 8.15.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5e20c8bc..8ea5bd80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.15.0", + "version": "8.15.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 3505cfa2..e5cd12e9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.15.0", + "version": "8.15.1", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 01c4cfdc8d6fbeb9589e820aa2414f5736a9eeb2 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 19 Dec 2018 17:58:38 +0000 Subject: [PATCH 24/25] Tidied up 'Play Media' operation --- CHANGELOG.md | 6 ++++++ src/core/config/Categories.json | 10 +++++----- src/core/operations/PlayMedia.mjs | 9 +++++---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6355831e..751ac287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.16.0] - 2018-12-19 +- 'Play Media' operation added [@anthony-arnold] | [#446] + ### [8.15.0] - 2018-12-18 - 'Text Encoding Brute Force' operation added [@Cynser] | [#439] @@ -76,6 +79,7 @@ All major and minor version changes will be documented in this file. Details of +[8.16.0]: https://github.com/gchq/CyberChef/releases/tag/v8.16.0 [8.15.0]: https://github.com/gchq/CyberChef/releases/tag/v8.15.0 [8.14.0]: https://github.com/gchq/CyberChef/releases/tag/v8.14.0 [8.13.0]: https://github.com/gchq/CyberChef/releases/tag/v8.13.0 @@ -112,6 +116,7 @@ All major and minor version changes will be documented in this file. Details of [@jarmovanlenthe]: https://github.com/jarmovanlenthe [@tcode2k16]: https://github.com/tcode2k16 [@Cynser]: https://github.com/Cynser +[@anthony-arnold]: https://github.com/anthony-arnold [#95]: https://github.com/gchq/CyberChef/pull/299 [#173]: https://github.com/gchq/CyberChef/pull/173 @@ -138,3 +143,4 @@ All major and minor version changes will be documented in this file. Details of [#439]: https://github.com/gchq/CyberChef/pull/439 [#441]: https://github.com/gchq/CyberChef/pull/441 [#443]: https://github.com/gchq/CyberChef/pull/443 +[#446]: https://github.com/gchq/CyberChef/pull/446 diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 690fea57..2e6bfe2d 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -342,8 +342,12 @@ ] }, { - "name": "Multimedia", + "name": "Forensics", "ops": [ + "Detect File Type", + "Scan for Embedded Files", + "Remove EXIF", + "Extract EXIF", "Render Image", "Play Media" ] @@ -354,16 +358,12 @@ "Entropy", "Frequency distribution", "Chi Square", - "Detect File Type", - "Scan for Embedded Files", "Disassemble x86", "Pseudo-Random Number Generator", "Generate UUID", "Generate TOTP", "Generate HOTP", "Haversine distance", - "Remove EXIF", - "Extract EXIF", "Numberwang", "XKCD Random Number" ] diff --git a/src/core/operations/PlayMedia.mjs b/src/core/operations/PlayMedia.mjs index e6ac53e4..81328a73 100644 --- a/src/core/operations/PlayMedia.mjs +++ b/src/core/operations/PlayMedia.mjs @@ -23,8 +23,8 @@ class PlayMedia extends Operation { super(); this.name = "Play Media"; - this.module = "Media"; - this.description = "Plays the input as sound or video depending on the type."; + this.module = "Default"; + this.description = "Plays the input as audio or video depending on the type.

Tags: sound, movie, mp3, mp4, mov, webm, wav, ogg"; this.infoURL = ""; this.inputType = "string"; this.outputType = "byteArray"; @@ -44,7 +44,7 @@ class PlayMedia extends Operation { * @returns {byteArray} The multimedia data as bytes. */ run(input, args) { - const inputFormat = args[0]; + const [inputFormat] = args; if (!input.length) return []; @@ -68,7 +68,7 @@ class PlayMedia extends Operation { // Determine file type const type = Magic.magicFileType(input); if (!(type && /^audio|video/.test(type.mime))) { - throw new OperationError("Invalid file type"); + throw new OperationError("Invalid or unrecognised file type"); } return input; @@ -77,6 +77,7 @@ class PlayMedia extends Operation { /** * Displays an audio or video element that may be able to play the media * file. + * * @param data {byteArray} Data containing an audio or video file. * @returns {string} Markup to display a media player. */ From 8c3569ea63728e1ccd8690f87a0eacde97385cc5 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 19 Dec 2018 17:59:55 +0000 Subject: [PATCH 25/25] 8.16.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8ea5bd80..f146cc0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.15.1", + "version": "8.16.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index e5cd12e9..cfa83d39 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.15.1", + "version": "8.16.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef",