From 1b16c26699d4f53917a2267308b2d8c4efbf3a13 Mon Sep 17 00:00:00 2001 From: mshwed Date: Mon, 11 Mar 2019 09:32:44 -0400 Subject: [PATCH 1/9] Operation: Added extract hash feature --- src/core/config/Categories.json | 1 + src/core/operations/ExtractHashes.mjs | 81 +++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 src/core/operations/ExtractHashes.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 2dd9f29f..2d7b30ff 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -337,6 +337,7 @@ "Extract domains", "Extract file paths", "Extract dates", + "Extract Hashes", "Regular expression", "XPath expression", "JPath expression", diff --git a/src/core/operations/ExtractHashes.mjs b/src/core/operations/ExtractHashes.mjs new file mode 100644 index 00000000..ed186fb3 --- /dev/null +++ b/src/core/operations/ExtractHashes.mjs @@ -0,0 +1,81 @@ +/** + * @author mshwed [m@ttshwed.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import { search } from "../lib/Extract"; + +/** + * Extract Hash Values operation + */ +class ExtractHashes extends Operation { + + /** + * ExtractHashValues constructor + */ + constructor() { + super(); + + this.name = "Extract Hashes"; + this.module = "Default"; + this.description = "Extracts hash values based on hash byte length"; + this.infoURL = "https://en.wikipedia.org/wiki/Comparison_of_cryptographic_hash_functions"; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + name: "Hash length", + type: "number", + value: 32 + }, + { + name: "All hashes", + type: "boolean", + value: false + }, + { + name: "Display Total", + type: "boolean", + value: false + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + let results = []; + let hashCount = 0; + + const hashLength = args[0]; + const searchAllHashes = args[1]; + const showDisplayTotal = args[2]; + + let hashLengths = [hashLength]; + if (searchAllHashes) hashLengths = [4, 8, 16, 32, 64, 128, 160, 192, 224, 256, 320, 384, 512, 1024]; + + for (let hashLength of hashLengths) { + const regex = new RegExp(`(\\b|^)[a-f0-9]{${hashLength}}(\\b|$)`, "g"); + const searchResults = search(input, regex, null, false); + + hashCount += searchResults.split("\n").length - 1; + results.push(searchResults); + } + + let output = ""; + if (showDisplayTotal) { + output = `Total Results: ${hashCount}\n\n`; + } + + output = output + results.join(""); + return output; + } + +} + +export default ExtractHashes; From 98edef389ca7100d1b3dde6ac39062550be7d90c Mon Sep 17 00:00:00 2001 From: mshwed Date: Mon, 11 Mar 2019 09:53:12 -0400 Subject: [PATCH 2/9] Corrected module type --- src/core/operations/ExtractHashes.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/ExtractHashes.mjs b/src/core/operations/ExtractHashes.mjs index ed186fb3..27402835 100644 --- a/src/core/operations/ExtractHashes.mjs +++ b/src/core/operations/ExtractHashes.mjs @@ -19,7 +19,7 @@ class ExtractHashes extends Operation { super(); this.name = "Extract Hashes"; - this.module = "Default"; + this.module = "Regex"; this.description = "Extracts hash values based on hash byte length"; this.infoURL = "https://en.wikipedia.org/wiki/Comparison_of_cryptographic_hash_functions"; this.inputType = "string"; From de8ed6962d99cc11b632da39753581b70e950846 Mon Sep 17 00:00:00 2001 From: mshwed Date: Mon, 11 Mar 2019 20:02:49 -0400 Subject: [PATCH 3/9] Improved description of operation --- src/core/operations/ExtractHashes.mjs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/core/operations/ExtractHashes.mjs b/src/core/operations/ExtractHashes.mjs index 27402835..f411f12d 100644 --- a/src/core/operations/ExtractHashes.mjs +++ b/src/core/operations/ExtractHashes.mjs @@ -20,13 +20,13 @@ class ExtractHashes extends Operation { this.name = "Extract Hashes"; this.module = "Regex"; - this.description = "Extracts hash values based on hash byte length"; + this.description = "Extracts potential hashes based on hash character length"; this.infoURL = "https://en.wikipedia.org/wiki/Comparison_of_cryptographic_hash_functions"; this.inputType = "string"; this.outputType = "string"; this.args = [ { - name: "Hash length", + name: "Hash character length", type: "number", value: 32 }, @@ -56,11 +56,16 @@ class ExtractHashes extends Operation { const searchAllHashes = args[1]; const showDisplayTotal = args[2]; - let hashLengths = [hashLength]; - if (searchAllHashes) hashLengths = [4, 8, 16, 32, 64, 128, 160, 192, 224, 256, 320, 384, 512, 1024]; + // Convert character length to bit length + let hashBitLengths = [(hashLength / 2) * 8]; - for (let hashLength of hashLengths) { - const regex = new RegExp(`(\\b|^)[a-f0-9]{${hashLength}}(\\b|$)`, "g"); + if (searchAllHashes) hashBitLengths = [4, 8, 16, 32, 64, 128, 160, 192, 224, 256, 320, 384, 512, 1024]; + + for (let hashBitLength of hashBitLengths) { + // Convert bit length to character length + let hashCharacterLength = (hashBitLength / 8) * 2; + + const regex = new RegExp(`(\\b|^)[a-f0-9]{${hashCharacterLength}}(\\b|$)`, "g"); const searchResults = search(input, regex, null, false); hashCount += searchResults.split("\n").length - 1; From a6b774da8168a86442d8a3240defaa7b154f4ddf Mon Sep 17 00:00:00 2001 From: mshwed Date: Tue, 12 Mar 2019 10:13:28 -0400 Subject: [PATCH 4/9] Fixed issues with const/let and changed default character length --- src/core/operations/ExtractHashes.mjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/operations/ExtractHashes.mjs b/src/core/operations/ExtractHashes.mjs index f411f12d..ac4765ec 100644 --- a/src/core/operations/ExtractHashes.mjs +++ b/src/core/operations/ExtractHashes.mjs @@ -28,7 +28,7 @@ class ExtractHashes extends Operation { { name: "Hash character length", type: "number", - value: 32 + value: 40 }, { name: "All hashes", @@ -49,7 +49,7 @@ class ExtractHashes extends Operation { * @returns {string} */ run(input, args) { - let results = []; + const results = []; let hashCount = 0; const hashLength = args[0]; @@ -61,9 +61,9 @@ class ExtractHashes extends Operation { if (searchAllHashes) hashBitLengths = [4, 8, 16, 32, 64, 128, 160, 192, 224, 256, 320, 384, 512, 1024]; - for (let hashBitLength of hashBitLengths) { + for (const hashBitLength of hashBitLengths) { // Convert bit length to character length - let hashCharacterLength = (hashBitLength / 8) * 2; + const hashCharacterLength = (hashBitLength / 8) * 2; const regex = new RegExp(`(\\b|^)[a-f0-9]{${hashCharacterLength}}(\\b|$)`, "g"); const searchResults = search(input, regex, null, false); From 3983e1a8e23ca4bf8e7c5d52b0bbb562b77d946f Mon Sep 17 00:00:00 2001 From: mshwed Date: Sun, 31 Mar 2024 10:57:03 -0400 Subject: [PATCH 5/9] Updated imports --- src/core/operations/ExtractHashes.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/operations/ExtractHashes.mjs b/src/core/operations/ExtractHashes.mjs index ac4765ec..ba31164f 100644 --- a/src/core/operations/ExtractHashes.mjs +++ b/src/core/operations/ExtractHashes.mjs @@ -4,8 +4,8 @@ * @license Apache-2.0 */ -import Operation from "../Operation"; -import { search } from "../lib/Extract"; +import Operation from "../Operation.mjs"; +import { search } from "../lib/Extract.mjs"; /** * Extract Hash Values operation From 61295a968ef932cb7d9b0f0f9a3b742feee0b700 Mon Sep 17 00:00:00 2001 From: a3957273 <89583054+a3957273@users.noreply.github.com> Date: Mon, 1 Apr 2024 16:01:48 +0000 Subject: [PATCH 6/9] Lower case 'hash' --- src/core/config/Categories.json | 2 +- src/core/operations/ExtractHashes.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 2d7b30ff..cfbe99e0 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -337,7 +337,7 @@ "Extract domains", "Extract file paths", "Extract dates", - "Extract Hashes", + "Extract hashes", "Regular expression", "XPath expression", "JPath expression", diff --git a/src/core/operations/ExtractHashes.mjs b/src/core/operations/ExtractHashes.mjs index ba31164f..abba0b54 100644 --- a/src/core/operations/ExtractHashes.mjs +++ b/src/core/operations/ExtractHashes.mjs @@ -18,7 +18,7 @@ class ExtractHashes extends Operation { constructor() { super(); - this.name = "Extract Hashes"; + this.name = "Extract hashes"; this.module = "Regex"; this.description = "Extracts potential hashes based on hash character length"; this.infoURL = "https://en.wikipedia.org/wiki/Comparison_of_cryptographic_hash_functions"; From 6b95ba7dd6f9afdfe39b8fe57d12f61064ed827a Mon Sep 17 00:00:00 2001 From: a3957273 <89583054+a3957273@users.noreply.github.com> Date: Mon, 1 Apr 2024 16:10:42 +0000 Subject: [PATCH 7/9] Fix regular expresion crash in extract hashes --- src/core/operations/ExtractHashes.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/operations/ExtractHashes.mjs b/src/core/operations/ExtractHashes.mjs index abba0b54..3f15e53c 100644 --- a/src/core/operations/ExtractHashes.mjs +++ b/src/core/operations/ExtractHashes.mjs @@ -68,8 +68,8 @@ class ExtractHashes extends Operation { const regex = new RegExp(`(\\b|^)[a-f0-9]{${hashCharacterLength}}(\\b|$)`, "g"); const searchResults = search(input, regex, null, false); - hashCount += searchResults.split("\n").length - 1; - results.push(searchResults); + hashCount += searchResults.length; + results.push(...searchResults); } let output = ""; @@ -77,7 +77,7 @@ class ExtractHashes extends Operation { output = `Total Results: ${hashCount}\n\n`; } - output = output + results.join(""); + output = output + results.join("\n"); return output; } From 8d4ad6ae75a26f0270f6bd4297097c15410aa52b Mon Sep 17 00:00:00 2001 From: mshwed Date: Mon, 1 Apr 2024 22:22:43 -0400 Subject: [PATCH 8/9] Minor changes. Added test cases. --- src/core/operations/ExtractHashes.mjs | 4 +- tests/operations/index.mjs | 1 + tests/operations/tests/ExtractHashes.mjs | 77 ++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 tests/operations/tests/ExtractHashes.mjs diff --git a/src/core/operations/ExtractHashes.mjs b/src/core/operations/ExtractHashes.mjs index 3f15e53c..386aab0e 100644 --- a/src/core/operations/ExtractHashes.mjs +++ b/src/core/operations/ExtractHashes.mjs @@ -52,9 +52,7 @@ class ExtractHashes extends Operation { const results = []; let hashCount = 0; - const hashLength = args[0]; - const searchAllHashes = args[1]; - const showDisplayTotal = args[2]; + const [hashLength, searchAllHashes, showDisplayTotal] = args; // Convert character length to bit length let hashBitLengths = [(hashLength / 2) * 8]; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index 9f9be2b7..17298700 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -62,6 +62,7 @@ import "./tests/DefangIP.mjs"; import "./tests/ELFInfo.mjs"; import "./tests/Enigma.mjs"; import "./tests/ExtractEmailAddresses.mjs"; +import "./tests/ExtractHashes.mjs"; import "./tests/Float.mjs"; import "./tests/FileTree.mjs"; import "./tests/FletcherChecksum.mjs"; diff --git a/tests/operations/tests/ExtractHashes.mjs b/tests/operations/tests/ExtractHashes.mjs new file mode 100644 index 00000000..1dfb1ff2 --- /dev/null +++ b/tests/operations/tests/ExtractHashes.mjs @@ -0,0 +1,77 @@ +/** + * ExtractHashes tests. + * + * @author mshwed [m@ttshwed.com] + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "Extract MD5 hash", + input: "The quick brown fox jumps over the lazy dog\n\nMD5: 9e107d9d372bb6826bd81d3542a419d6", + expectedOutput: "9e107d9d372bb6826bd81d3542a419d6", + recipeConfig: [ + { + "op": "Extract Hashes", + "args": [32, false, false] + }, + ], + }, + { + name: "Extract SHA1 hash", + input: "The quick brown fox jumps over the lazy dog\n\nSHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", + expectedOutput: "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", + recipeConfig: [ + { + "op": "Extract Hashes", + "args": [40, false, false] + }, + ], + }, + { + name: "Extract SHA256 hash", + input: "The quick brown fox jumps over the lazy dog\n\nSHA256: d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", + expectedOutput: "d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", + recipeConfig: [ + { + "op": "Extract Hashes", + "args": [64, false, false] + }, + ], + }, + { + name: "Extract SHA512 hash", + input: "The quick brown fox jumps over the lazy dog\n\nSHA512: 07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6", + expectedOutput: "07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6", + recipeConfig: [ + { + "op": "Extract Hashes", + "args": [128, false, false] + }, + ], + }, + { + name: "Extract all hashes", + input: "The quick brown fox jumps over the lazy dog\n\nMD5: 9e107d9d372bb6826bd81d3542a419d6\nSHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12\nSHA256: d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", + expectedOutput: "9e107d9d372bb6826bd81d3542a419d6\n2fd4e1c67a2d28fced849ee1bb76e7391b93eb12\nd7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", + recipeConfig: [ + { + "op": "Extract Hashes", + "args": [0, true, false] + }, + ], + }, + { + name: "Extract hashes with total count", + input: "The quick brown fox jumps over the lazy dog\n\nMD5: 9e107d9d372bb6826bd81d3542a419d6\nSHA1: 2fd4e1c67a2d28fced849ee1bb76e7391b93eb12\nSHA256: d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", + expectedOutput: "Total Results: 3\n\n9e107d9d372bb6826bd81d3542a419d6\n2fd4e1c67a2d28fced849ee1bb76e7391b93eb12\nd7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", + recipeConfig: [ + { + "op": "Extract Hashes", + "args": [0, true, true] + }, + ], + } +]); From 077b11e33bc9e1015147380309cd964f52bd657c Mon Sep 17 00:00:00 2001 From: mshwed Date: Mon, 1 Apr 2024 22:30:18 -0400 Subject: [PATCH 9/9] Fixed op name in test --- tests/operations/tests/ExtractHashes.mjs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/operations/tests/ExtractHashes.mjs b/tests/operations/tests/ExtractHashes.mjs index 1dfb1ff2..fe739418 100644 --- a/tests/operations/tests/ExtractHashes.mjs +++ b/tests/operations/tests/ExtractHashes.mjs @@ -14,7 +14,7 @@ TestRegister.addTests([ expectedOutput: "9e107d9d372bb6826bd81d3542a419d6", recipeConfig: [ { - "op": "Extract Hashes", + "op": "Extract hashes", "args": [32, false, false] }, ], @@ -25,7 +25,7 @@ TestRegister.addTests([ expectedOutput: "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12", recipeConfig: [ { - "op": "Extract Hashes", + "op": "Extract hashes", "args": [40, false, false] }, ], @@ -36,7 +36,7 @@ TestRegister.addTests([ expectedOutput: "d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", recipeConfig: [ { - "op": "Extract Hashes", + "op": "Extract hashes", "args": [64, false, false] }, ], @@ -47,7 +47,7 @@ TestRegister.addTests([ expectedOutput: "07e547d9586f6a73f73fbac0435ed76951218fb7d0c8d788a309d785436bbb642e93a252a954f23912547d1e8a3b5ed6e1bfd7097821233fa0538f3db854fee6", recipeConfig: [ { - "op": "Extract Hashes", + "op": "Extract hashes", "args": [128, false, false] }, ], @@ -58,7 +58,7 @@ TestRegister.addTests([ expectedOutput: "9e107d9d372bb6826bd81d3542a419d6\n2fd4e1c67a2d28fced849ee1bb76e7391b93eb12\nd7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", recipeConfig: [ { - "op": "Extract Hashes", + "op": "Extract hashes", "args": [0, true, false] }, ], @@ -69,7 +69,7 @@ TestRegister.addTests([ expectedOutput: "Total Results: 3\n\n9e107d9d372bb6826bd81d3542a419d6\n2fd4e1c67a2d28fced849ee1bb76e7391b93eb12\nd7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592", recipeConfig: [ { - "op": "Extract Hashes", + "op": "Extract hashes", "args": [0, true, true] }, ],