From 0f0e346a02bac4615ab3c7f168d65b2f3146dd8e Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 8 Jan 2019 11:12:02 +0000 Subject: [PATCH 1/4] Add new Subsection operation --- src/core/operations/Subsection.mjs | 141 +++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 src/core/operations/Subsection.mjs diff --git a/src/core/operations/Subsection.mjs b/src/core/operations/Subsection.mjs new file mode 100644 index 00000000..b9981100 --- /dev/null +++ b/src/core/operations/Subsection.mjs @@ -0,0 +1,141 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import XRegExp from "xregexp"; +import Operation from "../Operation"; +import Recipe from "../Recipe"; +import Dish from "../Dish"; + +/** + * Subsection operation + */ +class Subsection extends Operation { + + /** + * Subsection constructor + */ + constructor() { + super(); + + this.name = "Subsection"; + this.flowControl = true; + this.module = "Regex"; + this.description = "Select a part of the input data using regex, and run all subsequent operations on each match separately."; + this.infoURL = ""; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + "name": "Section (regex)", + "type": "string", + "value": "" + }, + { + "name": "Case sensitive matching", + "type": "boolean", + "value": true + }, + { + "name": "Ignore errors", + "type": "boolean", + "value": false + } + ]; + } + + /** + * @param {Object} state - The current state of the recipe. + * @param {number} state.progress - The current position in the recipe. + * @param {Dish} state.Dish - The Dish being operated on + * @param {Operation[]} state.opList - The list of operations in the recipe + * @returns {Object} - The updated state of the recipe + */ + async run(state) { + const opList = state.opList, + inputType = opList[state.progress].inputType, + outputType = opList[state.progress].outputType, + input = await state.dish.get(inputType), + ings = opList[state.progress].ingValues, + [section, caseSensitive, ignoreErrors] = ings, + subOpList = []; + + if (input && section !== "") { + // Create subOpList for each tranche to operate on + // (all remaining operations unless we encounter a Merge) + for (let i = state.progress + 1; i < opList.length; i++) { + if (opList[i].name === "Merge" && !opList[i].disabled) { + break; + } else { + subOpList.push(opList[i]); + } + } + + let flags = "g", + inOffset = 0, + output = "", + m, + progress = 0; + if (!caseSensitive) + flags += "i"; + const regex = new XRegExp(section, flags), + recipe = new Recipe(); + + recipe.addOperations(subOpList); + state.forkOffset += state.progress + 1; + + // Take a deep(ish) copy of the ingredient values + const ingValues = subOpList.map(op => JSON.parse(JSON.stringify(op.ingValues))); + let matched = false; + + // Run recipe over each match + while ((m = regex.exec(input))) { + matched = true; + // Add up to match + let matchStr = m[0]; + if (m.length === 1) { + output += input.slice(inOffset, m.index); + inOffset = regex.lastIndex; + } else if (m.length >= 2) { + matchStr = m[1]; + // Need to add some of the matched string that isn't in the capture group + output += input.slice(inOffset, m.index + m[0].indexOf(m[1])); + // Set i to be after the end of the first capture group + inOffset = regex.lastIndex - (m[0].length - (m[0].indexOf(m[1]) + m[1].length)); + } + // Baseline ing values for each tranche so that registers are reset + subOpList.forEach((op, i) => { + op.ingValues = JSON.parse(JSON.stringify(ingValues[i])); + }); + + const dish = new Dish(); + dish.set(matchStr, inputType); + + try { + progress = await recipe.execute(dish, 0, state); + } catch (err) { + if (!ignoreErrors) { + throw err; + } + progress = err.progress + 1; + } + output += await dish.get(outputType); + } + // If no matches were found, advance progress to after a Merge op + // Otherwise, the operations below Subsection will be run on all the input data + if (!matched) { + state.progress += subOpList.length + 1; + } + + output += input.slice(inOffset); + state.progress += progress; + state.dish.set(output, outputType); + } + return state; + } + +} + +export default Subsection; From 1a827ef44f034fc90c652e594fc97b4866935488 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 8 Jan 2019 11:17:06 +0000 Subject: [PATCH 2/4] Add Subsection to Flow Control category --- src/core/config/Categories.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index e9fe3399..832c91e8 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -364,6 +364,7 @@ "ops": [ "Magic", "Fork", + "Subsection", "Merge", "Register", "Label", From 8ac5b484938fe86da5fc332f0e6c4f686ae73db1 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 8 Jan 2019 11:51:33 +0000 Subject: [PATCH 3/4] Update operation description --- src/core/operations/Subsection.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Subsection.mjs b/src/core/operations/Subsection.mjs index b9981100..94258b6d 100644 --- a/src/core/operations/Subsection.mjs +++ b/src/core/operations/Subsection.mjs @@ -23,7 +23,7 @@ class Subsection extends Operation { this.name = "Subsection"; this.flowControl = true; this.module = "Regex"; - this.description = "Select a part of the input data using regex, and run all subsequent operations on each match separately."; + this.description = "Select a part of the input data using a regular expression (regex), and run all subsequent operations on each match separately.

You can use up to one capture group, where the recipe will only be run on the data in the capture group. If there's more than one capture group, only the first one will be operated on."; this.infoURL = ""; this.inputType = "string"; this.outputType = "string"; From c2068b343b3ae7c767d341f6c798f0fda35f3c59 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 10 Jan 2019 15:42:48 +0000 Subject: [PATCH 4/4] Tidied up and added global matching to Subsection operation --- CHANGELOG.md | 5 +++++ src/core/operations/Subsection.mjs | 30 +++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4761b540..71949c82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.22.0] - 2019-01-10 +- 'Subsection' operation added [@j433866] | [#467] + ### [8.21.0] - 2019-01-10 - 'To Case Insensitive Regex' and 'From Case Insensitive Regex' operations added [@masq] | [#461] @@ -94,6 +97,7 @@ All major and minor version changes will be documented in this file. Details of +[8.22.0]: https://github.com/gchq/CyberChef/releases/tag/v8.22.0 [8.21.0]: https://github.com/gchq/CyberChef/releases/tag/v8.21.0 [8.20.0]: https://github.com/gchq/CyberChef/releases/tag/v8.20.0 [8.19.0]: https://github.com/gchq/CyberChef/releases/tag/v8.19.0 @@ -171,3 +175,4 @@ All major and minor version changes will be documented in this file. Details of [#455]: https://github.com/gchq/CyberChef/pull/455 [#458]: https://github.com/gchq/CyberChef/pull/458 [#461]: https://github.com/gchq/CyberChef/pull/461 +[#467]: https://github.com/gchq/CyberChef/pull/467 diff --git a/src/core/operations/Subsection.mjs b/src/core/operations/Subsection.mjs index 94258b6d..8133d31c 100644 --- a/src/core/operations/Subsection.mjs +++ b/src/core/operations/Subsection.mjs @@ -38,6 +38,11 @@ class Subsection extends Operation { "type": "boolean", "value": true }, + { + "name": "Global matching", + "type": "boolean", + "value": true + }, { "name": "Ignore errors", "type": "boolean", @@ -49,7 +54,7 @@ class Subsection extends Operation { /** * @param {Object} state - The current state of the recipe. * @param {number} state.progress - The current position in the recipe. - * @param {Dish} state.Dish - The Dish being operated on + * @param {Dish} state.dish - The Dish being operated on * @param {Operation[]} state.opList - The list of operations in the recipe * @returns {Object} - The updated state of the recipe */ @@ -59,12 +64,12 @@ class Subsection extends Operation { outputType = opList[state.progress].outputType, input = await state.dish.get(inputType), ings = opList[state.progress].ingValues, - [section, caseSensitive, ignoreErrors] = ings, + [section, caseSensitive, global, ignoreErrors] = ings, subOpList = []; if (input && section !== "") { // Create subOpList for each tranche to operate on - // (all remaining operations unless we encounter a Merge) + // all remaining operations unless we encounter a Merge for (let i = state.progress + 1; i < opList.length; i++) { if (opList[i].name === "Merge" && !opList[i].disabled) { break; @@ -73,13 +78,15 @@ class Subsection extends Operation { } } - let flags = "g", + let flags = "", inOffset = 0, output = "", m, progress = 0; - if (!caseSensitive) - flags += "i"; + + if (!caseSensitive) flags += "i"; + if (global) flags += "g"; + const regex = new XRegExp(section, flags), recipe = new Recipe(); @@ -95,16 +102,19 @@ class Subsection extends Operation { matched = true; // Add up to match let matchStr = m[0]; - if (m.length === 1) { + + if (m.length === 1) { // No capture groups output += input.slice(inOffset, m.index); - inOffset = regex.lastIndex; + inOffset = m.index + m[0].length; } else if (m.length >= 2) { matchStr = m[1]; + // Need to add some of the matched string that isn't in the capture group output += input.slice(inOffset, m.index + m[0].indexOf(m[1])); // Set i to be after the end of the first capture group - inOffset = regex.lastIndex - (m[0].length - (m[0].indexOf(m[1]) + m[1].length)); + inOffset = m.index + m[0].indexOf(m[1]) + m[1].length; } + // Baseline ing values for each tranche so that registers are reset subOpList.forEach((op, i) => { op.ingValues = JSON.parse(JSON.stringify(ingValues[i])); @@ -122,7 +132,9 @@ class Subsection extends Operation { progress = err.progress + 1; } output += await dish.get(outputType); + if (!regex.global) break; } + // If no matches were found, advance progress to after a Merge op // Otherwise, the operations below Subsection will be run on all the input data if (!matched) {