From 59877b51389da0471c126dce242cd01c19688c31 Mon Sep 17 00:00:00 2001 From: d98762625 Date: Fri, 13 Apr 2018 12:14:40 +0100 Subject: [PATCH 001/421] Exporing options with API. --- package.json | 3 +- src/core/Dish.mjs | 20 +++++++ src/core/operations/ToBase32.mjs | 2 - src/node/Wrapper.mjs | 99 ++++++++++++++++++++++++++++++++ src/node/index.mjs | 42 +++++++++----- 5 files changed, 149 insertions(+), 17 deletions(-) create mode 100644 src/node/Wrapper.mjs diff --git a/package.json b/package.json index c44a3a1f..ff567519 100644 --- a/package.json +++ b/package.json @@ -118,6 +118,7 @@ "build": "grunt prod", "test": "grunt test", "docs": "grunt docs", - "lint": "grunt lint" + "lint": "grunt lint", + "build-node": "grunt node" } } diff --git a/src/core/Dish.mjs b/src/core/Dish.mjs index 6aeaf3e9..08c7206a 100755 --- a/src/core/Dish.mjs +++ b/src/core/Dish.mjs @@ -240,6 +240,26 @@ class Dish { } } + /** + * + */ + findType() { + if (!this.value) { + throw "Dish has no value"; + } + + const types = [Dish.BYTE_ARRAY, Dish.STRING, Dish.HTML, Dish.NUMBER, Dish.ARRAY_BUFFER, Dish.BIG_NUMBER, Dish.LIST_FILE]; + + types.find((type) => { + this.type = type; + if (this.valid()) { + return true; + } + }); + + return this.type; + } + /** * Determines how much space the Dish takes up. diff --git a/src/core/operations/ToBase32.mjs b/src/core/operations/ToBase32.mjs index 1b217a34..632d93e4 100644 --- a/src/core/operations/ToBase32.mjs +++ b/src/core/operations/ToBase32.mjs @@ -45,7 +45,6 @@ class ToBase32 extends Operation { chr1, chr2, chr3, chr4, chr5, enc1, enc2, enc3, enc4, enc5, enc6, enc7, enc8, i = 0; - while (i < input.length) { chr1 = input[i++]; chr2 = input[i++]; @@ -76,7 +75,6 @@ class ToBase32 extends Operation { alphabet.charAt(enc4) + alphabet.charAt(enc5) + alphabet.charAt(enc6) + alphabet.charAt(enc7) + alphabet.charAt(enc8); } - return output; } diff --git a/src/node/Wrapper.mjs b/src/node/Wrapper.mjs new file mode 100644 index 00000000..33f34b7b --- /dev/null +++ b/src/node/Wrapper.mjs @@ -0,0 +1,99 @@ +/** + * Wrap operations in a + * + * @author d98762625 [d98762625@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Dish from "../core/Dish"; +import log from "loglevel"; + +/** + * + */ +export default class Wrapper { + + /** + * + * @param arg + */ + extractArg(arg) { + if (arg.type === "option" || arg.type === "editableOption") { + return arg.value[0]; + } + + return arg.value; + } + + /** + * + */ + wrap(operation) { + this.operation = new operation(); + // This for just exposing run function: + // return this.run.bind(this); + + /** + * + * @param input + * @param args + */ + const _run = async(input, args=null) => { + const dish = new Dish(input); + + try { + dish.findType(); + } catch (e) { + log.debug(e); + } + + if (!args) { + args = this.operation.args.map(this.extractArg); + } else { + // Allows single arg ops to have arg defined not in array + if (!(args instanceof Array)) { + args = [args]; + } + } + const transformedInput = await dish.get(Dish.typeEnum(this.operation.inputType)); + return this.operation.innerRun(transformedInput, args); + }; + + // There's got to be a nicer way to do this! + this.operation.innerRun = this.operation.run; + this.operation.run = _run; + + return this.operation; + } + + /** + * + * @param input + */ + async run(input, args = null) { + const dish = new Dish(input); + + try { + dish.findType(); + } catch (e) { + log.debug(e); + } + + if (!args) { + args = this.operation.args.map(this.extractArg); + } else { + // Allows single arg ops to have arg defined not in array + if (!(args instanceof Array)) { + args = [args]; + } + } + + const transformedInput = await dish.get(Dish.typeEnum(this.operation.inputType)); + return this.operation.run(transformedInput, args); + + + + + } +} diff --git a/src/node/index.mjs b/src/node/index.mjs index c6e86c68..3dfda7d7 100644 --- a/src/node/index.mjs +++ b/src/node/index.mjs @@ -18,22 +18,36 @@ global.ENVIRONMENT_IS_WEB = function() { return typeof window === "object"; }; -import Chef from "../core/Chef"; +// import Chef from "../core/Chef"; -const CyberChef = { +// const CyberChef = { - bake: function(input, recipeConfig) { - this.chef = new Chef(); - return this.chef.bake( - input, - recipeConfig, - {}, - 0, - false - ); +// bake: function(input, recipeConfig) { +// this.chef = new Chef(); +// return this.chef.bake( +// input, +// recipeConfig, +// {}, +// 0, +// false +// ); +// } + +// }; + +// export default CyberChef; +// export {CyberChef}; + +import Wrapper from "./Wrapper"; + +import * as operations from "../core/operations/index"; + +const cyberChef = { + base32: { + from: new Wrapper().wrap(operations.FromBase32), + to: new Wrapper().wrap(operations.ToBase32), } - }; -export default CyberChef; -export {CyberChef}; +export default cyberChef; +export {cyberChef}; From fca4ed70131c57915e0119539c7adcd86986dbf7 Mon Sep 17 00:00:00 2001 From: d98762625 Date: Fri, 20 Apr 2018 10:55:17 +0100 Subject: [PATCH 002/421] simplified API --- src/node/Wrapper.mjs | 85 +++++++++++--------------------------------- src/node/index.mjs | 45 +++++++++-------------- 2 files changed, 37 insertions(+), 93 deletions(-) diff --git a/src/node/Wrapper.mjs b/src/node/Wrapper.mjs index 33f34b7b..66876295 100644 --- a/src/node/Wrapper.mjs +++ b/src/node/Wrapper.mjs @@ -10,68 +10,28 @@ import Dish from "../core/Dish"; import log from "loglevel"; /** - * + * Extract default arg value from operation argument + * @param {Object} arg - an arg from an operation */ -export default class Wrapper { - - /** - * - * @param arg - */ - extractArg(arg) { - if (arg.type === "option" || arg.type === "editableOption") { - return arg.value[0]; - } - - return arg.value; +function extractArg(arg) { + if (arg.type === "option" || arg.type === "editableOption") { + return arg.value[0]; } + return arg.value; +} + +/** + * Wrap an operation to be consumed by node API. + * new Operation().run() becomes operation() + * @param Operation + */ +export default function wrap(Operation) { /** * */ - wrap(operation) { - this.operation = new operation(); - // This for just exposing run function: - // return this.run.bind(this); - - /** - * - * @param input - * @param args - */ - const _run = async(input, args=null) => { - const dish = new Dish(input); - - try { - dish.findType(); - } catch (e) { - log.debug(e); - } - - if (!args) { - args = this.operation.args.map(this.extractArg); - } else { - // Allows single arg ops to have arg defined not in array - if (!(args instanceof Array)) { - args = [args]; - } - } - const transformedInput = await dish.get(Dish.typeEnum(this.operation.inputType)); - return this.operation.innerRun(transformedInput, args); - }; - - // There's got to be a nicer way to do this! - this.operation.innerRun = this.operation.run; - this.operation.run = _run; - - return this.operation; - } - - /** - * - * @param input - */ - async run(input, args = null) { + return async (input, args=null) => { + const operation = new Operation(); const dish = new Dish(input); try { @@ -81,19 +41,14 @@ export default class Wrapper { } if (!args) { - args = this.operation.args.map(this.extractArg); + args = operation.args.map(extractArg); } else { // Allows single arg ops to have arg defined not in array if (!(args instanceof Array)) { args = [args]; } } - - const transformedInput = await dish.get(Dish.typeEnum(this.operation.inputType)); - return this.operation.run(transformedInput, args); - - - - - } + const transformedInput = await dish.get(Dish.typeEnum(operation.inputType)); + return operation.run(transformedInput, args); + }; } diff --git a/src/node/index.mjs b/src/node/index.mjs index 3dfda7d7..8900fbd4 100644 --- a/src/node/index.mjs +++ b/src/node/index.mjs @@ -18,36 +18,25 @@ global.ENVIRONMENT_IS_WEB = function() { return typeof window === "object"; }; -// import Chef from "../core/Chef"; -// const CyberChef = { - -// bake: function(input, recipeConfig) { -// this.chef = new Chef(); -// return this.chef.bake( -// input, -// recipeConfig, -// {}, -// 0, -// false -// ); -// } - -// }; - -// export default CyberChef; -// export {CyberChef}; - -import Wrapper from "./Wrapper"; +import wrap from "./Wrapper"; import * as operations from "../core/operations/index"; -const cyberChef = { - base32: { - from: new Wrapper().wrap(operations.FromBase32), - to: new Wrapper().wrap(operations.ToBase32), - } -}; +/** + * + * @param name + */ +function decapitalise(name) { + return `${name.charAt(0).toLowerCase()}${name.substr(1)}`; +} -export default cyberChef; -export {cyberChef}; + +// console.log(operations); +const chef = {}; +Object.keys(operations).forEach(op => + chef[decapitalise(op)] = wrap(operations[op])); + + +export default chef; +export {chef}; From b8b98358d0643abe42dc7b5c0b4aafd3120c18e4 Mon Sep 17 00:00:00 2001 From: d98762625 Date: Fri, 20 Apr 2018 12:23:20 +0100 Subject: [PATCH 003/421] function tidy, add comments --- src/node/Wrapper.mjs | 54 -------------------- src/node/apiUtils.mjs | 112 ++++++++++++++++++++++++++++++++++++++++++ src/node/index.mjs | 22 +++------ 3 files changed, 119 insertions(+), 69 deletions(-) delete mode 100644 src/node/Wrapper.mjs create mode 100644 src/node/apiUtils.mjs diff --git a/src/node/Wrapper.mjs b/src/node/Wrapper.mjs deleted file mode 100644 index 66876295..00000000 --- a/src/node/Wrapper.mjs +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Wrap operations in a - * - * @author d98762625 [d98762625@gmail.com] - * @copyright Crown Copyright 2018 - * @license Apache-2.0 - */ - -import Dish from "../core/Dish"; -import log from "loglevel"; - -/** - * Extract default arg value from operation argument - * @param {Object} arg - an arg from an operation - */ -function extractArg(arg) { - if (arg.type === "option" || arg.type === "editableOption") { - return arg.value[0]; - } - - return arg.value; -} - -/** - * Wrap an operation to be consumed by node API. - * new Operation().run() becomes operation() - * @param Operation - */ -export default function wrap(Operation) { - /** - * - */ - return async (input, args=null) => { - const operation = new Operation(); - const dish = new Dish(input); - - try { - dish.findType(); - } catch (e) { - log.debug(e); - } - - if (!args) { - args = operation.args.map(extractArg); - } else { - // Allows single arg ops to have arg defined not in array - if (!(args instanceof Array)) { - args = [args]; - } - } - const transformedInput = await dish.get(Dish.typeEnum(operation.inputType)); - return operation.run(transformedInput, args); - }; -} diff --git a/src/node/apiUtils.mjs b/src/node/apiUtils.mjs new file mode 100644 index 00000000..f8457247 --- /dev/null +++ b/src/node/apiUtils.mjs @@ -0,0 +1,112 @@ +/** + * Wrap operations for consumption in Node + * + * @author d98762625 [d98762625@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Dish from "../core/Dish"; +import log from "loglevel"; + +/** + * Extract default arg value from operation argument + * @param {Object} arg - an arg from an operation + */ +function extractArg(arg) { + if (arg.type === "option" || arg.type === "editableOption") { + return arg.value[0]; + } + + return arg.value; +} + +/** + * Wrap an operation to be consumed by node API. + * new Operation().run() becomes operation() + * Perform type conversion on input + * @param {Operation} Operation + * @returns {Function} The operation's run function, wrapped in + * some type conversion logic + */ +export function wrap(Operation) { + /** + * Wrapped operation run function + */ + return async (input, args=null) => { + const operation = new Operation(); + const dish = new Dish(input); + + try { + dish.findType(); + } catch (e) { + log.debug(e); + } + + if (!args) { + args = operation.args.map(extractArg); + } else { + // Allows single arg ops to have arg defined not in array + if (!(args instanceof Array)) { + args = [args]; + } + } + const transformedInput = await dish.get(Dish.typeEnum(operation.inputType)); + return operation.run(transformedInput, args); + }; +} + +/** + * + * @param searchTerm + */ +export function search(searchTerm) { + +} + + +/** + * Extract properties from an operation by instantiating it and + * returning some of its properties for reference. + * @param {Operation} Operation - the operation to extract info from + * @returns {Object} operation properties + */ +function extractOperationInfo(Operation) { + const operation = new Operation(); + return { + name: operation.name, + module: operation.module, + description: operation.description, + inputType: operation.inputType, + outputType: operation.outputType, + args: Object.assign([], operation.args), + }; +} + + +/** + * @param {Object} operations - an object filled with operations. + * @param {String} searchTerm - the name of the operation to get help for. + * Case and whitespace are ignored in search. + * @returns {Object} listing properties of function + */ +export function help(operations, searchTerm) { + if (typeof searchTerm === "string") { + const operation = operations[Object.keys(operations).find(o => + o.toLowerCase() === searchTerm.replace(/ /g, "").toLowerCase())]; + if (operation) { + return extractOperationInfo(operation); + } + } + return null; +} + + +/** + * SomeName => someName + * @param {String} name - string to be altered + * @returns {String} decapitalised + */ +export function decapitalise(name) { + return `${name.charAt(0).toLowerCase()}${name.substr(1)}`; +} diff --git a/src/node/index.mjs b/src/node/index.mjs index 8900fbd4..1125685a 100644 --- a/src/node/index.mjs +++ b/src/node/index.mjs @@ -2,11 +2,14 @@ * Node view for CyberChef. * * @author n1474335 [n1474335@gmail.com] - * @copyright Crown Copyright 2017 + * @copyright Crown Copyright 2018 * @license Apache-2.0 */ import "babel-polyfill"; +import {wrap, help, decapitalise} from "./apiUtils"; +import * as operations from "../core/operations/index"; + // Define global environment functions global.ENVIRONMENT_IS_WORKER = function() { return typeof importScripts === "function"; @@ -19,24 +22,13 @@ global.ENVIRONMENT_IS_WEB = function() { }; -import wrap from "./Wrapper"; - -import * as operations from "../core/operations/index"; - -/** - * - * @param name - */ -function decapitalise(name) { - return `${name.charAt(0).toLowerCase()}${name.substr(1)}`; -} - - -// console.log(operations); const chef = {}; + +// Add in wrapped operations with camelCase names Object.keys(operations).forEach(op => chef[decapitalise(op)] = wrap(operations[op])); +chef.help = help.bind(null, operations); export default chef; export {chef}; From d5b5443a840bad6d5dcfb290eb6a00e123baea71 Mon Sep 17 00:00:00 2001 From: d98762625 Date: Fri, 27 Apr 2018 09:01:45 +0100 Subject: [PATCH 004/421] update readme --- .github/CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 6064121c..e90196e5 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -22,7 +22,7 @@ Before your contributions can be accepted, you must: * Line endings: UNIX style (\n) -## Design Principals +## Design Principles 1. If at all possible, all operations and features should be client-side and not rely on connections to an external server. This increases the utility of CyberChef on closed networks and in virtual machines that are not connected to the Internet. Calls to external APIs may be accepted if there is no other option, but not for critical components. 2. Latency should be kept to a minimum to enhance the user experience. This means that all operation code should sit on the client, rather than being loaded dynamically from a server. @@ -30,7 +30,7 @@ Before your contributions can be accepted, you must: 4. Minimise the use of large libraries, especially for niche operations that won't be used very often - these will be downloaded by everyone using the app, whether they use that operation or not (due to principal 2). -With these principals in mind, any changes or additions to CyberChef should keep it: +With these principles in mind, any changes or additions to CyberChef should keep it: - Standalone - Efficient From fbec0a1c7d5da7db3257dc8de6507284663c332d Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sat, 21 Apr 2018 12:25:48 +0100 Subject: [PATCH 005/421] The raw, unpresented dish is now returned to the app after baking, where it can be retrieved as various different data types. --- src/core/Chef.mjs | 22 ++++++- src/core/ChefWorker.js | 19 ++++++ src/core/Dish.mjs | 17 +++-- src/core/FlowControl.js | 4 +- src/core/Recipe.mjs | 26 +++++--- src/core/Utils.mjs | 37 ++++------- src/core/config/scripts/generateConfig.mjs | 2 +- src/web/HighlighterWaiter.js | 4 +- src/web/Manager.js | 1 - src/web/OutputWaiter.js | 75 ++++++++++++++-------- src/web/WorkerWaiter.js | 28 ++++++++ src/web/stylesheets/utils/_overrides.css | 1 + 12 files changed, 163 insertions(+), 73 deletions(-) diff --git a/src/core/Chef.mjs b/src/core/Chef.mjs index a935d75c..79172479 100755 --- a/src/core/Chef.mjs +++ b/src/core/Chef.mjs @@ -89,7 +89,14 @@ class Chef { const threshold = (options.ioDisplayThreshold || 1024) * 1024; const returnType = this.dish.size > threshold ? Dish.ARRAY_BUFFER : Dish.STRING; + // Create a raw version of the dish, unpresented + const rawDish = new Dish(this.dish); + + // Present the raw result + await recipe.present(this.dish); + return { + dish: rawDish, result: this.dish.type === Dish.HTML ? await this.dish.get(Dish.HTML, notUTF8) : await this.dish.get(returnType, notUTF8), @@ -123,7 +130,7 @@ class Chef { const startTime = new Date().getTime(), recipe = new Recipe(recipeConfig), - dish = new Dish("", Dish.STRING); + dish = new Dish(); try { recipe.execute(dish); @@ -167,6 +174,19 @@ class Chef { }; } + + /** + * Translates the dish to a specified type and returns it. + * + * @param {Dish} dish + * @param {string} type + * @returns {Dish} + */ + async getDishAs(dish, type) { + const newDish = new Dish(dish); + return await newDish.get(type); + } + } export default Chef; diff --git a/src/core/ChefWorker.js b/src/core/ChefWorker.js index 604189e7..dbbda126 100644 --- a/src/core/ChefWorker.js +++ b/src/core/ChefWorker.js @@ -60,6 +60,9 @@ self.addEventListener("message", function(e) { case "silentBake": silentBake(r.data); break; + case "getDishAs": + getDishAs(r.data); + break; case "docURL": // Used to set the URL of the current document so that scripts can be // imported into an inline worker. @@ -125,6 +128,22 @@ function silentBake(data) { } +/** + * Translates the dish to a given type. + */ +async function getDishAs(data) { + const value = await self.chef.getDishAs(data.dish, data.type); + + self.postMessage({ + action: "dishReturned", + data: { + value: value, + id: data.id + } + }); +} + + /** * Checks that all required modules are loaded and loads them if not. * diff --git a/src/core/Dish.mjs b/src/core/Dish.mjs index 08c7206a..888432da 100755 --- a/src/core/Dish.mjs +++ b/src/core/Dish.mjs @@ -17,14 +17,17 @@ class Dish { /** * Dish constructor * - * @param {byteArray|string|number|ArrayBuffer|BigNumber} [value=null] - * - The value of the input data. - * @param {number} [type=Dish.BYTE_ARRAY] - * - The data type of value, see Dish enums. + * @param {Dish} [dish=null] - A dish to clone */ - constructor(value=null, type=Dish.BYTE_ARRAY) { - this.value = value; - this.type = type; + constructor(dish=null) { + this.value = []; + this.type = Dish.BYTE_ARRAY; + + if (dish && + dish.hasOwnProperty("value") && + dish.hasOwnProperty("type")) { + this.set(dish.value, dish.type); + } } diff --git a/src/core/FlowControl.js b/src/core/FlowControl.js index 92440c49..f1bb8dab 100755 --- a/src/core/FlowControl.js +++ b/src/core/FlowControl.js @@ -68,7 +68,9 @@ const FlowControl = { op.ingValues = JSON.parse(JSON.stringify(ingValues[i])); }); - const dish = new Dish(inputs[i], inputType); + const dish = new Dish(); + dish.set(inputs[i], inputType); + try { progress = await recipe.execute(dish, 0, state); } catch (err) { diff --git a/src/core/Recipe.mjs b/src/core/Recipe.mjs index 006e431c..95dab22b 100755 --- a/src/core/Recipe.mjs +++ b/src/core/Recipe.mjs @@ -130,10 +130,12 @@ class Recipe { * - The final progress through the recipe */ async execute(dish, startFrom=0, forkState={}) { - let op, input, output, lastRunOp, + let op, input, output, numJumps = 0, numRegisters = forkState.numRegisters || 0; + if (startFrom === 0) this.lastRunOp = null; + log.debug(`[*] Executing recipe of ${this.opList.length} operations, starting at ${startFrom}`); for (let i = startFrom; i < this.opList.length; i++) { @@ -169,7 +171,7 @@ class Recipe { numRegisters = state.numRegisters; } else { output = await op.run(input, op.ingValues); - lastRunOp = op; + this.lastRunOp = op; dish.set(output, op.outputType); } } catch (err) { @@ -188,18 +190,24 @@ class Recipe { } } - // Present the results of the final operation - if (lastRunOp) { - // TODO try/catch - output = await lastRunOp.present(output); - dish.set(output, lastRunOp.presentType); - } - log.debug("Recipe complete"); return this.opList.length; } + /** + * Present the results of the final operation. + * + * @param {Dish} dish + */ + async present(dish) { + if (!this.lastRunOp) return; + + const output = await this.lastRunOp.present(await dish.get(this.lastRunOp.outputType)); + dish.set(output, this.lastRunOp.presentType); + } + + /** * Returns the recipe configuration in string format. * diff --git a/src/core/Utils.mjs b/src/core/Utils.mjs index 88cfa52e..f90bc394 100755 --- a/src/core/Utils.mjs +++ b/src/core/Utils.mjs @@ -7,7 +7,7 @@ import utf8 from "utf8"; import moment from "moment-timezone"; import {fromBase64} from "./lib/Base64"; -import {toHexFast, fromHex} from "./lib/Hex"; +import {fromHex} from "./lib/Hex"; /** @@ -833,39 +833,24 @@ class Utils { const formatFile = async function(file, i) { const buff = await Utils.readFile(file); - const fileStr = Utils.arrayBufferToStr(buff.buffer); const blob = new Blob( [buff], {type: "octet/stream"} ); - const blobUrl = URL.createObjectURL(blob); - - const viewFileElem = ``; - - const downloadFileElem = `💾`; - - const hexFileData = toHexFast(buff); - - const switchToInputElem = `⇧`; const html = `