From fadd7158ed4cd79c1a3800ff1a4973bee8a4f4c7 Mon Sep 17 00:00:00 2001 From: Dale Myers Date: Mon, 23 Jan 2017 21:39:08 +0000 Subject: [PATCH 01/78] Add string escape/unescape operations These operations are useful for taking the contents of a string, and making it suitable for use as a stand alone string. For example, in an IDE you might see a string which is represented as: "Say \"Hello\"". The escaped double quotes are shown to make it clear that they do not end the string, despite the fact that they are not truly part of the string. In order to get the raw string, you would need to copy this, then manually remove the backslashes. The new String_.run_unescape operation does this automatically. The String_.run_escape is the inverse. It allows you to take a string like the one above, and paste it between two quotes without having to manually escape it. --- src/js/config/Categories.js | 2 + src/js/config/OperationConfig.js | 16 +++++++- src/js/operations/String.js | 67 ++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100755 src/js/operations/String.js diff --git a/src/js/config/Categories.js b/src/js/config/Categories.js index db43721c..d7ad1407 100755 --- a/src/js/config/Categories.js +++ b/src/js/config/Categories.js @@ -167,6 +167,8 @@ var Categories = [ "Parse UNIX file permissions", "Swap endianness", "Parse colour code", + "Escape String", + "Unescape String", ] }, { diff --git a/src/js/config/OperationConfig.js b/src/js/config/OperationConfig.js index b5ea0427..cbed494f 100755 --- a/src/js/config/OperationConfig.js +++ b/src/js/config/OperationConfig.js @@ -2952,5 +2952,19 @@ var OperationConfig = { value: Cipher.SUBS_CIPHERTEXT } ] - } + }, + "Escape String": { + description: "Escapes a string so that it can be embedded in another. For example, Don't stop me now becomes Don\\'t stop me now.", + run: String_.run_escape, + input_type: "string", + output_type: "string", + args: [] + }, + "Unescape String": { + description: "Unescapes a string that was embedded inside another so that it can be used in it's own right. For example, Don\\'t stop me now becomes Don't stop me now.", + run: String_.run_unescape, + input_type: "string", + output_type: "string", + args: [] + }, }; diff --git a/src/js/operations/String.js b/src/js/operations/String.js new file mode 100755 index 00000000..a85b385b --- /dev/null +++ b/src/js/operations/String.js @@ -0,0 +1,67 @@ +/** + * String operations. + * Namespace is appended with an underscore to prevent overwriting the global String object. + * + * @author Vel0x + * @namespace + */ +var String_ = { + + /** + * @constant + * @default + */ + REPLACEMENTS: [ + {"escaped": "\\\\", "unescaped":"\\"}, // Must be first + {"escaped": "\\'", "unescaped":"'"}, + {"escaped": "\\\"", "unescaped":"\""}, + {"escaped": "\\n", "unescaped":"\n"}, + {"escaped": "\\r", "unescaped":"\r"}, + ], + + /** + * Escapes a string for embedding in another string. + * + * Example: "Don't do that" -> "Don\'t do that" + * + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run_escape: function(input, args) { + return String_._replace_by_keys(input, "unescaped", "escaped") + }, + + /** + * Unescapes a string that was part of another string + * + * Example: "Don\'t do that" -> "Don't do that" + * + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run_unescape: function(input, args) { + return String_._replace_by_keys(input, "escaped", "unescaped") + }, + + /** + * Replaces all matching tokens in REPLACEMENTS with the correction. The + * ordering is determined by the pattern_key and the replacement_key. + * + * @param {string} input + * @param {string} pattern_key + * @param {string} replacement_key + * @returns {string} + */ + _replace_by_keys: function(input, pattern_key, replacement_key) { + var output = input; + var replacementsLength = String_.REPLACEMENTS.length; + for (var i = 0; i < replacementsLength; i++) { + var replacement = String_.REPLACEMENTS[i]; + output = output.split(replacement[pattern_key]).join(replacement[replacement_key]); + } + return output + }, + +}; From 90ed62add26c7dd0d184298b2fe11795e4245da9 Mon Sep 17 00:00:00 2001 From: Matt C Date: Tue, 25 Jul 2017 11:49:23 +0100 Subject: [PATCH 02/78] Fixes gchq/CyberChef#137 Changes data-trigger to focus so scrolling works and sets max height. --- src/web/HTMLOperation.js | 2 +- src/web/stylesheets/utils/_overrides.css | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/web/HTMLOperation.js b/src/web/HTMLOperation.js index ea0d4397..499faab5 100755 --- a/src/web/HTMLOperation.js +++ b/src/web/HTMLOperation.js @@ -51,7 +51,7 @@ HTMLOperation.prototype.toStubHtml = function(removeIcon) { if (this.description) { html += " data-container='body' data-toggle='popover' data-placement='auto right'\ - data-content=\"" + this.description + "\" data-html='true' data-trigger='hover'"; + data-content=\"" + this.description + "\" data-html='true' data-trigger='focus' tabindex='0'"; } html += ">" + this.name; diff --git a/src/web/stylesheets/utils/_overrides.css b/src/web/stylesheets/utils/_overrides.css index 8957071a..e060db7d 100755 --- a/src/web/stylesheets/utils/_overrides.css +++ b/src/web/stylesheets/utils/_overrides.css @@ -140,6 +140,8 @@ optgroup { .popover { background-color: var(--popover-background); border-color: var(--popover-border-colour); + max-height: 100%; + overflow-y: auto; } From de80db73f26251ea28b72de2e32ed82cb2239726 Mon Sep 17 00:00:00 2001 From: Matt C Date: Tue, 25 Jul 2017 16:27:59 +0100 Subject: [PATCH 03/78] Adds initial JPath functionality --- package.json | 1 + src/core/config/Categories.js | 2 ++ src/core/config/OperationConfig.js | 18 +++++++++++++++ src/core/operations/Code.js | 36 ++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/package.json b/package.json index 208789bd..f1de5a5f 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,7 @@ "lodash": "^4.17.4", "moment": "^2.17.1", "moment-timezone": "^0.5.11", + "node-jpath": "^2.1.0", "sladex-blowfish": "^0.8.1", "sortablejs": "^1.5.1", "split.js": "^1.2.0", diff --git a/src/core/config/Categories.js b/src/core/config/Categories.js index ce46d221..b3e40b33 100755 --- a/src/core/config/Categories.js +++ b/src/core/config/Categories.js @@ -215,6 +215,7 @@ const Categories = [ "Extract dates", "Regular expression", "XPath expression", + "JPath expression", "CSS selector", "Extract EXIF", ] @@ -278,6 +279,7 @@ const Categories = [ "CSS Beautify", "CSS Minify", "XPath expression", + "JPath expression", "CSS selector", "Strip HTML tags", "Diff", diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index b0c005b3..c058d9e2 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -2243,6 +2243,24 @@ const OperationConfig = { } ] }, + "JPath expression": { + description: "Extract information from a JSON object with an JPath query", + run: Code.runJpath, + inputType: "string", + outputType: "string", + args: [ + { + name: "JPath", + type: "string", + value: Code.JPATH_INITIAL + }, + { + name: "Result delimiter", + type: "binaryShortString", + value: Code.JPATH_DELIMITER + } + ] + }, "CSS selector": { description: "Extract information from an HTML document with a CSS selector", run: Code.runCSSQuery, diff --git a/src/core/operations/Code.js b/src/core/operations/Code.js index 9840797d..d3f5c0f0 100755 --- a/src/core/operations/Code.js +++ b/src/core/operations/Code.js @@ -4,6 +4,7 @@ import Utils from "../Utils.js"; import vkbeautify from "vkbeautify"; import {DOMParser as dom} from "xmldom"; import xpath from "xpath"; +import jpath from "node-jpath"; import prettyPrintOne from "imports-loader?window=>global!exports-loader?prettyPrintOne!google-code-prettify/bin/prettify.min.js"; @@ -354,6 +355,41 @@ const Code = { return nodes.map(nodeToString).join(delimiter); }, + /** + * @constant + * @default + */ + JPATH_INITIAL: "", + + /** + * @constant + * @default + */ + JPATH_DELIMITER: "\\n", + + /** + * XPath expression operation. + * + * @author Matt C (matt@artemisbot.uk) + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + runJpath: function(input, args) { + let query = args[0], + delimiter = args[1]; + + let obj; + try { + obj = JSON.parse(input); + } catch (err) { + return "Invalid input JSON."; + } + + let results = jpath.filter(obj, query); + return results.map(result => JSON.stringify(result)).join(delimiter); + }, + /** * @constant From e0905255bae2c2c032e1afa430108b383649fe77 Mon Sep 17 00:00:00 2001 From: Matt C Date: Tue, 25 Jul 2017 16:36:01 +0100 Subject: [PATCH 04/78] Forgot about package-lock.json oops --- package-lock.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package-lock.json b/package-lock.json index c8e87320..42a33aa4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4687,6 +4687,11 @@ "lower-case": "1.1.4" } }, + "node-jpath": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/node-jpath/-/node-jpath-2.1.0.tgz", + "integrity": "sha1-IYckZgObw5adyzhjq0JnH2H/UbI=" + }, "node-libs-browser": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.0.0.tgz", From 33ecbfa95b96dbce1e815c08ad0db9a9354f7c5f Mon Sep 17 00:00:00 2001 From: Matt C Date: Fri, 28 Jul 2017 21:47:47 +0100 Subject: [PATCH 05/78] Fixed arrow issue --- src/web/stylesheets/utils/_overrides.css | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/web/stylesheets/utils/_overrides.css b/src/web/stylesheets/utils/_overrides.css index e060db7d..36069cd5 100755 --- a/src/web/stylesheets/utils/_overrides.css +++ b/src/web/stylesheets/utils/_overrides.css @@ -140,10 +140,12 @@ optgroup { .popover { background-color: var(--popover-background); border-color: var(--popover-border-colour); - max-height: 100%; - overflow-y: auto; } +.popover-content { + max-height: 90vh; + overflow-y: auto; +} .popover.right>.arrow { border-right-color: var(--popover-border-colour); From 9ee0964d0e223689bb462844b5a244bbab20f8b8 Mon Sep 17 00:00:00 2001 From: Matt C Date: Sat, 29 Jul 2017 00:45:41 +0100 Subject: [PATCH 06/78] Fixed hover issue - now allows scrolling --- src/web/HTMLOperation.js | 4 ++-- src/web/OperationsWaiter.js | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/web/HTMLOperation.js b/src/web/HTMLOperation.js index 499faab5..aae9f282 100755 --- a/src/web/HTMLOperation.js +++ b/src/web/HTMLOperation.js @@ -50,8 +50,8 @@ HTMLOperation.prototype.toStubHtml = function(removeIcon) { let html = "
  • Date: Tue, 1 Aug 2017 14:42:09 +0000 Subject: [PATCH 07/78] Dependencies in the node version are now kept external in the webpack build --- Gruntfile.js | 3 +++ package-lock.json | 14 ++++++++------ package.json | 5 +++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index 42cd5923..a025e940 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -1,6 +1,7 @@ const webpack = require("webpack"); const ExtractTextPlugin = require("extract-text-webpack-plugin"); const HtmlWebpackPlugin = require("html-webpack-plugin"); +const NodeExternals = require("webpack-node-externals"); const Inliner = require("web-resource-inliner"); module.exports = function (grunt) { @@ -294,6 +295,7 @@ module.exports = function (grunt) { tests: { target: "node", entry: "./test/index.js", + externals: [NodeExternals()], output: { filename: "index.js", path: __dirname + "/build/test" @@ -302,6 +304,7 @@ module.exports = function (grunt) { node: { target: "node", entry: "./src/node/index.js", + externals: [NodeExternals()], output: { filename: "CyberChef.js", path: __dirname + "/build/node", diff --git a/package-lock.json b/package-lock.json index c8e87320..8533827a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -865,7 +865,6 @@ "version": "6.23.0", "resolved": "https://registry.npmjs.org/babel-polyfill/-/babel-polyfill-6.23.0.tgz", "integrity": "sha1-g2TKYt+Or7gwSZ9pkXdGbDsDSZ0=", - "dev": true, "requires": { "babel-runtime": "6.23.0", "core-js": "2.4.1", @@ -929,7 +928,6 @@ "version": "6.23.0", "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.23.0.tgz", "integrity": "sha1-CpSJ8UTecO+zzkMArM2zKeL8VDs=", - "dev": true, "requires": { "core-js": "2.4.1", "regenerator-runtime": "0.10.5" @@ -1557,8 +1555,7 @@ "core-js": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.4.1.tgz", - "integrity": "sha1-TekR5mew6ukSTjQlS1OupvxhjT4=", - "dev": true + "integrity": "sha1-TekR5mew6ukSTjQlS1OupvxhjT4=" }, "core-util-is": { "version": "1.0.2", @@ -6208,8 +6205,7 @@ "regenerator-runtime": { "version": "0.10.5", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz", - "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=", - "dev": true + "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=" }, "regenerator-transform": { "version": "0.9.11", @@ -7438,6 +7434,12 @@ } } }, + "webpack-node-externals": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.6.0.tgz", + "integrity": "sha1-Iyxi7GCSsQBjWj0p2DwXRxKN+b0=", + "dev": true + }, "webpack-sources": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.0.1.tgz", diff --git a/package.json b/package.json index 208789bd..89729ed2 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,6 @@ "devDependencies": { "babel-core": "^6.24.0", "babel-loader": "^7.1.1", - "babel-polyfill": "^6.23.0", "babel-preset-env": "^1.6.0", "css-loader": "^0.28.4", "exports-loader": "^0.6.4", @@ -60,11 +59,13 @@ "style-loader": "^0.18.2", "url-loader": "^0.5.8", "web-resource-inliner": "^4.1.0", - "webpack": "^3.3.0" + "webpack": "^3.3.0", + "webpack-node-externals": "^1.6.0" }, "dependencies": { "bootstrap": "^3.3.7", "bootstrap-colorpicker": "^2.5.1", + "babel-polyfill": "^6.23.0", "bootstrap-switch": "^3.3.4", "crypto-api": "^0.6.2", "crypto-js": "^3.1.9-1", From 1435acdc28be451b175117cb8b2034806db635f8 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Aug 2017 14:42:52 +0000 Subject: [PATCH 08/78] 5.12.3 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7fe6de2a..d059c100 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "5.12.2", + "version": "5.12.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 90f81531..fd08e668 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "5.12.2", + "version": "5.12.3", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From e8e5eb9c53993521e886b56ad31bfeac7eda3f87 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Aug 2017 19:23:30 +0000 Subject: [PATCH 09/78] Fixed some edge cases for popover triggering --- src/web/HTMLOperation.js | 4 ++-- src/web/OperationsWaiter.js | 23 ++++++++++++++++++----- src/web/RecipeWaiter.js | 5 ++--- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/web/HTMLOperation.js b/src/web/HTMLOperation.js index aae9f282..ea0d4397 100755 --- a/src/web/HTMLOperation.js +++ b/src/web/HTMLOperation.js @@ -50,8 +50,8 @@ HTMLOperation.prototype.toStubHtml = function(removeIcon) { let html = "
  • Date: Tue, 1 Aug 2017 19:27:24 +0000 Subject: [PATCH 10/78] 5.12.4 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index d059c100..66bbd0c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "5.12.3", + "version": "5.12.4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index fd08e668..0c2c2c60 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "5.12.3", + "version": "5.12.4", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From ab1c9e27dc1f646be04810eb447bdd14265d0705 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 3 Aug 2017 10:57:54 +0000 Subject: [PATCH 11/78] Added more loading messages --- src/web/html/index.html | 15 ++++++++++----- src/web/stylesheets/preloader.css | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/web/html/index.html b/src/web/html/index.html index 34c0958a..294b41cc 100755 --- a/src/web/html/index.html +++ b/src/web/html/index.html @@ -48,10 +48,15 @@ "Initialising Skynet...", "[REDACTED]", "Downloading more RAM...", - "Loading more loading messages...", "Ordering 1s and 0s...", "Navigating neural network...", - "Importing machine learning..." + "Importing machine learning...", + "Issuing Alice and Bob one-time pads...", + "Mining bitcoin cash...", + "Generating key material by trying to escape vim...", + "for i in range(additional): Pylon()", + "(creating unresolved tension...", + "Symlinking emacs and vim to ed...", ]; // Shuffle array using Durstenfeld algorithm @@ -62,19 +67,19 @@ loadingMsgs[j] = temp; } - // Show next loading message then move it to the end of the array + // Show next loading message and move it to the end of the array function changeLoadingMsg() { const msg = loadingMsgs.shift(); + loadingMsgs.push(msg); try { const el = document.getElementById("preloader-msg"); el.className = "loading"; // Causes CSS transition on first message el.innerHTML = msg; } catch (err) {} // Ignore errors if DOM not yet ready - loadingMsgs.push(msg); } changeLoadingMsg(); - window.loadingMsgsInt = setInterval(changeLoadingMsg, (Math.random() * 1000) + 1000); + window.loadingMsgsInt = setInterval(changeLoadingMsg, (Math.random() * 2000) + 1500); <% if (!htmlWebpackPlugin.options.inline) { %>