Merge branch 'dynamic-import' into node-lib-dynamic

CHANGELOG.md

@@ -2,7 +2,10 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
 
 
-### [8.24.0] - 2019-01-18
+### [8.24.0] - 2019-02-08
+- 'DNS over HTTPS' operation added [@h345983745] | [#489]
+
+### [8.23.1] - 2019-01-18
 - 'Convert co-ordinate format' operation added [@j433866] | [#476]
 
 ### [8.23.0] - 2019-01-18
@@ -104,6 +107,7 @@ All major and minor version changes will be documented in this file. Details of
 
 
 [8.24.0]: https://github.com/gchq/CyberChef/releases/tag/v8.24.0
+[8.23.1]: https://github.com/gchq/CyberChef/releases/tag/v8.23.1
 [8.23.0]: https://github.com/gchq/CyberChef/releases/tag/v8.23.0
 [8.22.0]: https://github.com/gchq/CyberChef/releases/tag/v8.22.0
 [8.21.0]: https://github.com/gchq/CyberChef/releases/tag/v8.21.0
@@ -137,6 +141,7 @@ All major and minor version changes will be documented in this file. Details of
 [@d98762625]: https://github.com/d98762625
 [@j433866]: https://github.com/j433866
 [@GCHQ77703]: https://github.com/GCHQ77703
+[@h345983745]: https://github.com/h345983745
 [@artemisbot]: https://github.com/artemisbot
 [@picapi]: https://github.com/picapi
 [@Dachande663]: https://github.com/Dachande663
@@ -186,3 +191,4 @@ All major and minor version changes will be documented in this file. Details of
 [#467]: https://github.com/gchq/CyberChef/pull/467
 [#468]: https://github.com/gchq/CyberChef/pull/468
 [#476]: https://github.com/gchq/CyberChef/pull/476
+[#489]: https://github.com/gchq/CyberChef/pull/489

Gruntfile.js

@@ -203,7 +203,8 @@ module.exports = function (grunt) {
                         sitemap: "./src/web/static/sitemap.js"
                     }, moduleEntryPoints),
                     output: {
-                        path: __dirname + "/build/prod"
+                        path: __dirname + "/build/prod",
+                        globalObject: "this"
                     },
                     resolve: {
                         alias: {
@@ -379,6 +380,9 @@ module.exports = function (grunt) {
                             "./config/modules/OpModules": "./config/modules/Default"
                         }
                     },
+                    output: {
+                        globalObject: "this",
+                    },
                     plugins: [
                         new webpack.DefinePlugin(BUILD_CONSTANTS),
                         new HtmlWebpackPlugin({

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.23.0",
+  "version": "8.24.1",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -771,6 +771,22 @@
         "semver": "^5.3.0"
       }
     },
+    "@babel/runtime-corejs2": {
+      "version": "7.3.1",
+      "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.3.1.tgz",
+      "integrity": "sha512-YpO13776h3e6Wy8dl2J8T9Qwlvopr+b4trCEhHE+yek6yIqV8sx6g3KozdHMbXeBpjosbPi+Ii5Z7X9oXFHUKA==",
+      "requires": {
+        "core-js": "^2.5.7",
+        "regenerator-runtime": "^0.12.0"
+      },
+      "dependencies": {
+        "regenerator-runtime": {
+          "version": "0.12.1",
+          "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.12.1.tgz",
+          "integrity": "sha512-odxIc1/vDlo4iZcfXqRYFj0vpXFNoGdKMAUieAlFYO6m/nl5e9KR/beGf41z4a1FI+aQgtjhuaSlDxQ0hmkrHg=="
+        }
+      }
+    },
     "@babel/template": {
       "version": "7.2.2",
       "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.2.2.tgz",
@@ -2322,9 +2338,9 @@
       "dev": true
     },
     "bignumber.js": {
-      "version": "8.0.1",
+      "version": "8.0.2",
-      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-8.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-8.0.2.tgz",
-      "integrity": "sha512-zAySveTJXkgLYCBi0b14xzfnOs+f3G6x36I8w2a1+PFQpWk/dp0mI0F+ZZK2bu+3ELewDcSyP+Cfq++NcHX7sg=="
+      "integrity": "sha512-EiuvFrnbv0jFixEQ9f58jo7X0qI2lNGIr/MxntmVzQc5JUweDSh8y8hbTCAomFtqwUPIOWcLXP0VEOSZTG7FFw=="
     },
     "binary-extensions": {
       "version": "1.12.0",
@@ -14339,9 +14355,12 @@
       "integrity": "sha512-fg03WRxtkCV6ohClePNAECYsmpKKTv5L8y/X3Dn1hQrec3POx2jHZ/0P2qQ6HvsrU1BmeqXcof3NGGueG6LxwQ=="
     },
     "xregexp": {
-      "version": "4.2.0",
+      "version": "4.2.4",
-      "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.2.4.tgz",
-      "integrity": "sha512-IyMa7SVe9FyT4WbQVW3b95mTLVceHhLEezQ02+QMvmIqDnKTxk0MLWIQPSW2MXAr1zQb+9yvwYhcyQULneh3wA=="
+      "integrity": "sha512-sO0bYdYeJAJBcJA8g7MJJX7UrOZIfJPd8U2SC7B2Dd/J24U0aQNoGp33shCaBSWeb0rD5rh6VBUIXOkGal1TZA==",
+      "requires": {
+        "@babel/runtime-corejs2": "^7.2.0"
+      }
     },
     "xtend": {
       "version": "4.0.1",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.23.0",
+  "version": "8.24.1",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -84,7 +84,7 @@
     "babel-plugin-transform-builtin-extend": "1.1.2",
     "babel-polyfill": "^6.26.0",
     "bcryptjs": "^2.4.3",
-    "bignumber.js": "^8.0.1",
+    "bignumber.js": "^8.0.2",
     "bootstrap-colorpicker": "^2.5.3",
     "bootstrap-material-design": "^4.1.1",
     "bson": "^4.0.1",
@@ -135,7 +135,7 @@
     "vkbeautify": "^0.99.3",
     "xmldom": "^0.1.27",
     "xpath": "0.0.27",
-    "xregexp": "^4.2.0",
+    "xregexp": "^4.2.4",
     "zlibjs": "^0.3.1"
   },
   "scripts": {

src/core/Dish.mjs

@@ -180,7 +180,7 @@ class Dish {
                     this.value = Array.prototype.slice.call(new Uint8Array(this.value));
                     break;
                 case Dish.BIG_NUMBER:
-                    this.value = this.value instanceof BigNumber ? Utils.strToByteArray(this.value.toFixed()) : [];
+                    this.value = BigNumber.isBigNumber(this.value) ? Utils.strToByteArray(this.value.toFixed()) : [];
                     break;
                 case Dish.JSON:
                     this.value = this.value ? Utils.strToByteArray(JSON.stringify(this.value, null, 4)) : [];
@@ -277,7 +277,7 @@ class Dish {
             case Dish.ARRAY_BUFFER:
                 return this.value instanceof ArrayBuffer;
             case Dish.BIG_NUMBER:
-                return this.value instanceof BigNumber;
+                return BigNumber.isBigNumber(this.value);
             case Dish.JSON:
                 // All values can be serialised in some manner, so we return true in all cases
                 return true;

src/core/Operation.mjs

@@ -23,6 +23,7 @@ class Operation {
         this._breakpoint      = false;
         this._disabled        = false;
         this._flowControl     = false;
+        this._manualBake      = false;
         this._ingList         = [];
 
         // Public fields
@@ -282,6 +283,7 @@ class Operation {
         return this._flowControl;
     }
 
+
     /**
      * Set whether this Operation is a flowcontrol op.
      *
@@ -291,6 +293,26 @@ class Operation {
         this._flowControl = !!value;
     }
 
+
+    /**
+     * Returns true if this Operation should not trigger AutoBake.
+     *
+     * @returns {boolean}
+     */
+    get manualBake() {
+        return this._manualBake;
+    }
+
+
+    /**
+     * Set whether this Operation should trigger AutoBake.
+     *
+     * @param {boolean} value
+     */
+    set manualBake(value) {
+        this._manualBake = !!value;
+    }
+
 }
 
 export default Operation;

src/core/Recipe.mjs

@@ -185,11 +185,8 @@ class Recipe  {
         log.debug(`[*] Executing recipe of ${this.opList.length} operations, starting at ${startFrom}`);
 
         for (let i = startFrom; i < this.opList.length; i++) {
-
             op = this.opList[i];
-
             log.debug(`[${i}] ${op.name} ${JSON.stringify(op.ingValues)}`);
-
             if (op.disabled) {
                 log.debug("Operation is disabled, skipping");
                 continue;

src/core/config/Categories.json

@@ -155,6 +155,7 @@
         "name": "Networking",
         "ops": [
             "HTTP request",
+            "DNS over HTTPS",
             "Strip HTTP headers",
             "Dechunk HTTP response",
             "Parse User Agent",

src/core/config/scripts/generateConfig.mjs

@@ -41,6 +41,7 @@ for (const opObj in Ops) {
         inputType:   op.inputType,
         outputType:  op.presentType,
         flowControl: op.flowControl,
+        manualBake:  op.manualBake,
         args:        op.args
     };
 

src/core/lib/ConvertCoordinates.mjs

@@ -72,7 +72,7 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
         if (inDelim === null) {
             throw new OperationError("Unable to detect the input delimiter automatically.");
         }
-    } else {
+    } else if (!inDelim.includes("Direction")) {
         // Convert the delimiter argument value to the actual character
         inDelim = realDelim(inDelim);
     }
@@ -89,7 +89,16 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
     outDelim = realDelim(outDelim);
 
     if (!NO_CHANGE.includes(inFormat)) {
+        if (inDelim.includes("Direction")) {
+            // Split on directions
+            split = input.split(/[NnEeSsWw]/g);
+            if (split[0] === "") {
+                // Remove first element if direction preceding
+                split = split.slice(1);
+            }
+        } else {
             split = input.split(inDelim);
+        }
         // Replace any co-ordinate symbols with spaces so we can split on them later
         for (let i = 0; i < split.length; i++) {
             split[i] = split[i].replace(/[°˝´'"]/g, " ");
@@ -196,7 +205,7 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
     if (inFormat.includes("Degrees")) {
         // If the input string contains directions, we need to check if they're S or W.
         // If either of the directions are, we should make the decimal value negative
-        const dirs = input.match(/[NnEeSsWw]/g);
+        const dirs = input.toUpperCase().match(/[NESW]/g);
         if (dirs && dirs.length >= 1) {
             // Make positive lat/lon values with S/W directions into negative values
             if (dirs[0] === "S" || dirs[0] === "W" && latlon.lat > 0) {

src/core/operations/DNSOverHTTPS.mjs

@@ -0,0 +1,125 @@
+/**
+ * @author h345983745
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+
+/**
+ * DNS over HTTPS operation
+ */
+class DNSOverHTTPS extends Operation {
+
+    /**
+     * DNSOverHTTPS constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "DNS over HTTPS";
+        this.module = "Default";
+        this.description = [
+            "Takes a single domain name and performs a DNS lookup using DNS over HTTPS.",
+            "<br><br>",
+            "By default, <a href='https://developers.cloudflare.com/1.1.1.1/dns-over-https/'>Cloudflare</a> and <a href='https://developers.google.com/speed/public-dns/docs/dns-over-https'>Google</a> DNS over HTTPS services are supported.",
+            "<br><br>",
+            "Can be used with any service that supports the GET parameters <code>name</code> and <code>type</code>."
+        ].join("\n");
+        this.infoURL = "https://wikipedia.org/wiki/DNS_over_HTTPS";
+        this.inputType = "string";
+        this.outputType = "JSON";
+        this.manualBake = true;
+        this.args = [
+            {
+                name: "Resolver",
+                type: "editableOption",
+                value: [
+                    {
+                        name: "Google",
+                        value: "https://dns.google.com/resolve"
+                    },
+                    {
+                        name: "Cloudflare",
+                        value: "https://cloudflare-dns.com/dns-query"
+                    }
+                ]
+            },
+            {
+                name: "Request Type",
+                type: "option",
+                value: [
+                    "A",
+                    "AAAA",
+                    "TXT",
+                    "MX",
+                    "DNSKEY",
+                    "NS"
+                ]
+            },
+            {
+                name: "Answer Data Only",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Validate DNSSEC",
+                type: "boolean",
+                value: true
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        const [resolver, requestType, justAnswer, DNSSEC] = args;
+        let url = URL;
+        try {
+            url = new URL(resolver);
+        } catch (error) {
+            throw new OperationError(error.toString() +
+            "\n\nThis error could be caused by one of the following:\n" +
+            " - An invalid Resolver URL\n");
+        }
+        const params = {name: input, type: requestType, cd: DNSSEC};
+
+        url.search = new URLSearchParams(params);
+
+        return fetch(url, {headers: {"accept": "application/dns-json"}}).then(response => {
+            return response.json();
+        }).then(data => {
+            if (justAnswer) {
+                return extractData(data.Answer);
+            }
+            return data;
+        }).catch(e => {
+            throw new OperationError(`Error making request to ${url}\n${e.toString()}`);
+        });
+
+    }
+}
+
+/**
+ * Construct an array of just data from a DNS Answer section
+ *
+ * @private
+ * @param {JSON} data
+ * @returns {JSON}
+ */
+function extractData(data) {
+    if (typeof(data) == "undefined"){
+        return [];
+    } else {
+        const dataValues = [];
+        data.forEach(element => {
+            dataValues.push(element.data);
+        });
+        return dataValues;
+    }
+}
+
+export default DNSOverHTTPS;

src/core/operations/Divide.mjs

@@ -43,7 +43,7 @@ class Divide extends Operation {
      */
     run(input, args) {
         const val = div(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/Mean.mjs

@@ -43,7 +43,7 @@ class Mean extends Operation {
      */
     run(input, args) {
         const val = mean(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/Median.mjs

@@ -43,7 +43,7 @@ class Median extends Operation {
      */
     run(input, args) {
         const val = median(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/Multiply.mjs

@@ -44,7 +44,7 @@ class Multiply extends Operation {
      */
     run(input, args) {
         const val = multi(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/RegularExpression.mjs

@@ -240,7 +240,7 @@ function regexHighlight (input, regex, displayTotal) {
         if (groups.length) {
             title += "Groups:\n";
             for (let i = 0; i < groups.length; i++) {
-                title += `\t${i+1}: ${Utils.escapeHtml(groups[i])}\n`;
+                title += `\t${i+1}: ${Utils.escapeHtml(groups[i] || "")}\n`;
             }
         }
 

src/core/operations/StandardDeviation.mjs

@@ -44,7 +44,7 @@ class StandardDeviation extends Operation {
      */
     run(input, args) {
         const val = stdDev(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
 
     }
 

src/core/operations/Subtract.mjs

@@ -44,7 +44,7 @@ class Subtract extends Operation {
      */
     run(input, args) {
         const val = sub(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/Sum.mjs

@@ -44,7 +44,7 @@ class Sum extends Operation {
      */
     run(input, args) {
         const val = sum(createNumArray(input, args[0]));
-        return val instanceof BigNumber ? val : new BigNumber(NaN);
+        return BigNumber.isBigNumber(val) ? val : new BigNumber(NaN);
     }
 
 }

src/core/operations/ToTable.mjs

@@ -57,7 +57,7 @@ class ToTable extends Operation {
         const [cellDelims, rowDelims, firstRowHeader, format] = args;
 
         // Process the input into a nested array of elements.
-        const tableData = Utils.parseCSV(input, cellDelims.split(""), rowDelims.split(""));
+        const tableData = Utils.parseCSV(Utils.escapeHtml(input), cellDelims.split(""), rowDelims.split(""));
 
         if (!tableData.length) return "";
 

src/web/HTMLIngredient.mjs

@@ -4,6 +4,8 @@
  * @license Apache-2.0
  */
 
+import Utils from "../core/Utils";
+
 /**
  * Object to handle the creation of operation ingredients.
  */
@@ -156,7 +158,7 @@ class HTMLIngredient {
                     } else if ((m = this.value[i].name.match(/\[\/([a-z0-9 -()^]+)\]/i))) {
                         html += "</optgroup>";
                     } else {
-                        html += `<option populate-value="${this.value[i].value}">${this.value[i].name}</option>`;
+                        html += `<option populate-value="${Utils.escapeHtml(this.value[i].value)}">${this.value[i].name}</option>`;
                     }
                 }
                 html += `</select>

src/web/InputWaiter.mjs

@@ -243,14 +243,22 @@ class InputWaiter {
         }
 
         if (file) {
-            this.closeFile();
+            this.loadFile(file);
-            this.loaderWorker = new LoaderWorker();
-            this.loaderWorker.addEventListener("message", this.handleLoaderMessage.bind(this));
-            this.loaderWorker.postMessage({"file": file});
-            this.set(file);
         }
     }
 
+    /**
+     * Handler for open input button events
+     * Loads the opened data into the input textarea
+     *
+     * @param {event} e
+     */
+    inputOpen(e) {
+        e.preventDefault();
+        const file = e.srcElement.files[0];
+        this.loadFile(file);
+    }
+
 
     /**
      * Handler for messages sent back by the LoaderWorker.
@@ -306,6 +314,22 @@ class InputWaiter {
     }
 
 
+    /**
+     * Loads a file into the input.
+     *
+     * @param {File} file
+     */
+    loadFile(file) {
+        if (file) {
+            this.closeFile();
+            this.loaderWorker = new LoaderWorker();
+            this.loaderWorker.addEventListener("message", this.handleLoaderMessage.bind(this));
+            this.loaderWorker.postMessage({"file": file});
+            this.set(file);
+        }
+    }
+
+
     /**
      * Handler for clear IO events.
      * Resets the input, output and info areas.

src/web/Manager.mjs

@@ -146,6 +146,7 @@ class Manager {
         this.addMultiEventListener("#input-text", "paste", this.input.inputPaste, this.input);
         document.getElementById("reset-layout").addEventListener("click", this.app.resetLayout.bind(this.app));
         document.getElementById("clr-io").addEventListener("click", this.input.clearIoClick.bind(this.input));
+        this.addListeners("#open-file", "change", this.input.inputOpen, this.input);
         this.addListeners("#input-text,#input-file", "dragover", this.input.inputDragover, this.input);
         this.addListeners("#input-text,#input-file", "dragleave", this.input.inputDragleave, this.input);
         this.addListeners("#input-text,#input-file", "drop", this.input.inputDrop, this.input);

src/web/OutputWaiter.mjs

@@ -478,7 +478,7 @@ class OutputWaiter {
      */
     showMagicButton(opSequence, result, recipeConfig) {
         const magicButton = document.getElementById("magic");
-        magicButton.setAttribute("data-original-title", `<i>${opSequence}</i> will produce <span class="data-text">"${Utils.truncate(result, 30)}"</span>`);
+        magicButton.setAttribute("data-original-title", `<i>${opSequence}</i> will produce <span class="data-text">"${Utils.escapeHtml(Utils.truncate(result), 30)}"</span>`);
         magicButton.setAttribute("data-recipe", JSON.stringify(recipeConfig), null, "");
         magicButton.classList.remove("hidden");
     }

src/web/RecipeWaiter.mjs

@@ -561,6 +561,8 @@ class RecipeWaiter {
             this.ingredientChildRuleID = null;
 
             // Find relevant rules in the stylesheet
+            // try/catch for chrome 64+ CORS error on cssRules.
+            try {
                 for (const i in document.styleSheets[0].cssRules) {
                     if (document.styleSheets[0].cssRules[i].selectorText === ".ingredients") {
                         this.ingredientRuleID = i;
@@ -569,12 +571,15 @@ class RecipeWaiter {
                         this.ingredientChildRuleID = i;
                     }
                 }
+            } catch (e) {
+                // Do nothing.
+            }
         }
 
         if (!this.ingredientRuleID || !this.ingredientChildRuleID) return;
 
-        const ingredientRule = document.styleSheets[0].cssRules[this.ingredientRuleID],
+        const ingredientRule = document.styleSheets[0].cssRules[this.ingredientRuleID];
-            ingredientChildRule = document.styleSheets[0].cssRules[this.ingredientChildRuleID];
+        const ingredientChildRule = document.styleSheets[0].cssRules[this.ingredientChildRuleID];
 
         if (recList.clientWidth < 450) {
             ingredientRule.style.gridTemplateColumns = "auto auto";

src/web/html/index.html

@@ -225,6 +225,10 @@
                         <div class="title no-select">
                             <label for="input-text">Input</label>
                             <span class="float-right">
+                                <button type="button" class="btn btn-primary bmd-btn-icon" id="btn-open-file" data-toggle="tooltip" title="Open file as input" onclick="document.getElementById('open-file').click();">
+                                    <i class="material-icons">input</i>
+                                    <input type="file" id="open-file" style="display: none">
+                                </button>
                                 <button type="button" class="btn btn-primary bmd-btn-icon" id="clr-io" data-toggle="tooltip" title="Clear input and output">
                                     <i class="material-icons">delete</i>
                                 </button>

webpack.config.js

@@ -56,24 +56,6 @@ module.exports = {
             jquery: "jquery/src/jquery",
         },
     },
-    // optimization: {
-    //     splitChunks: {
-    //         cacheGroups: {
-    //             styles: {
-    //                 name: "styles",
-    //                 test: /\.css$/,
-    //                 chunks: "all",
-    //                 enforce: true
-    //             },
-    //             vendor: {
-    //                 name: "vendor",
-    //                 test: /\.scss$/,
-    //                 chunks: "all",
-    //                 enforce: true,
-    //             },
-    //         }
-    //     }
-    // },
     module: {
         rules: [
             {