Merge branch 'artemisbot-features/yara'

CHANGELOG.md

@@ -2,6 +2,9 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
 
 
+### [8.23.0] - 2019-01-18
+- 'YARA Rules' operatio added [@artemisbot] | [#468]
+
 ### [8.22.0] - 2019-01-10
 - 'Subsection' operation added [@j433866] | [#467]
 
@@ -97,6 +100,7 @@ All major and minor version changes will be documented in this file. Details of
 
 
 
+[8.23.0]: https://github.com/gchq/CyberChef/releases/tag/v8.23.0
 [8.22.0]: https://github.com/gchq/CyberChef/releases/tag/v8.22.0
 [8.21.0]: https://github.com/gchq/CyberChef/releases/tag/v8.21.0
 [8.20.0]: https://github.com/gchq/CyberChef/releases/tag/v8.20.0
@@ -176,3 +180,4 @@ All major and minor version changes will be documented in this file. Details of
 [#458]: https://github.com/gchq/CyberChef/pull/458
 [#461]: https://github.com/gchq/CyberChef/pull/461
 [#467]: https://github.com/gchq/CyberChef/pull/467
+[#468]: https://github.com/gchq/CyberChef/pull/468

package-lock.json

@@ -5077,7 +5077,8 @@
         "ansi-regex": {
           "version": "2.1.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -5120,7 +5121,8 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
@@ -5131,7 +5133,8 @@
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -5248,7 +5251,8 @@
         "inherits": {
           "version": "2.0.3",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -5260,6 +5264,7 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -5282,12 +5287,14 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "minipass": {
           "version": "2.2.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.1",
             "yallist": "^3.0.0"
@@ -5306,6 +5313,7 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -5386,7 +5394,8 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -5398,6 +5407,7 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -5483,7 +5493,8 @@
         "safe-buffer": {
           "version": "5.1.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -5519,6 +5530,7 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -5538,6 +5550,7 @@
           "version": "3.0.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -5581,12 +5594,14 @@
         "wrappy": {
           "version": "1.0.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "yallist": {
           "version": "3.0.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         }
       }
     },
@@ -7759,6 +7774,11 @@
       "resolved": "https://registry.npmjs.org/lex-parser/-/lex-parser-0.1.4.tgz",
       "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA="
     },
+    "libyara-wasm": {
+      "version": "0.0.11",
+      "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.11.tgz",
+      "integrity": "sha512-rglapPFo0IHPNksWYQXI8oqftXYj5mOGOf4BXtbSySVRX71pro4BehNjJ5qEpjYx+roGvNkcAD9zCsitA08sxw=="
+    },
     "livereload-js": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/livereload-js/-/livereload-js-2.4.0.tgz",
@@ -9760,7 +9780,7 @@
       "dependencies": {
         "async": {
           "version": "1.5.2",
-          "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
+          "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz",
           "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
           "dev": true
         }

package.json

@@ -106,6 +106,7 @@
     "jsqr": "^1.1.1",
     "jsrsasign": "8.0.12",
     "kbpgp": "^2.0.82",
+    "libyara-wasm": "0.0.11",
     "lodash": "^4.17.11",
     "loglevel": "^1.6.1",
     "loglevel-message-prefix": "^3.0.0",

src/core/Ingredient.mjs

@@ -23,6 +23,7 @@ class Ingredient {
         this._value = null;
         this.disabled = false;
         this.hint = "";
+        this.rows = 0;
         this.toggleValues = [];
         this.target = null;
         this.defaultIndex = 0;
@@ -45,6 +46,7 @@ class Ingredient {
         this.defaultValue = ingredientConfig.value;
         this.disabled = !!ingredientConfig.disabled;
         this.hint = ingredientConfig.hint || false;
+        this.rows = ingredientConfig.rows || false;
         this.toggleValues = ingredientConfig.toggleValues;
         this.target = typeof ingredientConfig.target !== "undefined" ? ingredientConfig.target : null;
         this.defaultIndex = typeof ingredientConfig.defaultIndex !== "undefined" ? ingredientConfig.defaultIndex : 0;

src/core/Operation.mjs

@@ -179,6 +179,7 @@ class Operation {
 
             if (ing.toggleValues) conf.toggleValues = ing.toggleValues;
             if (ing.hint) conf.hint = ing.hint;
+            if (ing.rows) conf.rows = ing.rows;
             if (ing.disabled) conf.disabled = ing.disabled;
             if (ing.target) conf.target = ing.target;
             if (ing.defaultIndex) conf.defaultIndex = ing.defaultIndex;

src/core/operations/YARARules.mjs

@@ -0,0 +1,122 @@
+/**
+ * @author Matt C [matt@artemisbot.uk]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Yara from "libyara-wasm";
+
+/**
+ * YARA Rules operation
+ */
+class YARARules extends Operation {
+
+    /**
+     * YARARules constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "YARA Rules";
+        this.module = "Yara";
+        this.description = "YARA is a tool developed at VirusTotal, primarily aimed at helping malware researchers to identify and classify malware samples. It matches based on rules specified by the user containing textual or binary patterns and a boolean expression. For help on writing rules, see the <a href='https://yara.readthedocs.io/en/latest/writingrules.html'>YARA documentation.</a>";
+        this.infoURL = "https://wikipedia.org/wiki/YARA";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "string";
+        this.args = [
+            {
+                name: "Rules",
+                type: "text",
+                value: "",
+                rows: 5
+            },
+            {
+                name: "Show strings",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Show string lengths",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Show metadata",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Show counts",
+                type: "boolean",
+                value: true
+            }
+        ];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        if (ENVIRONMENT_IS_WORKER())
+            self.sendStatusMessage("Instantiating YARA...");
+        const [rules, showStrings, showLengths, showMeta, showCounts] = args;
+        return new Promise((resolve, reject) => {
+            Yara().then(yara => {
+                if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Converting data for YARA.");
+                let matchString = "";
+
+                const inpArr = new Uint8Array(input); // Turns out embind knows that JS uint8array <==> C++ std::string
+
+                if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Running YARA matching.");
+
+                const resp = yara.run(inpArr, rules);
+
+                if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Processing data.");
+
+                if (resp.compileErrors.size() > 0) {
+                    for (let i = 0; i < resp.compileErrors.size(); i++) {
+                        const compileError = resp.compileErrors.get(i);
+                        if (!compileError.warning) {
+                            reject(new OperationError(`Error on line ${compileError.lineNumber}: ${compileError.message}`));
+                        } else {
+                            matchString += `Warning on line ${compileError.lineNumber}: ${compileError.message}`;
+                        }
+                    }
+                }
+                const matchedRules = resp.matchedRules;
+                for (let i = 0; i < matchedRules.size(); i++) {
+                    const rule = matchedRules.get(i);
+                    const matches = rule.resolvedMatches;
+                    let meta = "";
+                    if (showMeta && rule.metadata.size() > 0) {
+                        meta += " [";
+                        for (let j = 0; j < rule.metadata.size(); j++) {
+                            meta += `${rule.metadata.get(j).identifier}: ${rule.metadata.get(j).data}, `;
+                        }
+                        meta = meta.slice(0, -2) + "]";
+                    }
+                    const countString = showCounts ? `${matches.size()} time${matches.size() > 1 ? "s" : ""}` : "";
+                    if (matches.size() === 0 || !(showStrings || showLengths)) {
+                        matchString += `Input matches rule "${rule.ruleName}"${meta}${countString.length > 0 ? ` ${countString}`: ""}.\n`;
+                    } else {
+                        matchString += `Rule "${rule.ruleName}"${meta} matches (${countString}):\n`;
+                        for (let j = 0; j < matches.size(); j++) {
+                            const match = matches.get(j);
+                            if (showStrings || showLengths) {
+                                matchString += `Pos ${match.location}, ${showLengths ? `length ${match.matchLength}, ` : ""}identifier ${match.stringIdentifier}${showStrings ? `, data: "${match.data}"` : ""}\n`;
+                            }
+                        }
+                    }
+                }
+                resolve(matchString);
+            });
+        });
+    }
+
+}
+
+export default YARARules;

src/web/HTMLIngredient.mjs

@@ -25,6 +25,7 @@ class HTMLIngredient {
         this.value = config.value;
         this.disabled = config.disabled || false;
         this.hint = config.hint || false;
+        this.rows = config.rows || false;
         this.target = config.target;
         this.defaultIndex = config.defaultIndex || 0;
         this.toggleValues = config.toggleValues;
@@ -229,6 +230,7 @@ class HTMLIngredient {
                         class="form-control arg"
                         id="${this.id}"
                         arg-name="${this.name}"
+                        rows="${this.rows ? this.rows : 3}"
                         ${this.disabled ? "disabled" : ""}>${this.value}</textarea>
                     ${this.hint ? "<span class='bmd-help'>" + this.hint + "</span>" : ""}
                 </div>`;

src/web/Manager.mjs

@@ -137,6 +137,9 @@ class Manager {
         this.addDynamicListener("#rec-list li.operation > div", "dblclick", this.recipe.operationChildDblclick, this.recipe);
         this.addDynamicListener("#rec-list .dropdown-menu.toggle-dropdown a", "click", this.recipe.dropdownToggleClick, this.recipe);
         this.addDynamicListener("#rec-list", "operationremove", this.recipe.opRemove.bind(this.recipe));
+        this.addDynamicListener("textarea.arg", "dragover", this.recipe.textArgDragover, this.recipe);
+        this.addDynamicListener("textarea.arg", "dragleave", this.recipe.textArgDragLeave, this.recipe);
+        this.addDynamicListener("textarea.arg", "drop", this.recipe.textArgDrop, this.recipe);
 
         // Input
         this.addMultiEventListener("#input-text", "keyup", this.input.inputChange, this.input);

src/web/RecipeWaiter.mjs

@@ -376,6 +376,7 @@ class RecipeWaiter {
         }
     }
 
+
     /**
      * Adds the specified operation to the recipe.
      *
@@ -454,6 +455,75 @@ class RecipeWaiter {
     }
 
 
+    /**
+     * Handler for text argument dragover events.
+     * Gives the user a visual cue to show that items can be dropped here.
+     *
+     * @param {event} e
+     */
+    textArgDragover (e) {
+        // This will be set if we're dragging an operation
+        if (e.dataTransfer.effectAllowed === "move")
+            return false;
+
+        e.stopPropagation();
+        e.preventDefault();
+        e.target.closest("textarea.arg").classList.add("dropping-file");
+    }
+
+
+    /**
+     * Handler for text argument dragleave events.
+     * Removes the visual cue.
+     *
+     * @param {event} e
+     */
+    textArgDragLeave (e) {
+        e.stopPropagation();
+        e.preventDefault();
+        e.target.classList.remove("dropping-file");
+    }
+
+
+    /**
+     * Handler for text argument drop events.
+     * Loads the dragged data into the argument textarea.
+     *
+     * @param {event} e
+     */
+    textArgDrop(e) {
+        // This will be set if we're dragging an operation
+        if (e.dataTransfer.effectAllowed === "move")
+            return false;
+
+        e.stopPropagation();
+        e.preventDefault();
+        const targ = e.target;
+        const file = e.dataTransfer.files[0];
+        const text = e.dataTransfer.getData("Text");
+
+        targ.classList.remove("dropping-file");
+
+        if (text) {
+            targ.value = text;
+            return;
+        }
+
+        if (file) {
+            const reader = new FileReader();
+            const self = this;
+            reader.onload = function (e) {
+                targ.value = e.target.result;
+                // Trigger floating label move
+                const changeEvent = new Event('change');
+                targ.dispatchEvent(changeEvent);
+                window.dispatchEvent(self.manager.statechange);
+            };
+            reader.readAsText(file);
+        }
+    }
+
+
     /**
      * Sets register values.
      *
@@ -479,6 +549,7 @@ class RecipeWaiter {
         op.insertAdjacentHTML("beforeend", registerListEl);
     }
 
+
     /**
      * Adjusts the number of ingredient columns as the width of the recipe changes.
      */

src/web/stylesheets/layout/_io.css

@@ -123,6 +123,7 @@
 
 .dropping-file {
     border: 5px dashed var(--drop-file-border-colour) !important;
+    margin: -5px;
 }
 
 #stale-indicator {

tests/operations/index.mjs

@@ -83,6 +83,7 @@ import "./tests/Magic";
 import "./tests/ParseTLV";
 import "./tests/Media";
 import "./tests/ToFromInsensitiveRegex";
+import "./tests/YARA.mjs";
 
 // Cannot test operations that use the File type yet
 //import "./tests/SplitColourChannels";

tests/operations/tests/YARA.mjs

@@ -0,0 +1,24 @@
+/**
+ * YARA Rules tests.
+ *
+ * @author Matt C [matt@artemisbot.uk]
+ *
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "YARA Match: simple foobar",
+        input: "foobar foobar bar foo foobar",
+        expectedOutput: "Rule \"foo\" matches (4 times):\nPos 0, length 3, identifier $re1, data: \"foo\"\nPos 7, length 3, identifier $re1, data: \"foo\"\nPos 18, length 3, identifier $re1, data: \"foo\"\nPos 22, length 3, identifier $re1, data: \"foo\"\nRule \"bar\" matches (4 times):\nPos 3, length 3, identifier $re1, data: \"bar\"\nPos 10, length 3, identifier $re1, data: \"bar\"\nPos 14, length 3, identifier $re1, data: \"bar\"\nPos 25, length 3, identifier $re1, data: \"bar\"\n",
+        recipeConfig: [
+            {
+                "op": "YARA Rules",
+                "args": ["rule foo {strings: $re1 = /foo/ condition: $re1} rule bar {strings: $re1 = /bar/ condition: $re1}", true, true, true, true],
+            }
+        ],
+    },
+]);
+