Added dynamically generated sitemap and robots.txt file to prevent crawling of inline version

.travis.yml

@@ -11,6 +11,7 @@ script:
   - grunt node
   - grunt prod --msg="$COMPILE_MSG"
 before_deploy:
+  - grunt exec:sitemap
   - grunt copy:ghPages
 deploy:
   - provider: pages

Gruntfile.js

@@ -218,7 +218,8 @@ module.exports = function (grunt) {
             web: {
                 target: "web",
                 entry: Object.assign({
-                    main: "./src/web/index.js"
+                    main: "./src/web/index.js",
+                    sitemap: "./src/web/static/sitemap.js"
                 }, moduleEntryPoints),
                 output: {
                     path: __dirname + "/build/prod"
@@ -377,6 +378,10 @@ module.exports = function (grunt) {
                         expand: true,
                         src: "docs/**",
                         dest: "build/prod/"
+                    },
+                    {
+                        src: "src/web/static/robots.txt",
+                        dest: "build/prod/"
                     }
                 ]
             }
@@ -406,6 +411,9 @@ module.exports = function (grunt) {
             cleanGit: {
                 command: "git gc --prune=now --aggressive"
             },
+            sitemap: {
+                command: "node build/prod/sitemap.js > build/prod/sitemap.xml"
+            }
         },
         execute: {
             test: "build/test/index.js"

package-lock.json

@@ -9566,6 +9566,16 @@
       "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
       "dev": true
     },
+    "sitemap": {
+      "version": "1.13.0",
+      "resolved": "https://registry.npmjs.org/sitemap/-/sitemap-1.13.0.tgz",
+      "integrity": "sha1-Vpy+IYAgKSamKiZs094Jyc60P4M=",
+      "dev": true,
+      "requires": {
+        "underscore": "1.7.0",
+        "url-join": "1.1.0"
+      }
+    },
     "sladex-blowfish": {
       "version": "0.8.1",
       "resolved": "https://registry.npmjs.org/sladex-blowfish/-/sladex-blowfish-0.8.1.tgz",
@@ -10726,6 +10736,12 @@
         }
       }
     },
+    "url-join": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/url-join/-/url-join-1.1.0.tgz",
+      "integrity": "sha1-dBxsL0WWxIMNZxhGCSDQySIC3Hg=",
+      "dev": true
+    },
     "url-loader": {
       "version": "0.6.2",
       "resolved": "https://registry.npmjs.org/url-loader/-/url-loader-0.6.2.tgz",

package.json

@@ -58,6 +58,7 @@
     "postcss-css-variables": "^0.8.0",
     "postcss-import": "^11.0.0",
     "postcss-loader": "^2.0.10",
+    "sitemap": "^1.13.0",
     "style-loader": "^0.19.1",
     "url-loader": "^0.6.2",
     "val-loader": "^1.1.0",

src/web/App.js

@@ -377,6 +377,7 @@ App.prototype.loadURIParams = function() {
         window.location.href.split("#")[1] ||
         window.location.hash;
     this.uriParams = Utils.parseURIParams(params);
+    this.autoBakePause = true;
 
     // Read in recipe from URI params
     if (this.uriParams.recipe) {
@@ -387,35 +388,29 @@ App.prototype.loadURIParams = function() {
     } else if (this.uriParams.op) {
         // If there's no recipe, look for single operations
         this.manager.recipe.clearRecipe();
-        try {
-            this.manager.recipe.addOperation(this.uriParams.op);
-        } catch (err) {
-            // If no exact match, search for nearest match and add that
-            const matchedOps = this.manager.ops.filterOperations(this.uriParams.op, false);
-            if (matchedOps.length) {
-                this.manager.recipe.addOperation(matchedOps[0].name);
-            }
 
-            // Populate search with the string
-            const search = document.getElementById("search");
-
-            search.value = this.uriParams.op;
-            search.dispatchEvent(new Event("search"));
+        // Search for nearest match and add it
+        const matchedOps = this.manager.ops.filterOperations(this.uriParams.op, false);
+        if (matchedOps.length) {
+            this.manager.recipe.addOperation(matchedOps[0].name);
         }
+
+        // Populate search with the string
+        const search = document.getElementById("search");
+
+        search.value = this.uriParams.op;
+        search.dispatchEvent(new Event("search"));
     }
 
     // Read in input data from URI params
     if (this.uriParams.input) {
-        this.autoBakePause = true;
         try {
             const inputData = Utils.fromBase64(this.uriParams.input);
             this.setInput(inputData);
-        } catch (err) {
-        } finally {
-            this.autoBakePause = false;
-        }
+        } catch (err) {}
     }
 
+    this.autoBakePause = false;
     this.autoBake();
 };
 

src/web/static/robots.txt

@@ -0,0 +1,4 @@
+user-agent: *
+disallow: /cyberchef.htm
+
+sitemap: https://gchq.github.io/CyberChef/sitemap.xml

src/web/static/sitemap.js

@@ -0,0 +1,33 @@
+import sm from "sitemap";
+import OperationConfig from "../../core/config/MetaConfig.js";
+
+
+/**
+ * Generates an XML sitemap for all CyberChef operations and a number of recipes.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+const sitemap = sm.createSitemap({
+    hostname: "https://gchq.github.io/CyberChef",
+});
+
+sitemap.add({
+    url: "/",
+    changefreq: "weekly",
+    priority: 1.0
+});
+
+for (let op in OperationConfig) {
+    sitemap.add({
+        url: `/op=${encodeURIComponent(op)}`,
+        changeFreq: "yearly",
+        priority: 0.5
+    });
+}
+
+const xml = sitemap.toString();
+
+console.log(xml); // eslint-disable-line no-console