diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5cc4c553..6bfce1df 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,9 @@
# Changelog
All notable changes to CyberChef will be documented in this file.
+### [8.9.0] - 2018-11-07
+- 'Defang URL' operation added [@arnydo] | [#394]
+
### [8.8.0] - 2018-10-10
- 'Parse TLV' operation added [@GCHQ77703] | [#351]
@@ -76,6 +79,7 @@ All notable changes to CyberChef will be documented in this file.
[@JustAnotherMark]: https://github.com/JustAnotherMark
[@sevzero]: https://github.com/sevzero
[@PenguinGeorge]: https://github.com/PenguinGeorge
+[@arnydo]: https://github.com/arnydo
[#95]: https://github.com/gchq/CyberChef/pull/299
[#173]: https://github.com/gchq/CyberChef/pull/173
@@ -95,3 +99,4 @@ All notable changes to CyberChef will be documented in this file.
[#344]: https://github.com/gchq/CyberChef/pull/344
[#348]: https://github.com/gchq/CyberChef/pull/348
[#351]: https://github.com/gchq/CyberChef/pull/351
+[#394]: https://github.com/gchq/CyberChef/pull/394
diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json
index 6f02abb8..36576b27 100755
--- a/src/core/config/Categories.json
+++ b/src/core/config/Categories.json
@@ -159,7 +159,8 @@
"Change IP format",
"Group IP addresses",
"Encode NetBIOS Name",
- "Decode NetBIOS Name"
+ "Decode NetBIOS Name",
+ "Defang URL"
]
},
{
diff --git a/src/core/lib/Extract.mjs b/src/core/lib/Extract.mjs
index ba57d758..8b9f957e 100644
--- a/src/core/lib/Extract.mjs
+++ b/src/core/lib/Extract.mjs
@@ -39,3 +39,21 @@ export function search (input, searchRegex, removeRegex, includeTotal) {
return output;
}
+
+
+/**
+ * URL regular expression
+ */
+const protocol = "[A-Z]+://",
+ hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+",
+ port = ":\\d+",
+ path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*" +
+ "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*";
+
+export const URL_REGEX = new RegExp(protocol + hostname + "(?:" + port + ")?(?:" + path + ")?", "ig");
+
+
+/**
+ * Domain name regular expression
+ */
+export const DOMAIN_REGEX = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig;
diff --git a/src/core/operations/DefangURL.mjs b/src/core/operations/DefangURL.mjs
new file mode 100644
index 00000000..57d4298e
--- /dev/null
+++ b/src/core/operations/DefangURL.mjs
@@ -0,0 +1,102 @@
+/**
+ * @author arnydo [arnydo@protonmail.com]
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import {URL_REGEX, DOMAIN_REGEX} from "../lib/Extract";
+
+/**
+ * DefangURL operation
+ */
+class DefangURL extends Operation {
+
+ /**
+ * DefangURL constructor
+ */
+ constructor() {
+ super();
+
+ this.name = "Defang URL";
+ this.module = "Default";
+ this.description = "Takes a Universal Resource Locator (URL) and 'Defangs' it; meaning the URL becomes invalid, neutralising the risk of accidentally clicking on a malicious link.
This is often used when dealing with malicious links or IOCs.
Works well when combined with the 'Extract URLs' operation.";
+ this.infoURL = "https://isc.sans.edu/forums/diary/Defang+all+the+things/22744/";
+ this.inputType = "string";
+ this.outputType = "string";
+ this.args = [
+ {
+ name: "Escape dots",
+ type: "boolean",
+ value: true
+ },
+ {
+ name: "Escape http",
+ type: "boolean",
+ value: true
+ },
+ {
+ name: "Escape ://",
+ type: "boolean",
+ value: true
+ },
+ {
+ name: "Process",
+ type: "option",
+ value: ["Valid domains and full URLs", "Only full URLs", "Everything"]
+ }
+ ];
+ }
+
+ /**
+ * @param {string} input
+ * @param {Object[]} args
+ * @returns {string}
+ */
+ run(input, args) {
+ const [dots, http, slashes, process] = args;
+
+ switch (process) {
+ case "Valid domains and full URLs":
+ input = input.replace(URL_REGEX, x => {
+ return defangURL(x, dots, http, slashes);
+ });
+ input = input.replace(DOMAIN_REGEX, x => {
+ return defangURL(x, dots, http, slashes);
+ });
+ break;
+ case "Only full URLs":
+ input = input.replace(URL_REGEX, x => {
+ return defangURL(x, dots, http, slashes);
+ });
+ break;
+ case "Everything":
+ input = defangURL(input, dots, http, slashes);
+ break;
+ }
+
+ return input;
+ }
+
+}
+
+
+/**
+ * Defangs a given URL
+ *
+ * @param {string} url
+ * @param {boolean} dots
+ * @param {boolean} http
+ * @param {boolean} slashes
+ * @returns {string}
+ */
+function defangURL(url, dots, http, slashes) {
+ if (dots) url = url.replace(/\./g, "[.]");
+ if (http) url = url.replace(/http/gi, "hxxp");
+ if (slashes) url = url.replace(/:\/\//g, "[://]");
+
+ return url;
+}
+
+export default DefangURL;
diff --git a/src/core/operations/ExtractDomains.mjs b/src/core/operations/ExtractDomains.mjs
index 8eae8064..ddd7ca97 100644
--- a/src/core/operations/ExtractDomains.mjs
+++ b/src/core/operations/ExtractDomains.mjs
@@ -5,7 +5,7 @@
*/
import Operation from "../Operation";
-import { search } from "../lib/Extract";
+import { search, DOMAIN_REGEX } from "../lib/Extract";
/**
* Extract domains operation
@@ -38,10 +38,8 @@ class ExtractDomains extends Operation {
* @returns {string}
*/
run(input, args) {
- const displayTotal = args[0],
- regex = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig;
-
- return search(input, regex, null, displayTotal);
+ const displayTotal = args[0];
+ return search(input, DOMAIN_REGEX, null, displayTotal);
}
}
diff --git a/src/core/operations/ExtractURLs.mjs b/src/core/operations/ExtractURLs.mjs
index ab306d3f..5e76c6d9 100644
--- a/src/core/operations/ExtractURLs.mjs
+++ b/src/core/operations/ExtractURLs.mjs
@@ -5,7 +5,7 @@
*/
import Operation from "../Operation";
-import { search } from "../lib/Extract";
+import { search, URL_REGEX } from "../lib/Extract";
/**
* Extract URLs operation
@@ -38,16 +38,8 @@ class ExtractURLs extends Operation {
* @returns {string}
*/
run(input, args) {
- const displayTotal = args[0],
- protocol = "[A-Z]+://",
- hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+",
- port = ":\\d+";
- let path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*";
-
- path += "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*";
- const regex = new RegExp(protocol + hostname + "(?:" + port +
- ")?(?:" + path + ")?", "ig");
- return search(input, regex, null, displayTotal);
+ const displayTotal = args[0];
+ return search(input, URL_REGEX, null, displayTotal);
}
}