From 53c500eb1bd302cf37f0b4060ec92c697d97fa41 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 7 Nov 2018 13:23:05 +0000 Subject: [PATCH] Added various options to the 'Defang URL' operation. --- CHANGELOG.md | 5 ++ src/core/config/Categories.json | 6 +- src/core/lib/Extract.mjs | 18 ++++++ src/core/operations/DefangURL.mjs | 77 +++++++++++++++++++++++--- src/core/operations/ExtractDomains.mjs | 8 +-- src/core/operations/ExtractURLs.mjs | 14 +---- 6 files changed, 100 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cc4c553..6bfce1df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog All notable changes to CyberChef will be documented in this file. +### [8.9.0] - 2018-11-07 +- 'Defang URL' operation added [@arnydo] | [#394] + ### [8.8.0] - 2018-10-10 - 'Parse TLV' operation added [@GCHQ77703] | [#351] @@ -76,6 +79,7 @@ All notable changes to CyberChef will be documented in this file. [@JustAnotherMark]: https://github.com/JustAnotherMark [@sevzero]: https://github.com/sevzero [@PenguinGeorge]: https://github.com/PenguinGeorge +[@arnydo]: https://github.com/arnydo [#95]: https://github.com/gchq/CyberChef/pull/299 [#173]: https://github.com/gchq/CyberChef/pull/173 @@ -95,3 +99,4 @@ All notable changes to CyberChef will be documented in this file. [#344]: https://github.com/gchq/CyberChef/pull/344 [#348]: https://github.com/gchq/CyberChef/pull/348 [#351]: https://github.com/gchq/CyberChef/pull/351 +[#394]: https://github.com/gchq/CyberChef/pull/394 diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 85d397c5..36576b27 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -159,7 +159,8 @@ "Change IP format", "Group IP addresses", "Encode NetBIOS Name", - "Decode NetBIOS Name" + "Decode NetBIOS Name", + "Defang URL" ] }, { @@ -208,8 +209,7 @@ "Escape string", "Unescape string", "Pseudo-Random Number Generator", - "Sleep", - "Defang URL" + "Sleep" ] }, { diff --git a/src/core/lib/Extract.mjs b/src/core/lib/Extract.mjs index ba57d758..8b9f957e 100644 --- a/src/core/lib/Extract.mjs +++ b/src/core/lib/Extract.mjs @@ -39,3 +39,21 @@ export function search (input, searchRegex, removeRegex, includeTotal) { return output; } + + +/** + * URL regular expression + */ +const protocol = "[A-Z]+://", + hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+", + port = ":\\d+", + path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*" + + "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*"; + +export const URL_REGEX = new RegExp(protocol + hostname + "(?:" + port + ")?(?:" + path + ")?", "ig"); + + +/** + * Domain name regular expression + */ +export const DOMAIN_REGEX = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig; diff --git a/src/core/operations/DefangURL.mjs b/src/core/operations/DefangURL.mjs index 9917b47a..57d4298e 100644 --- a/src/core/operations/DefangURL.mjs +++ b/src/core/operations/DefangURL.mjs @@ -1,10 +1,12 @@ /** * @author arnydo [arnydo@protonmail.com] - * @copyright Crown Copyright 2016 + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2018 * @license Apache-2.0 */ import Operation from "../Operation"; +import {URL_REGEX, DOMAIN_REGEX} from "../lib/Extract"; /** * DefangURL operation @@ -18,12 +20,33 @@ class DefangURL extends Operation { super(); this.name = "Defang URL"; - this.module = "URL"; - this.description = "Takes a Universal Resource Locator (URL) and 'Defangs' it; meaning, the URL becomes invalid and neutralizes the risk of accidentally clicking on a malicious link.

This is often used when dealing with malicious links or IOCs.

Works well when combined with the 'Extract URLs' operation."; - this.infoURL = ""; + this.module = "Default"; + this.description = "Takes a Universal Resource Locator (URL) and 'Defangs' it; meaning the URL becomes invalid, neutralising the risk of accidentally clicking on a malicious link.

This is often used when dealing with malicious links or IOCs.

Works well when combined with the 'Extract URLs' operation."; + this.infoURL = "https://isc.sans.edu/forums/diary/Defang+all+the+things/22744/"; this.inputType = "string"; this.outputType = "string"; - this.args = []; + this.args = [ + { + name: "Escape dots", + type: "boolean", + value: true + }, + { + name: "Escape http", + type: "boolean", + value: true + }, + { + name: "Escape ://", + type: "boolean", + value: true + }, + { + name: "Process", + type: "option", + value: ["Valid domains and full URLs", "Only full URLs", "Everything"] + } + ]; } /** @@ -32,12 +55,48 @@ class DefangURL extends Operation { * @returns {string} */ run(input, args) { - let defang = input.replace(/http/gi, "hxxp"); - defang = defang.replace(/\./g, "[.]"); - defang = defang.replace(/:\/\//g, "[://]"); - return defang; + const [dots, http, slashes, process] = args; + + switch (process) { + case "Valid domains and full URLs": + input = input.replace(URL_REGEX, x => { + return defangURL(x, dots, http, slashes); + }); + input = input.replace(DOMAIN_REGEX, x => { + return defangURL(x, dots, http, slashes); + }); + break; + case "Only full URLs": + input = input.replace(URL_REGEX, x => { + return defangURL(x, dots, http, slashes); + }); + break; + case "Everything": + input = defangURL(input, dots, http, slashes); + break; + } + + return input; } } + +/** + * Defangs a given URL + * + * @param {string} url + * @param {boolean} dots + * @param {boolean} http + * @param {boolean} slashes + * @returns {string} + */ +function defangURL(url, dots, http, slashes) { + if (dots) url = url.replace(/\./g, "[.]"); + if (http) url = url.replace(/http/gi, "hxxp"); + if (slashes) url = url.replace(/:\/\//g, "[://]"); + + return url; +} + export default DefangURL; diff --git a/src/core/operations/ExtractDomains.mjs b/src/core/operations/ExtractDomains.mjs index 8eae8064..ddd7ca97 100644 --- a/src/core/operations/ExtractDomains.mjs +++ b/src/core/operations/ExtractDomains.mjs @@ -5,7 +5,7 @@ */ import Operation from "../Operation"; -import { search } from "../lib/Extract"; +import { search, DOMAIN_REGEX } from "../lib/Extract"; /** * Extract domains operation @@ -38,10 +38,8 @@ class ExtractDomains extends Operation { * @returns {string} */ run(input, args) { - const displayTotal = args[0], - regex = /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/ig; - - return search(input, regex, null, displayTotal); + const displayTotal = args[0]; + return search(input, DOMAIN_REGEX, null, displayTotal); } } diff --git a/src/core/operations/ExtractURLs.mjs b/src/core/operations/ExtractURLs.mjs index ab306d3f..5e76c6d9 100644 --- a/src/core/operations/ExtractURLs.mjs +++ b/src/core/operations/ExtractURLs.mjs @@ -5,7 +5,7 @@ */ import Operation from "../Operation"; -import { search } from "../lib/Extract"; +import { search, URL_REGEX } from "../lib/Extract"; /** * Extract URLs operation @@ -38,16 +38,8 @@ class ExtractURLs extends Operation { * @returns {string} */ run(input, args) { - const displayTotal = args[0], - protocol = "[A-Z]+://", - hostname = "[-\\w]+(?:\\.\\w[-\\w]*)+", - port = ":\\d+"; - let path = "/[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]*"; - - path += "(?:[.!,?]+[^.!,?\"<>\\[\\]{}\\s\\x7F-\\xFF]+)*"; - const regex = new RegExp(protocol + hostname + "(?:" + port + - ")?(?:" + path + ")?", "ig"); - return search(input, regex, null, displayTotal); + const displayTotal = args[0]; + return search(input, URL_REGEX, null, displayTotal); } }