ESM: Ported StrUtils and NetBIOS operations.

This commit is contained in:
n1474335 2018-05-14 14:31:04 +00:00
parent a98d37e61c
commit 037e2f3771
15 changed files with 907 additions and 6 deletions

View File

@ -154,9 +154,9 @@ function loadRequiredModules(recipeConfig) {
const module = self.OperationConfig[op.op].module;
if (!OpModules.hasOwnProperty(module)) {
log.info("Loading module " + module);
self.sendStatusMessage("Loading module " + module);
self.importScripts(self.docURL + "/" + module + ".js");
log.info(`Loading ${module} module`);
self.sendStatusMessage(`Loading ${module} module`);
self.importScripts(`${self.docURL}/${module}.js`);
self.sendStatusMessage("");
}
});

View File

@ -12,6 +12,7 @@
import process from "process";
import fs from "fs";
import path from "path";
import EscapeString from "../../operations/EscapeString";
if (process.argv.length < 4) {
console.log("Pass an operation name and legacy filename as arguments.");
@ -58,6 +59,8 @@ function main() {
const author = legacyFile.match(/@author [^\n]+/)[0];
const copyright = legacyFile.match(/@copyright [^\n]+/)[0];
const utilsUsed = /Utils/.test(legacyFile);
const esc = new EscapeString();
const desc = esc.run(op.description, ["Special chars", "Double"]);
const template = `/**
* ${author}
@ -80,7 +83,7 @@ class ${moduleName} extends Operation {
this.name = "${opName}";${op.flowControl ? "\n this.flowControl = true;" : ""}
this.module = "${op.module}";
this.description = "${op.description}";
this.description = "${desc}";
this.inputType = "${op.inputType}";
this.outputType = "${op.outputType}";${op.manualBake ? "\n this.manualBake = true;" : ""}
this.args = ${JSON.stringify(op.args, null, 4).split("\n").join("\n ")};
@ -126,16 +129,18 @@ ${op.highlight ? `
export default ${moduleName};
`;
console.log("\nLegacy operation config\n-----------------------\n");
console.log(template);
console.log(JSON.stringify(op, null, 4));
console.log("\n-----------------------\n");
const filename = path.join(dir, `../operations/${moduleName}.mjs`);
if (fs.existsSync(filename)) {
console.log(`${filename} already exists. It has NOT been overwritten.`);
console.log(`\u274c ${filename} already exists. It has NOT been overwritten.`);
process.exit(0);
}
fs.writeFileSync(filename, template);
console.log("Written to " + filename);
console.log("\u2714 Written to " + filename);
console.log(`Open ${legacyFilename} and copy the relevant code over. Make sure you check imports, args and highlights.`);
}

View File

@ -25,3 +25,35 @@ export const LETTER_DELIM_OPTIONS = ["Space", "Line feed", "CRLF", "Forward slas
* Word sequence delimiters.
*/
export const WORD_DELIM_OPTIONS = ["Line feed", "CRLF", "Forward slash", "Backslash", "Comma", "Semi-colon", "Colon"];
/**
* Input sequence delimiters.
*/
export const INPUT_DELIM_OPTIONS = ["Line feed", "CRLF", "Space", "Comma", "Semi-colon", "Colon", "Nothing (separate chars)"];
/**
* Split delimiters.
*/
export const SPLIT_DELIM_OPTIONS = [
{name: "Comma", value: ","},
{name: "Space", value: " "},
{name: "Line feed", value: "\\n"},
{name: "CRLF", value: "\\r\\n"},
{name: "Semi-colon", value: ";"},
{name: "Colon", value: ":"},
{name: "Nothing (separate chars)", value: ""}
];
/**
* Join delimiters.
*/
export const JOIN_DELIM_OPTIONS = [
{name: "Line feed", value: "\\n"},
{name: "CRLF", value: "\\r\\n"},
{name: "Space", value: " "},
{name: "Comma", value: ","},
{name: "Semi-colon", value: ";"},
{name: "Colon", value: ":"},
{name: "Nothing (join chars)", value: ""}
];

View File

@ -0,0 +1,59 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2017
* @license Apache-2.0
*/
import Operation from "../Operation";
/**
* Decode NetBIOS Name operation
*/
class DecodeNetBIOSName extends Operation {
/**
* DecodeNetBIOSName constructor
*/
constructor() {
super();
this.name = "Decode NetBIOS Name";
this.module = "Default";
this.description = "NetBIOS names as seen across the client interface to NetBIOS are exactly 16 bytes long. Within the NetBIOS-over-TCP protocols, a longer representation is used.<br><br>There are two levels of encoding. The first level maps a NetBIOS name into a domain system name. The second level maps the domain system name into the 'compressed' representation required for interaction with the domain name system.<br><br>This operation decodes the first level of encoding. See RFC 1001 for full details.";
this.inputType = "byteArray";
this.outputType = "byteArray";
this.args = [
{
"name": "Offset",
"type": "number",
"value": 65
}
];
}
/**
* @param {byteArray} input
* @param {Object[]} args
* @returns {byteArray}
*/
run(input, args) {
const output = [],
offset = args[0];
if (input.length <= 32 && (input.length % 2) === 0) {
for (let i = 0; i < input.length; i += 2) {
output.push((((input[i] & 0xff) - offset) << 4) |
(((input[i + 1] & 0xff) - offset) & 0xf));
}
for (let i = output.length - 1; i > 0; i--) {
if (output[i] === 32) output.splice(i, i);
else break;
}
}
return output;
}
}
export default DecodeNetBIOSName;

View File

@ -0,0 +1,59 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2017
* @license Apache-2.0
*/
import Operation from "../Operation";
/**
* Encode NetBIOS Name operation
*/
class EncodeNetBIOSName extends Operation {
/**
* EncodeNetBIOSName constructor
*/
constructor() {
super();
this.name = "Encode NetBIOS Name";
this.module = "Default";
this.description = "NetBIOS names as seen across the client interface to NetBIOS are exactly 16 bytes long. Within the NetBIOS-over-TCP protocols, a longer representation is used.<br><br>There are two levels of encoding. The first level maps a NetBIOS name into a domain system name. The second level maps the domain system name into the 'compressed' representation required for interaction with the domain name system.<br><br>This operation carries out the first level of encoding. See RFC 1001 for full details.";
this.inputType = "byteArray";
this.outputType = "byteArray";
this.args = [
{
"name": "Offset",
"type": "number",
"value": 65
}
];
}
/**
* @param {byteArray} input
* @param {Object[]} args
* @returns {byteArray}
*/
run(input, args) {
const output = [],
offset = args[0];
if (input.length <= 16) {
const len = input.length;
input.length = 16;
input.fill(32, len, 16);
for (let i = 0; i < input.length; i++) {
output.push((input[i] >> 4) + offset);
output.push((input[i] & 0xf) + offset);
}
}
return output;
}
}
export default EncodeNetBIOSName;

View File

@ -0,0 +1,87 @@
/**
* @author Vel0x [dalemy@microsoft.com]
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import jsesc from "jsesc";
/**
* Escape string operation
*/
class EscapeString extends Operation {
/**
* EscapeString constructor
*/
constructor() {
super();
this.name = "Escape string";
this.module = "Default";
this.description = "Escapes special characters in a string so that they do not cause conflicts. For example, <code>Don't stop me now</code> becomes <code>Don\\'t stop me now</code>.<br><br>Supports the following escape sequences:<ul><li><code>\\n</code> (Line feed/newline)</li><li><code>\\r</code> (Carriage return)</li><li><code>\\t</code> (Horizontal tab)</li><li><code>\\b</code> (Backspace)</li><li><code>\\f</code> (Form feed)</li><li><code>\\xnn</code> (Hex, where n is 0-f)</li><li><code>\\\\</code> (Backslash)</li><li><code>\\'</code> (Single quote)</li><li><code>\\&quot;</code> (Double quote)</li><li><code>\\unnnn</code> (Unicode character)</li><li><code>\\u{nnnnnn}</code> (Unicode code point)</li></ul>";
this.inputType = "string";
this.outputType = "string";
this.args = [
{
"name": "Escape level",
"type": "option",
"value": ["Special chars", "Everything", "Minimal"]
},
{
"name": "Escape quote",
"type": "option",
"value": ["Single", "Double", "Backtick"]
},
{
"name": "JSON compatible",
"type": "boolean",
"value": false
},
{
"name": "ES6 compatible",
"type": "boolean",
"value": true
},
{
"name": "Uppercase hex",
"type": "boolean",
"value": false
}
];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {string}
*
* @example
* EscapeString.run("Don't do that", [])
* > "Don\'t do that"
* EscapeString.run(`Hello
* World`, [])
* > "Hello\nWorld"
*/
run(input, args) {
const level = args[0],
quotes = args[1],
jsonCompat = args[2],
es6Compat = args[3],
lowercaseHex = !args[4];
return jsesc(input, {
quotes: quotes.toLowerCase(),
es6: es6Compat,
escapeEverything: level === "Everything",
minimal: level === "Minimal",
json: jsonCompat,
lowercaseHex: lowercaseHex,
});
}
}
export default EscapeString;

View File

@ -0,0 +1,71 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import Utils from "../Utils";
import {INPUT_DELIM_OPTIONS} from "../lib/Delim";
/**
* Filter operation
*/
class Filter extends Operation {
/**
* Filter constructor
*/
constructor() {
super();
this.name = "Filter";
this.module = "Default";
this.description = "Splits up the input using the specified delimiter and then filters each branch based on a regular expression.";
this.inputType = "string";
this.outputType = "string";
this.args = [
{
"name": "Delimiter",
"type": "option",
"value": INPUT_DELIM_OPTIONS
},
{
"name": "Regex",
"type": "string",
"value": ""
},
{
"name": "Invert condition",
"type": "boolean",
"value": false
}
];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
const delim = Utils.charRep(args[0]),
reverse = args[2];
let regex;
try {
regex = new RegExp(args[1]);
} catch (err) {
return "Invalid regex. Details: " + err.message;
}
const regexFilter = function(value) {
return reverse ^ regex.test(value);
};
return input.split(delim).filter(regexFilter).join(delim);
}
}
export default Filter;

View File

@ -0,0 +1,96 @@
/**
* @author GCHQ Contributor [2]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import Utils from "../Utils";
import {fromHex} from "../lib/Hex";
/**
* Hamming Distance operation
*/
class HammingDistance extends Operation {
/**
* HammingDistance constructor
*/
constructor() {
super();
this.name = "Hamming Distance";
this.module = "Default";
this.description = "In information theory, the Hamming distance between two strings of equal length is the number of positions at which the corresponding symbols are different. In other words, it measures the minimum number of substitutions required to change one string into the other, or the minimum number of errors that could have transformed one string into the other. In a more general context, the Hamming distance is one of several string metrics for measuring the edit distance between two sequences.";
this.inputType = "string";
this.outputType = "string";
this.args = [
{
"name": "Delimiter",
"type": "binaryShortString",
"value": "\\n\\n"
},
{
"name": "Unit",
"type": "option",
"value": ["Byte", "Bit"]
},
{
"name": "Input type",
"type": "option",
"value": ["Raw string", "Hex"]
}
];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
const delim = args[0],
byByte = args[1] === "Byte",
inputType = args[2],
samples = input.split(delim);
if (samples.length !== 2) {
return "Error: You can only calculae the edit distance between 2 strings. Please ensure exactly two inputs are provided, separated by the specified delimiter.";
}
if (samples[0].length !== samples[1].length) {
return "Error: Both inputs must be of the same length.";
}
if (inputType === "Hex") {
samples[0] = fromHex(samples[0]);
samples[1] = fromHex(samples[1]);
} else {
samples[0] = Utils.strToByteArray(samples[0]);
samples[1] = Utils.strToByteArray(samples[1]);
}
let dist = 0;
for (let i = 0; i < samples[0].length; i++) {
const lhs = samples[0][i],
rhs = samples[1][i];
if (byByte && lhs !== rhs) {
dist++;
} else if (!byByte) {
let xord = lhs ^ rhs;
while (xord) {
dist++;
xord &= xord - 1;
}
}
}
return dist.toString();
}
}
export default HammingDistance;

View File

@ -0,0 +1,68 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import Utils from "../Utils";
import {INPUT_DELIM_OPTIONS} from "../lib/Delim";
/**
* Head operation
*/
class Head extends Operation {
/**
* Head constructor
*/
constructor() {
super();
this.name = "Head";
this.module = "Default";
this.description = "Like the UNIX head utility.<br>Gets the first n lines.<br>You can select all but the last n lines by entering a negative value for n.<br>The delimiter can be changed so that instead of lines, fields (i.e. commas) are selected instead.";
this.inputType = "string";
this.outputType = "string";
this.args = [
{
"name": "Delimiter",
"type": "option",
"value": INPUT_DELIM_OPTIONS
},
{
"name": "Number",
"type": "number",
"value": 10
}
];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
let delimiter = args[0];
const number = args[1];
delimiter = Utils.charRep(delimiter);
const splitInput = input.split(delimiter);
return splitInput
.filter((line, lineIndex) => {
lineIndex += 1;
if (number < 0) {
return lineIndex <= splitInput.length + number;
} else {
return lineIndex <= number;
}
})
.join(delimiter);
}
}
export default Head;

View File

@ -0,0 +1,106 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import Utils from "../Utils";
/**
* Offset checker operation
*/
class OffsetChecker extends Operation {
/**
* OffsetChecker constructor
*/
constructor() {
super();
this.name = "Offset checker";
this.module = "Default";
this.description = "Compares multiple inputs (separated by the specified delimiter) and highlights matching characters which appear at the same position in all samples.";
this.inputType = "string";
this.outputType = "html";
this.args = [
{
"name": "Sample delimiter",
"type": "binaryString",
"value": "\\n\\n"
}
];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {html}
*/
run(input, args) {
const sampleDelim = args[0],
samples = input.split(sampleDelim),
outputs = new Array(samples.length);
let i = 0,
s = 0,
match = false,
inMatch = false,
chr;
if (!samples || samples.length < 2) {
return "Not enough samples, perhaps you need to modify the sample delimiter or add more data?";
}
// Initialise output strings
outputs.fill("", 0, samples.length);
// Loop through each character in the first sample
for (i = 0; i < samples[0].length; i++) {
chr = samples[0][i];
match = false;
// Loop through each sample to see if the chars are the same
for (s = 1; s < samples.length; s++) {
if (samples[s][i] !== chr) {
match = false;
break;
}
match = true;
}
// Write output for each sample
for (s = 0; s < samples.length; s++) {
if (samples[s].length <= i) {
if (inMatch) outputs[s] += "</span>";
if (s === samples.length - 1) inMatch = false;
continue;
}
if (match && !inMatch) {
outputs[s] += "<span class='hl5'>" + Utils.escapeHtml(samples[s][i]);
if (samples[s].length === i + 1) outputs[s] += "</span>";
if (s === samples.length - 1) inMatch = true;
} else if (!match && inMatch) {
outputs[s] += "</span>" + Utils.escapeHtml(samples[s][i]);
if (s === samples.length - 1) inMatch = false;
} else {
outputs[s] += Utils.escapeHtml(samples[s][i]);
if (inMatch && samples[s].length === i + 1) {
outputs[s] += "</span>";
if (samples[s].length - 1 !== i) inMatch = false;
}
}
if (samples[0].length - 1 === i) {
if (inMatch) outputs[s] += "</span>";
outputs[s] += Utils.escapeHtml(samples[s].substring(i + 1));
}
}
}
return outputs.join(sampleDelim);
}
}
export default OffsetChecker;

View File

@ -0,0 +1,55 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import {SPLIT_DELIM_OPTIONS, JOIN_DELIM_OPTIONS} from "../lib/Delim";
/**
* Split operation
*/
class Split extends Operation {
/**
* Split constructor
*/
constructor() {
super();
this.name = "Split";
this.module = "Default";
this.description = "Splits a string into sections around a given delimiter.";
this.inputType = "string";
this.outputType = "string";
this.args = [
{
"name": "Split delimiter",
"type": "editableOption",
"value": SPLIT_DELIM_OPTIONS
},
{
"name": "Join delimiter",
"type": "editableOption",
"value": JOIN_DELIM_OPTIONS
}
];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
const splitDelim = args[0],
joinDelim = args[1],
sections = input.split(splitDelim);
return sections.join(joinDelim);
}
}
export default Split;

View File

@ -0,0 +1,69 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import Utils from "../Utils";
import {INPUT_DELIM_OPTIONS} from "../lib/Delim";
/**
* Tail operation
*/
class Tail extends Operation {
/**
* Tail constructor
*/
constructor() {
super();
this.name = "Tail";
this.module = "Default";
this.description = "Like the UNIX tail utility.<br>Gets the last n lines.<br>Optionally you can select all lines after line n by entering a negative value for n.<br>The delimiter can be changed so that instead of lines, fields (i.e. commas) are selected instead.";
this.inputType = "string";
this.outputType = "string";
this.args = [
{
"name": "Delimiter",
"type": "option",
"value": INPUT_DELIM_OPTIONS
},
{
"name": "Number",
"type": "number",
"value": 10
}
];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
let delimiter = args[0];
const number = args[1];
delimiter = Utils.charRep(delimiter);
const splitInput = input.split(delimiter);
return splitInput
.filter((line, lineIndex) => {
lineIndex += 1;
if (number < 0) {
return lineIndex > -number;
} else {
return lineIndex > splitInput.length - number;
}
})
.join(delimiter);
}
}
export default Tail;

View File

@ -0,0 +1,65 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
/**
* To Lower case operation
*/
class ToLowerCase extends Operation {
/**
* ToLowerCase constructor
*/
constructor() {
super();
this.name = "To Lower case";
this.module = "Default";
this.description = "Converts every character in the input to lower case.";
this.inputType = "string";
this.outputType = "string";
this.args = [];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
return input.toLowerCase();
}
/**
* Highlight To Lower case
*
* @param {Object[]} pos
* @param {number} pos[].start
* @param {number} pos[].end
* @param {Object[]} args
* @returns {Object[]} pos
*/
highlight(pos, args) {
return pos;
}
/**
* Highlight To Lower case in reverse
*
* @param {Object[]} pos
* @param {number} pos[].start
* @param {number} pos[].end
* @param {Object[]} args
* @returns {Object[]} pos
*/
highlightReverse(pos, args) {
return pos;
}
}
export default ToLowerCase;

View File

@ -0,0 +1,89 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
/**
* To Upper case operation
*/
class ToUpperCase extends Operation {
/**
* ToUpperCase constructor
*/
constructor() {
super();
this.name = "To Upper case";
this.module = "Default";
this.description = "Converts the input string to upper case, optionally limiting scope to only the first character in each word, sentence or paragraph.";
this.inputType = "string";
this.outputType = "string";
this.args = [
{
"name": "Scope",
"type": "option",
"value": ["All", "Word", "Sentence", "Paragraph"]
}
];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
const scope = args[0];
switch (scope) {
case "Word":
return input.replace(/(\b\w)/gi, function(m) {
return m.toUpperCase();
});
case "Sentence":
return input.replace(/(?:\.|^)\s*(\b\w)/gi, function(m) {
return m.toUpperCase();
});
case "Paragraph":
return input.replace(/(?:\n|^)\s*(\b\w)/gi, function(m) {
return m.toUpperCase();
});
case "All": /* falls through */
default:
return input.toUpperCase();
}
}
/**
* Highlight To Upper case
*
* @param {Object[]} pos
* @param {number} pos[].start
* @param {number} pos[].end
* @param {Object[]} args
* @returns {Object[]} pos
*/
highlight(pos, args) {
return pos;
}
/**
* Highlight To Upper case in reverse
*
* @param {Object[]} pos
* @param {number} pos[].start
* @param {number} pos[].end
* @param {Object[]} args
* @returns {Object[]} pos
*/
highlightReverse(pos, args) {
return pos;
}
}
export default ToUpperCase;

View File

@ -0,0 +1,40 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import Utils from "../Utils";
/**
* Unescape string operation
*/
class UnescapeString extends Operation {
/**
* UnescapeString constructor
*/
constructor() {
super();
this.name = "Unescape string";
this.module = "Default";
this.description = "Unescapes characters in a string that have been escaped. For example, <code>Don\\'t stop me now</code> becomes <code>Don't stop me now</code>.<br><br>Supports the following escape sequences:<ul><li><code>\\n</code> (Line feed/newline)</li><li><code>\\r</code> (Carriage return)</li><li><code>\\t</code> (Horizontal tab)</li><li><code>\\b</code> (Backspace)</li><li><code>\\f</code> (Form feed)</li><li><code>\\xnn</code> (Hex, where n is 0-f)</li><li><code>\\\\</code> (Backslash)</li><li><code>\\'</code> (Single quote)</li><li><code>\\&quot;</code> (Double quote)</li><li><code>\\unnnn</code> (Unicode character)</li><li><code>\\u{nnnnnn}</code> (Unicode code point)</li></ul>";
this.inputType = "string";
this.outputType = "string";
this.args = [];
}
/**
* @param {string} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
return Utils.parseEscapedChars(input);
}
}
export default UnescapeString;