Mirror
/
CyberChef
mirror of https://github.com/gchq/CyberChef.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'esm' of github.com:gchq/CyberChef into esm

This commit is contained in:
d98762625 2018-05-21 09:02:21 +01:00
parent 3f08fa3b23 093a7c8c8a
commit 9f52689fde
37 changed files with 2735 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.eslintignore
View File

@ -1,2 +1,2 @@
src/core/vendor/**
src/core/operations/legacy/**
src/core/operations/legacy/**

3
Gruntfile.js
View File

@ -379,6 +379,9 @@ module.exports = function (grunt) {
generateConfig: {
command: [
"echo '\n--- Regenerating config files. ---'",
"mkdir -p src/core/config/modules",
"echo 'export default {};\n' > src/core/config/modules/OpModules.mjs",
"echo '[]\n' > src/core/config/OperationConfig.json",
"node --experimental-modules src/core/config/scripts/generateOpsIndex.mjs",
"node --experimental-modules src/core/config/scripts/generateConfig.mjs",
"echo '--- Config scripts finished. ---\n'"

25
package-lock.json generated
View File

@ -1897,6 +1897,14 @@
"integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=",
"dev": true
},
"chi-squared": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/chi-squared/-/chi-squared-1.1.0.tgz",
"integrity": "sha1-iShlz/qOCnIPkhv8nGNcGawqNG0=",
"requires": {
"gamma": "^1.0.0"
}
},
"chokidar": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.0.3.tgz",
@ -4157,8 +4165,8 @@
"dev": true,
"optional": true,
"requires": {
"nan": "2.10.0",
"node-pre-gyp": "0.10.0"
"nan": "^2.9.2",
"node-pre-gyp": "^0.10.0"
},
"dependencies": {
"abbrev": {
@ -4531,10 +4539,10 @@
"dev": true,
"optional": true,
"requires": {
"deep-extend": "0.5.1",
"ini": "1.3.5",
"minimist": "1.2.0",
"strip-json-comments": "2.0.1"
"deep-extend": "^0.5.1",
"ini": "~1.3.0",
"minimist": "^1.2.0",
"strip-json-comments": "~2.0.1"
},
"dependencies": {
"minimist": {
@ -4691,6 +4699,11 @@
"integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=",
"dev": true
},
"gamma": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/gamma/-/gamma-1.0.0.tgz",
"integrity": "sha1-mDwck5/iPZMnAVhXEeHZpDDLdMs="
},
"gaze": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/gaze/-/gaze-1.1.2.tgz",

1
package.json
View File

@ -77,6 +77,7 @@
"bootstrap-colorpicker": "^2.5.2",
"bootstrap-switch": "^3.3.4",
"bson": "^2.0.6",
"chi-squared": "^1.1.0",
"crypto-api": "^0.8.0",
"crypto-js": "^3.1.9-1",
"ctph.js": "0.0.5",

40
src/core/ChefWorker.js
View File

@ -91,7 +91,7 @@ self.addEventListener("message", function(e) {
*/
async function bake(data) {
// Ensure the relevant modules are loaded
loadRequiredModules(data.recipeConfig);
self.loadRequiredModules(data.recipeConfig);
try {
const response = await self.chef.bake(
@ -144,25 +144,6 @@ async function getDishAs(data) {
}
/**
* Checks that all required modules are loaded and loads them if not.
*
* @param {Object} recipeConfig
*/
function loadRequiredModules(recipeConfig) {
recipeConfig.forEach(op => {
const module = self.OperationConfig[op.op].module;
if (!OpModules.hasOwnProperty(module)) {
log.info(`Loading ${module} module`);
self.sendStatusMessage(`Loading ${module} module`);
self.importScripts(`${self.docURL}/${module}.js`);
self.sendStatusMessage("");
}
});
}
/**
* Calculates highlight offsets if possible.
*
@ -182,6 +163,25 @@ function calculateHighlights(recipeConfig, direction, pos) {
}
/**
* Checks that all required modules are loaded and loads them if not.
*
* @param {Object} recipeConfig
*/
self.loadRequiredModules = function(recipeConfig) {
recipeConfig.forEach(op => {
const module = self.OperationConfig[op.op].module;
if (!OpModules.hasOwnProperty(module)) {
log.info(`Loading ${module} module`);
self.sendStatusMessage(`Loading ${module} module`);
self.importScripts(`${self.docURL}/${module}.js`);
self.sendStatusMessage("");
}
});
};
/**
* Send status update to the app.
*

11
src/core/Operation.mjs
View File

@ -188,7 +188,7 @@ class Operation {
get config() {
return {
"op": this.name,
"args": this._ingList.map(ing => ing.conf)
"args": this._ingList.map(ing => ing.config)
};
}
@ -274,6 +274,15 @@ class Operation {
return this._flowControl;
}
/**
* Set whether this Operation is a flowcontrol op.
*
* @param {boolean} value
*/
set flowControl(value) {
this._flowControl = !!value;
}
}
export default Operation;

2
src/core/Recipe.mjs
View File

@ -172,9 +172,9 @@ class Recipe {
numRegisters = state.numRegisters;
} else {
output = await op.run(input, op.ingValues);
this.lastRunOp = op;
dish.set(output, op.outputType);
}
this.lastRunOp = op;
} catch (err) {
// Return expected errors as output
if (err instanceof OperationError ||

2
src/core/Utils.mjs
View File

@ -720,7 +720,7 @@ class Utils {
* @param {boolean} [newline=false] - whether to add a newline after each operation
* @returns {string}
*/
static generatePrettyRecipe(recipeConfig, newline=false) {
static generatePrettyRecipe(recipeConfig, newline = false) {
let prettyConfig = "",
name = "",
args = "",

1
src/core/config/Categories.json
View File

@ -338,6 +338,7 @@
{
"name": "Flow control",
"ops": [
"Magic",
"Fork",
"Merge",
"Register",

4
src/core/config/scripts/generateConfig.mjs
View File

@ -43,6 +43,10 @@ for (const opObj in Ops) {
args: op.args
};
if (op.hasOwnProperty("patterns")) {
operationConfig[op.name].patterns = op.patterns;
}
if (!modules.hasOwnProperty(op.module))
modules[op.module] = {};
modules[op.module][op.name] = opObj;

412
src/core/config/scripts/portOperation.mjs
View File

@ -61,6 +61,8 @@ function main() {
const utilsUsed = /Utils/.test(legacyFile);
const esc = new EscapeString();
const desc = esc.run(op.description, ["Special chars", "Double"]);
const patterns = op.hasOwnProperty("patterns") ? `
this.patterns = ${JSON.stringify(op.patterns, null, 4).split("\n").join("\n ")};` : "";
// Attempt to find the operation run function based on the JSDoc comment
const regex = `\\* ${opName} operation[^:]+:(?: function ?\\(input, args\\))? ?{([\\s\\S]+?)\n }`;
@ -105,7 +107,7 @@ class ${moduleName} extends Operation {
this.description = "${desc}";
this.inputType = "${op.inputType}";
this.outputType = "${op.outputType}";${op.manualBake ? "\n this.manualBake = true;" : ""}
this.args = ${JSON.stringify(op.args, null, 4).split("\n").join("\n ")};
this.args = ${JSON.stringify(op.args, null, 4).split("\n").join("\n ")};${patterns}
}
/**
@ -172,6 +174,30 @@ export default ${moduleName};
const OP_CONFIG = {
"Magic": {
module: "Default",
description: "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.",
inputType: "ArrayBuffer",
outputType: "html",
flowControl: true,
args: [
{
name: "Depth",
type: "number",
value: 3
},
{
name: "Intensive mode",
type: "boolean",
value: false
},
{
name: "Extensive language support",
type: "boolean",
value: false
}
]
},
"Fork": {
module: "Default",
description: "Split the input data up based on the specified delimiter and run all subsequent operations on each branch separately.<br><br>For example, to decode multiple Base64 strings, enter them all on separate lines then add the 'Fork' and 'From Base64' operations to the recipe. Each string will be decoded separately.",
@ -330,6 +356,73 @@ const OP_CONFIG = {
type: "boolean",
value: "Base64.REMOVE_NON_ALPH_CHARS"
}
],
patterns: [
{
match: "^(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
flags: "i",
args: ["A-Za-z0-9+/=", false]
},
{
match: "^[A-Z\\d\\-_]{20,}$",
flags: "i",
args: ["A-Za-z0-9-_", false]
},
{
match: "^(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?$",
flags: "i",
args: ["A-Za-z0-9+\\-=", false]
},
{
match: "^(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?$",
flags: "i",
args: ["./0-9A-Za-z=", false]
},
{
match: "^[A-Z\\d_.]{20,}$",
flags: "i",
args: ["A-Za-z0-9_.", false]
},
{
match: "^(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?$",
flags: "i",
args: ["A-Za-z0-9._-", false]
},
{
match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
flags: "i",
args: ["0-9a-zA-Z+/=", false]
},
{
match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
flags: "i",
args: ["0-9A-Za-z+/=", false]
},
{
match: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
flags: "",
args: [" -_", false]
},
{
match: "^[A-Z\\d+\\-]{20,}$",
flags: "i",
args: ["+\\-0-9A-Za-z", false]
},
{
match: "^[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}$",
flags: "",
args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", false]
},
{
match: "^(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?$",
flags: "i",
args: ["N-ZA-Mn-za-m0-9+/=", false]
},
{
match: "^[A-Z\\d./]{20,}$",
flags: "i",
args: ["./0-9A-Za-z", false]
},
]
},
"To Base64": {
@ -363,6 +456,18 @@ const OP_CONFIG = {
type: "boolean",
value: "Base58.REMOVE_NON_ALPH_CHARS"
}
],
patterns: [
{
match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
flags: "",
args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", false]
},
{
match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
flags: "",
args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", false]
},
]
},
"To Base58": {
@ -394,6 +499,13 @@ const OP_CONFIG = {
type: "boolean",
value: "Base64.REMOVE_NON_ALPH_CHARS"
}
],
patterns: [
{
match: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$",
flags: "",
args: ["A-Z2-7=", false]
},
]
},
"To Base32": {
@ -744,6 +856,53 @@ const OP_CONFIG = {
type: "option",
value: "ByteRepr.HEX_DELIM_OPTIONS"
}
],
patterns: [
{
match: "^(?:[\\dA-F]{2})+$",
flags: "i",
args: ["None"]
},
{
match: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$",
flags: "i",
args: ["Space"]
},
{
match: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$",
flags: "i",
args: ["Comma"]
},
{
match: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$",
flags: "i",
args: ["Semi-colon"]
},
{
match: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$",
flags: "i",
args: ["Colon"]
},
{
match: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$",
flags: "i",
args: ["Line feed"]
},
{
match: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$",
flags: "i",
args: ["CRLF"]
},
{
match: "^[\\dA-F]{2}(?:0x[\\dA-F]{2})*$",
flags: "i",
args: ["0x"]
},
{
match: "^[\\dA-F]{2}(?:\\\\x[\\dA-F]{2})*$",
flags: "i",
args: ["\\x"]
}
]
},
"To Hex": {
@ -774,6 +933,38 @@ const OP_CONFIG = {
type: "option",
value: "ByteRepr.DELIM_OPTIONS"
}
],
patterns: [
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["Space"]
},
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["Comma"]
},
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["Semi-colon"]
},
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["Colon"]
},
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["Line feed"]
},
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["CRLF"]
},
]
},
"To Octal": {
@ -811,7 +1002,6 @@ const OP_CONFIG = {
}
]
},
"To Charcode": {
module: "Default",
description: "Converts text to its unicode character code equivalent.<br><br>e.g. <code>Γειά σου</code> becomes <code>0393 03b5 03b9 03ac 20 03c3 03bf 03c5</code>",
@ -845,6 +1035,43 @@ const OP_CONFIG = {
type: "option",
value: "ByteRepr.BIN_DELIM_OPTIONS"
}
],
patterns: [
{
match: "^(?:[01]{8})+$",
flags: "",
args: ["None"]
},
{
match: "^(?:[01]{8})(?: [01]{8})*$",
flags: "",
args: ["Space"]
},
{
match: "^(?:[01]{8})(?:,[01]{8})*$",
flags: "",
args: ["Comma"]
},
{
match: "^(?:[01]{8})(?:;[01]{8})*$",
flags: "",
args: ["Semi-colon"]
},
{
match: "^(?:[01]{8})(?::[01]{8})*$",
flags: "",
args: ["Colon"]
},
{
match: "^(?:[01]{8})(?:\\n[01]{8})*$",
flags: "",
args: ["Line feed"]
},
{
match: "^(?:[01]{8})(?:\\r\\n[01]{8})*$",
flags: "",
args: ["CRLF"]
},
]
},
"To Binary": {
@ -873,6 +1100,38 @@ const OP_CONFIG = {
type: "option",
value: "ByteRepr.DELIM_OPTIONS"
}
],
patterns: [
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["Space"]
},
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["Comma"]
},
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["Semi-colon"]
},
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["Colon"]
},
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["Line feed"]
},
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["CRLF"]
},
]
},
"To Decimal": {
@ -895,7 +1154,14 @@ const OP_CONFIG = {
highlightReverse: "func",
inputType: "string",
outputType: "byteArray",
args: []
args: [],
patterns: [
{
match: "^(?:(?:[\\dA-F]{4,16}:?)?\\s*((?:[\\dA-F]{2}\\s){1,8}(?:\\s|[\\dA-F]{2}-)(?:[\\dA-F]{2}\\s){1,8}|(?:[\\dA-F]{2}\\s|[\\dA-F]{4}\\s)+)[^\\n]*\\n?)+$",
flags: "i",
args: []
},
]
},
"To Hexdump": {
module: "Default",
@ -953,7 +1219,14 @@ const OP_CONFIG = {
description: "Converts HTML entities back to characters<br><br>e.g. <code>&amp;<span>amp;</span></code> becomes <code>&amp;</code>", // <span> tags required to stop the browser just printing &
inputType: "string",
outputType: "string",
args: []
args: [],
patterns: [
{
match: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});",
flags: "i",
args: []
},
]
},
"To HTML Entity": {
module: "Default",
@ -996,7 +1269,14 @@ const OP_CONFIG = {
description: "Converts URI/URL percent-encoded characters back to their raw values.<br><br>e.g. <code>%3d</code> becomes <code>=</code>",
inputType: "string",
outputType: "string",
args: []
args: [],
patterns: [
{
match: ".*(?:%[\\da-f]{2}.*){4}",
flags: "i",
args: []
},
]
},
"URL Encode": {
module: "URL",
@ -1057,6 +1337,23 @@ const OP_CONFIG = {
type: "boolean",
value: true
}
],
patterns: [
{
match: "\\\\u(?:[\\da-f]{4,6})",
flags: "i",
args: ["\\u"]
},
{
match: "%u(?:[\\da-f]{4,6})",
flags: "i",
args: ["%u"]
},
{
match: "U\\+(?:[\\da-f]{4,6})",
flags: "i",
args: ["U+"]
},
]
},
"From Quoted Printable": {
@ -1064,7 +1361,14 @@ const OP_CONFIG = {
description: "Converts QP-encoded text back to standard text.",
inputType: "string",
outputType: "byteArray",
args: []
args: [],
patterns: [
{
match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]*(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
flags: "i",
args: []
},
]
},
"To Quoted Printable": {
module: "Default",
@ -1226,7 +1530,7 @@ const OP_CONFIG = {
type: "option",
value: "Object.keys(CharEnc.IO_FORMAT),"
},
]
],
},
"Decode text": {
module: "CharEnc",
@ -2573,6 +2877,28 @@ const OP_CONFIG = {
type: "option",
value: "DateTime.UNITS"
}
],
patterns: [
{
match: "^1?\\d{9}$",
flags: "",
args: ["Seconds (s)"]
},
{
match: "^1?\\d{12}$",
flags: "",
args: ["Milliseconds (ms)"]
},
{
match: "^1?\\d{15}$",
flags: "",
args: ["Microseconds (μs)"]
},
{
match: "^1?\\d{18}$",
flags: "",
args: ["Nanoseconds (ns)"]
},
]
},
"To UNIX Timestamp": {
@ -2885,6 +3211,13 @@ const OP_CONFIG = {
type: "boolean",
value: "Compress.INFLATE_VERIFY"
}
],
patterns: [
{
match: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)",
flags: "",
args: [0, 0, "Adaptive", false, false]
},
]
},
"Gzip": {
@ -2920,7 +3253,14 @@ const OP_CONFIG = {
description: "Decompresses data which has been compressed using the deflate algorithm with gzip headers.",
inputType: "byteArray",
outputType: "byteArray",
args: []
args: [],
patterns: [
{
match: "^\\x1f\\x8b\\x08",
flags: "",
args: []
},
]
},
"Zip": {
module: "Compression",
@ -2976,6 +3316,13 @@ const OP_CONFIG = {
type: "boolean",
value: "Compress.PKUNZIP_VERIFY"
}
],
patterns: [
{
match: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)",
flags: "",
args: ["", false]
},
]
},
"Bzip2 Decompress": {
@ -2983,7 +3330,14 @@ const OP_CONFIG = {
description: "Decompresses data using the Bzip2 algorithm.",
inputType: "byteArray",
outputType: "string",
args: []
args: [],
patterns: [
{
match: "^\\x42\\x5a\\x68",
flags: "",
args: []
},
]
},
"Generic Code Beautify": {
module: "Code",
@ -3468,7 +3822,7 @@ const OP_CONFIG = {
},
"Chi Square": {
module: "Default",
description: "Calculates the Chi Square distribution of values.",
description: "Calculates the Chi-Squared distribution of values.",
inputType: "ArrayBuffer",
outputType: "number",
args: []
@ -3491,6 +3845,13 @@ const OP_CONFIG = {
type: "option",
value: "PublicKey.X509_INPUT_FORMAT"
}
],
patterns: [
{
match: "^-+BEGIN CERTIFICATE-+\\r?\\n[\\da-z+/\\n\\r]+-+END CERTIFICATE-+\\r?\\n?$",
flags: "i",
args: ["PEM"]
},
]
},
"PEM to Hex": {
@ -3783,6 +4144,13 @@ const OP_CONFIG = {
type: "option",
value: "MorseCode.WORD_DELIM_OPTIONS"
}
],
patterns: [
{
match: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)",
flags: "i",
args: ["Space", "Line feed"]
},
]
},
"Tar": {
@ -3803,7 +4171,13 @@ const OP_CONFIG = {
description: "Unpacks a tarball and displays it per file.",
inputType: "byteArray",
outputType: "html",
args: [
args: [],
patterns: [
{
match: "^.{257}\\x75\\x73\\x74\\x61\\x72",
flags: "",
args: []
},
]
},
"Head": {
@ -3945,6 +4319,14 @@ const OP_CONFIG = {
type: "option",
value: "Image.INPUT_FORMAT"
}
],
patterns: [
{
match: "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
flags: "",
args: ["Raw"],
useful: true
},
]
},
"Remove EXIF": {
@ -4026,8 +4408,14 @@ const OP_CONFIG = {
type: "option",
value: "BCD.FORMAT"
}
],
patterns: [
{
match: "^(?:\\d{4} ){3,}\\d{4}$",
flags: "",
args: ["8 4 2 1", true, false, "Nibbles"]
},
]
},
"To BCD": {
module: "Default",

1535
src/core/lib/Magic.mjs Normal file
View File

File diff suppressed because it is too large Load Diff

54
src/core/operations/DetectFileType.mjs Normal file
View File

@ -0,0 +1,54 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import Magic from "../lib/Magic";
/**
* Detect File Type operation
*/
class DetectFileType extends Operation {
/**
* DetectFileType constructor
*/
constructor() {
super();
this.name = "Detect File Type";
this.module = "Default";
this.description = "Attempts to guess the MIME (Multipurpose Internet Mail Extensions) type of the data based on 'magic bytes'.<br><br>Currently supports the following file types: 7z, amr, avi, bmp, bz2, class, cr2, crx, dex, dmg, doc, elf, eot, epub, exe, flac, flv, gif, gz, ico, iso, jpg, jxr, m4a, m4v, mid, mkv, mov, mp3, mp4, mpg, ogg, otf, pdf, png, ppt, ps, psd, rar, rtf, sqlite, swf, tar, tar.z, tif, ttf, utf8, vmdk, wav, webm, webp, wmv, woff, woff2, xls, xz, zip.";
this.inputType = "ArrayBuffer";
this.outputType = "string";
this.args = [];
}
/**
* @param {ArrayBuffer} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
const data = new Uint8Array(input),
type = Magic.magicFileType(data);
if (!type) {
return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?";
} else {
let output = "File extension: " + type.ext + "\n" +
"MIME type: " + type.mime;
if (type.desc && type.desc.length) {
output += "\nDescription: " + type.desc;
}
return output;
}
}
}
export default DetectFileType;

17
src/core/operations/EscapeUnicodeCharacters.mjs
View File

@ -44,6 +44,23 @@ class EscapeUnicodeCharacters extends Operation {
"value": true
}
];
this.patterns = [
{
match: "\\\\u(?:[\\da-f]{4,6})",
flags: "i",
args: ["\\u"]
},
{
match: "%u(?:[\\da-f]{4,6})",
flags: "i",
args: ["%u"]
},
{
match: "U\\+(?:[\\da-f]{4,6})",
flags: "i",
args: ["U+"]
},
];
}
/**

7
src/core/operations/FromBCD.mjs
View File

@ -48,6 +48,13 @@ class FromBCD extends Operation {
"value": FORMAT
}
];
this.patterns = [
{
match: "^(?:\\d{4} ){3,}\\d{4}$",
flags: "",
args: ["8 4 2 1", true, false, "Nibbles"]
},
];
}
/**

7
src/core/operations/FromBase32.mjs
View File

@ -35,6 +35,13 @@ class FromBase32 extends Operation {
value: true
}
];
this.patterns = [
{
match: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$",
flags: "",
args: ["A-Z2-7=", false]
},
];
}
/**

12
src/core/operations/FromBase58.mjs
View File

@ -37,6 +37,18 @@ class FromBase58 extends Operation {
"value": true
}
];
this.patterns = [
{
match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
flags: "",
args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", false]
},
{
match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
flags: "",
args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", false]
},
];
}
/**

67
src/core/operations/FromBase64.mjs
View File

@ -35,6 +35,73 @@ class FromBase64 extends Operation {
value: true
}
];
this.patterns = [
{
match: "^(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
flags: "i",
args: ["A-Za-z0-9+/=", false]
},
{
match: "^[A-Z\\d\\-_]{20,}$",
flags: "i",
args: ["A-Za-z0-9-_", false]
},
{
match: "^(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?$",
flags: "i",
args: ["A-Za-z0-9+\\-=", false]
},
{
match: "^(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?$",
flags: "i",
args: ["./0-9A-Za-z=", false]
},
{
match: "^[A-Z\\d_.]{20,}$",
flags: "i",
args: ["A-Za-z0-9_.", false]
},
{
match: "^(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?$",
flags: "i",
args: ["A-Za-z0-9._-", false]
},
{
match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
flags: "i",
args: ["0-9a-zA-Z+/=", false]
},
{
match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
flags: "i",
args: ["0-9A-Za-z+/=", false]
},
{
match: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
flags: "",
args: [" -_", false]
},
{
match: "^[A-Z\\d+\\-]{20,}$",
flags: "i",
args: ["+\\-0-9A-Za-z", false]
},
{
match: "^[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}$",
flags: "",
args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", false]
},
{
match: "^(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?$",
flags: "i",
args: ["N-ZA-Mn-za-m0-9+/=", false]
},
{
match: "^[A-Z\\d./]{20,}$",
flags: "i",
args: ["./0-9A-Za-z", false]
},
];
}
/**

37
src/core/operations/FromBinary.mjs
View File

@ -31,6 +31,43 @@ class FromBinary extends Operation {
"value": BIN_DELIM_OPTIONS
}
];
this.patterns = [
{
match: "^(?:[01]{8})+$",
flags: "",
args: ["None"]
},
{
match: "^(?:[01]{8})(?: [01]{8})*$",
flags: "",
args: ["Space"]
},
{
match: "^(?:[01]{8})(?:,[01]{8})*$",
flags: "",
args: ["Comma"]
},
{
match: "^(?:[01]{8})(?:;[01]{8})*$",
flags: "",
args: ["Semi-colon"]
},
{
match: "^(?:[01]{8})(?::[01]{8})*$",
flags: "",
args: ["Colon"]
},
{
match: "^(?:[01]{8})(?:\\n[01]{8})*$",
flags: "",
args: ["Line feed"]
},
{
match: "^(?:[01]{8})(?:\\r\\n[01]{8})*$",
flags: "",
args: ["CRLF"]
},
];
}
/**

32
src/core/operations/FromDecimal.mjs
View File

@ -31,6 +31,38 @@ class FromDecimal extends Operation {
"value": DELIM_OPTIONS
}
];
this.patterns = [
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["Space"]
},
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["Comma"]
},
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["Semi-colon"]
},
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["Colon"]
},
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["Line feed"]
},
{
match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
flags: "",
args: ["CRLF"]
},
];
}
/**

7
src/core/operations/FromHTMLEntity.mjs
View File

@ -24,6 +24,13 @@ class FromHTMLEntity extends Operation {
this.inputType = "string";
this.outputType = "string";
this.args = [];
this.patterns = [
{
match: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});",
flags: "i",
args: []
},
];
}
/**

47
src/core/operations/FromHex.mjs
View File

@ -31,6 +31,53 @@ class FromHex extends Operation {
value: FROM_HEX_DELIM_OPTIONS
}
];
this.patterns = [
{
match: "^(?:[\\dA-F]{2})+$",
flags: "i",
args: ["None"]
},
{
match: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$",
flags: "i",
args: ["Space"]
},
{
match: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$",
flags: "i",
args: ["Comma"]
},
{
match: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$",
flags: "i",
args: ["Semi-colon"]
},
{
match: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$",
flags: "i",
args: ["Colon"]
},
{
match: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$",
flags: "i",
args: ["Line feed"]
},
{
match: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$",
flags: "i",
args: ["CRLF"]
},
{
match: "^[\\dA-F]{2}(?:0x[\\dA-F]{2})*$",
flags: "i",
args: ["0x"]
},
{
match: "^[\\dA-F]{2}(?:\\\\x[\\dA-F]{2})*$",
flags: "i",
args: ["\\x"]
}
];
}
/**

7
src/core/operations/FromHexdump.mjs
View File

@ -24,6 +24,13 @@ class FromHexdump extends Operation {
this.inputType = "string";
this.outputType = "byteArray";
this.args = [];
this.patterns = [
{
match: "^(?:(?:[\\dA-F]{4,16}:?)?\\s*((?:[\\dA-F]{2}\\s){1,8}(?:\\s|[\\dA-F]{2}-)(?:[\\dA-F]{2}\\s){1,8}|(?:[\\dA-F]{2}\\s|[\\dA-F]{4}\\s)+)[^\\n]*\\n?)+$",
flags: "i",
args: []
},
];
}
/**

7
src/core/operations/FromMorseCode.mjs
View File

@ -36,6 +36,13 @@ class FromMorseCode extends Operation {
"value": WORD_DELIM_OPTIONS
}
];
this.patterns = [
{
match: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)",
flags: "i",
args: ["Space", "Line feed"]
},
];
}
/**

32
src/core/operations/FromOctal.mjs
View File

@ -31,6 +31,38 @@ class FromOctal extends Operation {
"value": DELIM_OPTIONS
}
];
this.patterns = [
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["Space"]
},
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["Comma"]
},
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["Semi-colon"]
},
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["Colon"]
},
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["Line feed"]
},
{
match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
flags: "",
args: ["CRLF"]
},
];
}
/**

7
src/core/operations/FromQuotedPrintable.mjs
View File

@ -27,6 +27,13 @@ class FromQuotedPrintable extends Operation {
this.inputType = "string";
this.outputType = "byteArray";
this.args = [];
this.patterns = [
{
match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]*(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
flags: "i",
args: []
},
];
}
/**

22
src/core/operations/FromUNIXTimestamp.mjs
View File

@ -32,6 +32,28 @@ class FromUNIXTimestamp extends Operation {
"value": UNITS
}
];
this.patterns = [
{
match: "^1?\\d{9}$",
flags: "",
args: ["Seconds (s)"]
},
{
match: "^1?\\d{12}$",
flags: "",
args: ["Milliseconds (ms)"]
},
{
match: "^1?\\d{15}$",
flags: "",
args: ["Microseconds (μs)"]
},
{
match: "^1?\\d{18}$",
flags: "",
args: ["Nanoseconds (ns)"]
},
];
}
/**

7
src/core/operations/Gunzip.mjs
View File

@ -26,6 +26,13 @@ class Gunzip extends Operation {
this.inputType = "ArrayBuffer";
this.outputType = "ArrayBuffer";
this.args = [];
this.patterns = [
{
match: "^\\x1f\\x8b\\x08",
flags: "",
args: []
},
];
}
/**

140
src/core/operations/Magic.mjs Normal file
View File

@ -0,0 +1,140 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import Utils from "../Utils";
import Dish from "../Dish";
import MagicLib from "../lib/Magic";
/**
* Magic operation
*/
class Magic extends Operation {
/**
* Magic constructor
*/
constructor() {
super();
this.name = "Magic";
this.flowControl = true;
this.module = "Default";
this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.";
this.inputType = "ArrayBuffer";
this.outputType = "html";
this.args = [
{
"name": "Depth",
"type": "number",
"value": 3
},
{
"name": "Intensive mode",
"type": "boolean",
"value": false
},
{
"name": "Extensive language support",
"type": "boolean",
"value": false
}
];
}
/**
* @param {Object} state - The current state of the recipe.
* @param {number} state.progress - The current position in the recipe.
* @param {Dish} state.dish - The Dish being operated on.
* @param {Operation[]} state.opList - The list of operations in the recipe.
* @returns {Object} The updated state of the recipe.
*/
async run(state) {
const ings = state.opList[state.progress].ingValues,
[depth, intensive, extLang] = ings,
dish = state.dish,
currentRecipeConfig = state.opList.map(op => op.config),
magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)),
options = await magic.speculativeExecution(depth, extLang, intensive);
let output = `<table
class='table table-hover table-condensed table-bordered'
style='table-layout: fixed;'>
<tr>
<th>Recipe (click to load)</th>
<th>Result snippet</th>
<th>Properties</th>
</tr>`;
/**
* Returns a CSS colour value based on an integer input.
*
* @param {number} val
* @returns {string}
*/
function chooseColour(val) {
if (val < 3) return "green";
if (val < 5) return "goldenrod";
return "red";
}
options.forEach(option => {
// Construct recipe URL
// Replace this Magic op with the generated recipe
const recipeConfig = currentRecipeConfig.slice(0, state.progress)
.concat(option.recipe)
.concat(currentRecipeConfig.slice(state.progress + 1)),
recipeURL = "recipe=" + Utils.encodeURIFragment(Utils.generatePrettyRecipe(recipeConfig));
let language = "",
fileType = "",
matchingOps = "",
useful = "";
const entropy = `<span data-toggle="tooltip" data-container="body" title="Shannon Entropy is measured from 0 to 8. High entropy suggests encrypted or compressed data. Normal text is usually around 3.5 to 5.">Entropy: <span style="color: ${chooseColour(option.entropy)}">${option.entropy.toFixed(2)}</span></span>`,
validUTF8 = option.isUTF8 ? "<span data-toggle='tooltip' data-container='body' title='The data could be a valid UTF8 string based on its encoding.'>Valid UTF8</span>\n" : "";
if (option.languageScores[0].probability > 0) {
let likelyLangs = option.languageScores.filter(l => l.probability > 0);
if (likelyLangs.length < 1) likelyLangs = [option.languageScores[0]];
language = "<span data-toggle='tooltip' data-container='body' title='Based on a statistical comparison of the frequency of bytes in various languages. Ordered by likelihood.'>" +
"Possible languages:\n " +
likelyLangs.map(lang => {
return MagicLib.codeToLanguage(lang.lang);
}).join("\n ") +
"</span>\n";
}
if (option.fileType) {
fileType = `<span data-toggle="tooltip" data-container="body" title="Based on the presence of magic bytes.">File type: ${option.fileType.mime} (${option.fileType.ext})</span>\n`;
}
if (option.matchingOps.length) {
matchingOps = `Matching ops: ${[...new Set(option.matchingOps.map(op => op.op))].join(", ")}\n`;
}
if (option.useful) {
useful = "<span data-toggle='tooltip' data-container='body' title='This could be an operation that displays data in a useful way, such as rendering an image.'>Useful op detected</span>\n";
}
output += `<tr>
<td><a href="#${recipeURL}">${Utils.generatePrettyRecipe(option.recipe, true)}</a></td>
<td>${Utils.escapeHtml(Utils.printable(Utils.truncate(option.data, 99)))}</td>
<td>${language}${fileType}${matchingOps}${useful}${validUTF8}${entropy}</td>
</tr>`;
});
output += "</table><script type='application/javascript'>$('[data-toggle=\"tooltip\"]').tooltip()</script>";
if (!options.length) {
output = "Nothing of interest could be detected about the input data.\nHave you tried modifying the operation arguments?";
}
dish.set(output, Dish.HTML);
return state;
}
}
export default Magic;

85
src/core/operations/ScanForEmbeddedFiles.mjs Normal file
View File

@ -0,0 +1,85 @@
/**
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2016
* @license Apache-2.0
*/
import Operation from "../Operation";
import Utils from "../Utils";
import Magic from "../lib/Magic";
/**
* Scan for Embedded Files operation
*/
class ScanForEmbeddedFiles extends Operation {
/**
* ScanForEmbeddedFiles constructor
*/
constructor() {
super();
this.name = "Scan for Embedded Files";
this.module = "Default";
this.description = "Scans the data for potential embedded files by looking for magic bytes at all offsets. This operation is prone to false positives.<br><br>WARNING: Files over about 100KB in size will take a VERY long time to process.";
this.inputType = "ArrayBuffer";
this.outputType = "string";
this.args = [
{
"name": "Ignore common byte sequences",
"type": "boolean",
"value": true
}
];
}
/**
* @param {ArrayBuffer} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
let output = "Scanning data for 'magic bytes' which may indicate embedded files. The following results may be false positives and should not be treat as reliable. Any suffiently long file is likely to contain these magic bytes coincidentally.\n",
type,
numFound = 0,
numCommonFound = 0;
const ignoreCommon = args[0],
commonExts = ["ico", "ttf", ""],
data = new Uint8Array(input);
for (let i = 0; i < data.length; i++) {
type = Magic.magicFileType(data.slice(i));
if (type) {
if (ignoreCommon && commonExts.indexOf(type.ext) > -1) {
numCommonFound++;
continue;
}
numFound++;
output += "\nOffset " + i + " (0x" + Utils.hex(i) + "):\n" +
" File extension: " + type.ext + "\n" +
" MIME type: " + type.mime + "\n";
if (type.desc && type.desc.length) {
output += " Description: " + type.desc + "\n";
}
}
}
if (numFound === 0) {
output += "\nNo embedded files were found.";
}
if (numCommonFound > 0) {
output += "\n\n" + numCommonFound;
output += numCommonFound === 1 ?
" file type was detected that has a common byte sequence. This is likely to be a false positive." :
" file types were detected that have common byte sequences. These are likely to be false positives.";
output += " Run this operation with the 'Ignore common byte sequences' option unchecked to see details.";
}
return output;
}
}
export default ScanForEmbeddedFiles;

7
src/core/operations/URLDecode.mjs
View File

@ -23,6 +23,13 @@ class URLDecode extends Operation {
this.inputType = "string";
this.outputType = "string";
this.args = [];
this.patterns = [
{
match: ".*(?:%[\\da-f]{2}.*){4}",
flags: "i",
args: []
},
];
}
/**

7
src/core/operations/Unzip.mjs
View File

@ -39,6 +39,13 @@ class Unzip extends Operation {
value: false
}
];
this.patterns = [
{
match: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)",
flags: "",
args: ["", false]
},
];
}
/**

7
src/core/operations/ZlibInflate.mjs
View File

@ -58,6 +58,13 @@ class ZlibInflate extends Operation {
value: false
}
];
this.patterns = [
{
match: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)",
flags: "",
args: [0, 0, "Adaptive", false, false]
},
];
}
/**

26
src/core/operations/legacy/FileType.js
View File

@ -225,8 +225,8 @@ const FileType = {
if (buf[0] === 0x78 && buf[1] === 0x01) {
return {
ext: "dmg",
mime: "application/x-apple-diskimage"
ext: "dmg, zlib",
mime: "application/x-apple-diskimage, application/x-deflate"
};
}
@ -434,8 +434,11 @@ const FileType = {
};
}
// Added by n1474335 [n1474335@gmail.com] from here on
// ################################################################## //
/**
*
* Added by n1474335 [n1474335@gmail.com] from here on
*
*/
if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) {
return {
ext: "z, tar.z",
@ -469,16 +472,16 @@ const FileType = {
// Must be before Little-endian UTF-16 BOM
if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) {
return {
ext: "",
mime: "",
ext: "UTF32LE",
mime: "charset/utf32le",
desc: "Little-endian UTF-32 encoded Unicode byte order mark detected."
};
}
if (buf[0] === 0xFF && buf[1] === 0xFE) {
return {
ext: "",
mime: "",
ext: "UTF16LE",
mime: "charset/utf16le",
desc: "Little-endian UTF-16 encoded Unicode byte order mark detected."
};
}
@ -524,6 +527,13 @@ const FileType = {
};
}
if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) {
return {
ext: "zlib",
mime: "application/x-deflate"
};
}
return null;
},

103
src/core/operations/legacy/FlowControl.js
View File

@ -1,5 +1,7 @@
import Recipe from "./Recipe";
import Dish from "./Dish";
import Recipe from "./Recipe.js";
import Dish from "./Dish.js";
import Magic from "./lib/Magic.js";
import Utils from "./Utils.js";
/**
@ -183,7 +185,7 @@ const FlowControl = {
* @returns {Object} The updated state of the recipe.
*/
runJump: function(state) {
const ings = state.opList[state.progress].ingValues,
const ings = state.opList[state.progress].ingValues,
label = ings[0],
maxJumps = ings[1],
jmpIndex = FlowControl._getLabelIndex(label, state);
@ -211,7 +213,7 @@ const FlowControl = {
* @returns {Object} The updated state of the recipe.
*/
runCondJump: async function(state) {
const ings = state.opList[state.progress].ingValues,
const ings = state.opList[state.progress].ingValues,
dish = state.dish,
regexStr = ings[0],
invert = ings[1],
@ -266,6 +268,99 @@ const FlowControl = {
},
/**
* Magic operation.
*
* @param {Object} state - The current state of the recipe.
* @param {number} state.progress - The current position in the recipe.
* @param {Dish} state.dish - The Dish being operated on.
* @param {Operation[]} state.opList - The list of operations in the recipe.
* @returns {Object} The updated state of the recipe.
*/
runMagic: async function(state) {
const ings = state.opList[state.progress].ingValues,
depth = ings[0],
intensive = ings[1],
extLang = ings[2],
dish = state.dish,
currentRecipeConfig = state.opList.map(op => op.getConfig()),
magic = new Magic(dish.get(Dish.ARRAY_BUFFER)),
options = await magic.speculativeExecution(depth, extLang, intensive);
let output = `<table
class='table table-hover table-condensed table-bordered'
style='table-layout: fixed;'>
<tr>
<th>Recipe (click to load)</th>
<th>Result snippet</th>
<th>Properties</th>
</tr>`;
/**
* Returns a CSS colour value based on an integer input.
*
* @param {number} val
* @returns {string}
*/
function chooseColour(val) {
if (val < 3) return "green";
if (val < 5) return "goldenrod";
return "red";
}
options.forEach(option => {
// Construct recipe URL
// Replace this Magic op with the generated recipe
const recipeConfig = currentRecipeConfig.slice(0, state.progress)
.concat(option.recipe)
.concat(currentRecipeConfig.slice(state.progress + 1)),
recipeURL = "recipe=" + Utils.encodeURIFragment(Utils.generatePrettyRecipe(recipeConfig));
let language = "",
fileType = "",
matchingOps = "",
useful = "",
entropy = `<span data-toggle="tooltip" data-container="body" title="Shannon Entropy is measured from 0 to 8. High entropy suggests encrypted or compressed data. Normal text is usually around 3.5 to 5.">Entropy: <span style="color: ${chooseColour(option.entropy)}">${option.entropy.toFixed(2)}</span></span>`,
validUTF8 = option.isUTF8 ? "<span data-toggle='tooltip' data-container='body' title='The data could be a valid UTF8 string based on its encoding.'>Valid UTF8</span>\n" : "";
if (option.languageScores[0].probability > 0) {
let likelyLangs = option.languageScores.filter(l => l.probability > 0);
if (likelyLangs.length < 1) likelyLangs = [option.languageScores[0]];
language = "<span data-toggle='tooltip' data-container='body' title='Based on a statistical comparison of the frequency of bytes in various languages. Ordered by likelihood.'>" +
"Possible languages:\n " + likelyLangs.map(lang => {
return Magic.codeToLanguage(lang.lang);
}).join("\n ") + "</span>\n";
}
if (option.fileType) {
fileType = `<span data-toggle="tooltip" data-container="body" title="Based on the presence of magic bytes.">File type: ${option.fileType.mime} (${option.fileType.ext})</span>\n`;
}
if (option.matchingOps.length) {
matchingOps = `Matching ops: ${[...new Set(option.matchingOps.map(op => op.op))].join(", ")}\n`;
}
if (option.useful) {
useful = "<span data-toggle='tooltip' data-container='body' title='This could be an operation that displays data in a useful way, such as rendering an image.'>Useful op detected</span>\n";
}
output += `<tr>
<td><a href="#${recipeURL}">${Utils.generatePrettyRecipe(option.recipe, true)}</a></td>
<td>${Utils.escapeHtml(Utils.printable(Utils.truncate(option.data, 99)))}</td>
<td>${language}${fileType}${matchingOps}${useful}${validUTF8}${entropy}</td>
</tr>`;
});
output += "</table><script type='application/javascript'>$('[data-toggle=\"tooltip\"]').tooltip()</script>";
if (!options.length) {
output = "Nothing of interest could be detected about the input data.\nHave you tried modifying the operation arguments?";
}
dish.set(output, Dish.HTML);
return state;
},
/**
* Returns the index of a label.
*

6
src/core/operations/legacy/PublicKey.js
View File

@ -62,7 +62,7 @@ const PublicKey = {
pkStr = "",
sig = cert.getSignatureValueHex(),
sigStr = "",
extensions = cert.getInfo().split("X509v3 Extensions:\n")[1].split("signature")[0];
extensions = "";
// Public Key fields
pkFields.push({
@ -144,6 +144,10 @@ const PublicKey = {
sigStr = " Signature: " + PublicKey._formatByteStr(sig, 16, 18);
}
// Extensions
try {
extensions = cert.getInfo().split("X509v3 Extensions:\n")[1].split("signature")[0];
} catch (err) {}
let issuerStr = PublicKey._formatDnStr(issuer, 2),
nbDate = PublicKey._formatDate(cert.getNotBefore()),

3
src/web/index.js
View File

@ -35,7 +35,8 @@ function main() {
"URL Decode",
"Regular expression",
"Entropy",
"Fork"
"Fork",
"Magic"
];
const defaultOptions = {