Mirror/CyberChef
1
0
Fork 0
mirror of https://github.com/gchq/CyberChef.git synced 2024-11-02 14:11:02 +01:00
Code Issues Packages Projects Releases Wiki Activity

Magic operation now brute forces character encodings. Linted.

Browse Source
This commit is contained in:
n1474335 2018-02-15 17:38:39 +00:00
parent 27ec4aa923
commit b3c52a8601
2 changed files with 96 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/core/config/OperationConfig.js
View File

@ -83,7 +83,7 @@ import URL_ from "../operations/URL.js";
const OperationConfig = {
"Magic": {
module: "Default",
description: "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various encodings like XOR and bit rotates are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.",
description: "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.",
inputType: "ArrayBuffer",
outputType: "html",
flowControl: true,
@ -1381,7 +1381,7 @@ const OperationConfig = {
type: "option",
value: Object.keys(CharEnc.IO_FORMAT),
},
]
],
},
"Decode text": {
module: "CharEnc",

99
src/core/lib/Magic.js
View File

@ -178,7 +178,7 @@ class Magic {
*
* @returns {Object[]} - The encoded data and an operation config to generate it.
*/
bruteForce() {
async bruteForce() {
const sample = new Uint8Array(this.inputBuffer).slice(0, 100);
let results = [];
@ -205,6 +205,32 @@ class Magic {
});
}
// Character encodings
const encodings = OperationConfig["Encode text"].args[0].value;
/**
* Test character encodings and add them if they change the data.
*/
const testEnc = async op => {
for (let i = 0; i < encodings.length; i++) {
const conf = {
op: "Encode text",
args: [encodings[i]]
},
data = await this._runRecipe([conf], sample.buffer);
// Only add to the results if it changed the data
if (!_buffersEqual(data, sample.buffer)) {
results.push({
data: data,
conf: conf
});
}
}
};
await testEnc("Encode text");
await testEnc("Decode text");
return results;
}
@ -243,20 +269,12 @@ class Magic {
// Execute each of the matching operations, then recursively call the speculativeExecution()
// method on the resulting data, recording the properties of each option.
await Promise.all(matchingOps.map(async op => {
const dish = new Dish(this.inputBuffer, Dish.ARRAY_BUFFER),
opConfig = {
const opConfig = {
op: op.op,
args: op.args
};
if (ENVIRONMENT_IS_WORKER()) self.loadRequiredModules([opConfig]);
const recipe = new Recipe([opConfig]);
try {
await recipe.execute(dish, 0);
} catch (err) {} // Ignore errors
const magic = new Magic(dish.get(Dish.ARRAY_BUFFER), this.opPatterns),
},
output = await this._runRecipe([opConfig]),
magic = new Magic(output, this.opPatterns),
speculativeResults = await magic.speculativeExecution(
depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful);
@ -265,7 +283,7 @@ class Magic {
if (intensive) {
// Run brute forcing of various types on the data and create a new branch for each option
const bfEncodings = this.bruteForce();
const bfEncodings = await this.bruteForce();
await Promise.all(bfEncodings.map(async enc => {
const magic = new Magic(enc.data, this.opPatterns),
@ -302,10 +320,34 @@ class Magic {
if (a.useful) aScore = 100;
if (b.useful) bScore = 100;
// Shorter recipes are better, so we add the length of the recipe to the score
aScore += a.recipe.length;
bScore += b.recipe.length;
return aScore - bScore;
});
}
/**
* Runs the given recipe over the input buffer and returns the output.
*
* @param {Object[]} recipeConfig
* @param {ArrayBuffer} [input=this.inputBuffer]
* @returns {ArrayBuffer}
*/
async _runRecipe(recipeConfig, input=this.inputBuffer) {
const dish = new Dish(input, Dish.ARRAY_BUFFER);
if (ENVIRONMENT_IS_WORKER()) self.loadRequiredModules(recipeConfig);
const recipe = new Recipe(recipeConfig);
try {
await recipe.execute(dish, 0);
} catch (err) {} // Ignore errors
return dish.get(Dish.ARRAY_BUFFER);
}
/**
* Calculates the number of times each byte appears in the input
*
@ -991,4 +1033,33 @@ const EXTENSIVE_LANG_FREQS = Object.assign({}, COMMON_LANG_FREQS, {
"zu": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.261, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 10.94, 0.004, 0.267, 0.001, 0.002, 0.016, 0.003, 0.041, 0.181, 0.181, 0.001, 0.001, 0.907, 0.49, 0.797, 0.099, 0.343, 0.379, 0.279, 0.134, 0.118, 0.116, 0.102, 0.097, 0.11, 0.223, 0.065, 0.035, 0.134, 0.003, 0.135, 0.005, 0.0001, 0.296, 0.147, 0.142, 0.093, 0.11, 0.08, 0.077, 0.065, 0.3, 0.114, 0.141, 0.151, 0.361, 0.387, 0.067, 0.128, 0.012, 0.082, 0.239, 0.152, 0.188, 0.039, 0.182, 0.012, 0.045, 0.07, 0.138, 0.0001, 0.139, 0.0001, 0.001, 0.0001, 10.325, 2.215, 0.829, 1.627, 7.521, 0.687, 2.042, 3.525, 7.719, 0.199, 3.874, 4.421, 2.406, 6.494, 4.881, 0.951, 0.342, 1.361, 3.011, 2.552, 4.691, 0.394, 2.227, 0.134, 1.688, 1.779, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.08, 0.007, 0.001, 0.001, 0.002, 0.0001, 0.014, 0.002, 0.002, 0.001, 0.0001, 0.001, 0.003, 0.005, 0.001, 0.002, 0.002, 0.014, 0.001, 0.01, 0.003, 0.0001, 0.001, 0.001, 0.002, 0.06, 0.0001, 0.001, 0.003, 0.003, 0.001, 0.0001, 0.084, 0.004, 0.0001, 0.001, 0.001, 0.0001, 0.002, 0.004, 0.002, 0.005, 0.003, 0.001, 0.001, 0.002, 0.001, 0.0001, 0.004, 0.003, 0.003, 0.005, 0.002, 0.002, 0.001, 0.006, 0.005, 0.002, 0.003, 0.005, 0.002, 0.003, 0.003, 0.0001, 0.0001, 0.0001, 0.089, 0.024, 0.004, 0.007, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.0001, 0.0001, 0.002, 0.001, 0.029, 0.011, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.005, 0.002, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.002, 0.091, 0.001, 0.0001, 0.001, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
});
/**
* Determines whether two ArrayBuffers contain the same values.
*
* @param {ArrayBuffer} a
* @param {ArrayBuffer} b
* @returns {boolean}
*/
function _buffersEqual(a, b) {
if (a === b) {
return true;
}
if (a.byteLength !== b.byteLength) {
return false;
}
const ai = new Uint8Array(a),
bi = new Uint8Array(b);
let i = a.byteLength;
while (i--) {
if (ai[i] !== bi[i]) {
return false;
}
}
return true;
}
export default Magic;