mirror of https://github.com/gchq/CyberChef.git
Magic operation now brute forces character encodings. Linted.
This commit is contained in:
n1474335
parent
27ec4aa923
commit
b3c52a8601
|
|
@ -83,7 +83,7 @@ import URL_ from "../operations/URL.js";
|
|||
const OperationConfig = {
|
||||
"Magic": {
|
||||
module: "Default",
|
||||
description: "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various encodings like XOR and bit rotates are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.",
|
||||
description: "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.",
|
||||
inputType: "ArrayBuffer",
|
||||
outputType: "html",
|
||||
flowControl: true,
|
||||
|
|
@ -1381,7 +1381,7 @@ const OperationConfig = {
|
|||
type: "option",
|
||||
value: Object.keys(CharEnc.IO_FORMAT),
|
||||
},
|
||||
]
|
||||
],
|
||||
},
|
||||
"Decode text": {
|
||||
module: "CharEnc",
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ class Magic {
|
|||
|
||||
/**
|
||||
* Detects any matching file types for the input.
|
||||
*
|
||||
*
|
||||
* @returns {Object} type
|
||||
* @returns {string} type.ext - File extension
|
||||
* @returns {string} type.mime - Mime type
|
||||
|
|
@ -178,7 +178,7 @@ class Magic {
|
|||
*
|
||||
* @returns {Object[]} - The encoded data and an operation config to generate it.
|
||||
*/
|
||||
bruteForce() {
|
||||
async bruteForce() {
|
||||
const sample = new Uint8Array(this.inputBuffer).slice(0, 100);
|
||||
|
||||
let results = [];
|
||||
|
|
@ -205,13 +205,39 @@ class Magic {
|
|||
});
|
||||
}
|
||||
|
||||
// Character encodings
|
||||
const encodings = OperationConfig["Encode text"].args[0].value;
|
||||
|
||||
/**
|
||||
* Test character encodings and add them if they change the data.
|
||||
*/
|
||||
const testEnc = async op => {
|
||||
for (let i = 0; i < encodings.length; i++) {
|
||||
const conf = {
|
||||
op: "Encode text",
|
||||
args: [encodings[i]]
|
||||
},
|
||||
data = await this._runRecipe([conf], sample.buffer);
|
||||
|
||||
// Only add to the results if it changed the data
|
||||
if (!_buffersEqual(data, sample.buffer)) {
|
||||
results.push({
|
||||
data: data,
|
||||
conf: conf
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await testEnc("Encode text");
|
||||
await testEnc("Decode text");
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Speculatively executes matching operations, recording metadata of each result.
|
||||
*
|
||||
*
|
||||
* @param {number} [depth=0] - How many levels to try to execute
|
||||
* @param {boolean} [extLang=false] - Extensive language support (false = only check the most
|
||||
* common Internet languages)
|
||||
|
|
@ -243,29 +269,21 @@ class Magic {
|
|||
// Execute each of the matching operations, then recursively call the speculativeExecution()
|
||||
// method on the resulting data, recording the properties of each option.
|
||||
await Promise.all(matchingOps.map(async op => {
|
||||
const dish = new Dish(this.inputBuffer, Dish.ARRAY_BUFFER),
|
||||
opConfig = {
|
||||
const opConfig = {
|
||||
op: op.op,
|
||||
args: op.args
|
||||
};
|
||||
|
||||
if (ENVIRONMENT_IS_WORKER()) self.loadRequiredModules([opConfig]);
|
||||
|
||||
const recipe = new Recipe([opConfig]);
|
||||
try {
|
||||
await recipe.execute(dish, 0);
|
||||
} catch (err) {} // Ignore errors
|
||||
|
||||
const magic = new Magic(dish.get(Dish.ARRAY_BUFFER), this.opPatterns),
|
||||
},
|
||||
output = await this._runRecipe([opConfig]),
|
||||
magic = new Magic(output, this.opPatterns),
|
||||
speculativeResults = await magic.speculativeExecution(
|
||||
depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful);
|
||||
depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful);
|
||||
|
||||
results = results.concat(speculativeResults);
|
||||
}));
|
||||
|
||||
if (intensive) {
|
||||
// Run brute forcing of various types on the data and create a new branch for each option
|
||||
const bfEncodings = this.bruteForce();
|
||||
const bfEncodings = await this.bruteForce();
|
||||
|
||||
await Promise.all(bfEncodings.map(async enc => {
|
||||
const magic = new Magic(enc.data, this.opPatterns),
|
||||
|
|
@ -278,11 +296,11 @@ class Magic {
|
|||
|
||||
// Prune branches that do not match anything
|
||||
results = results.filter(r =>
|
||||
r.languageScores[0].probability > 0 ||
|
||||
r.fileType ||
|
||||
r.isUTF8 ||
|
||||
r.matchingOps.length ||
|
||||
r.useful);
|
||||
r.languageScores[0].probability > 0 ||
|
||||
r.fileType ||
|
||||
r.isUTF8 ||
|
||||
r.matchingOps.length ||
|
||||
r.useful);
|
||||
|
||||
// Return a sorted list of possible recipes along with their properties
|
||||
return results.sort((a, b) => {
|
||||
|
|
@ -302,10 +320,34 @@ class Magic {
|
|||
if (a.useful) aScore = 100;
|
||||
if (b.useful) bScore = 100;
|
||||
|
||||
// Shorter recipes are better, so we add the length of the recipe to the score
|
||||
aScore += a.recipe.length;
|
||||
bScore += b.recipe.length;
|
||||
|
||||
return aScore - bScore;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the given recipe over the input buffer and returns the output.
|
||||
*
|
||||
* @param {Object[]} recipeConfig
|
||||
* @param {ArrayBuffer} [input=this.inputBuffer]
|
||||
* @returns {ArrayBuffer}
|
||||
*/
|
||||
async _runRecipe(recipeConfig, input=this.inputBuffer) {
|
||||
const dish = new Dish(input, Dish.ARRAY_BUFFER);
|
||||
|
||||
if (ENVIRONMENT_IS_WORKER()) self.loadRequiredModules(recipeConfig);
|
||||
|
||||
const recipe = new Recipe(recipeConfig);
|
||||
try {
|
||||
await recipe.execute(dish, 0);
|
||||
} catch (err) {} // Ignore errors
|
||||
|
||||
return dish.get(Dish.ARRAY_BUFFER);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the number of times each byte appears in the input
|
||||
*
|
||||
|
|
@ -952,7 +994,7 @@ const EXTENSIVE_LANG_FREQS = Object.assign({}, COMMON_LANG_FREQS, {
|
|||
"sw": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.454, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 14.93, 0.002, 0.217, 0.002, 0.001, 0.01, 0.003, 0.027, 0.171, 0.171, 0.001, 0.001, 0.703, 0.109, 0.942, 0.015, 0.41, 0.383, 0.266, 0.126, 0.108, 0.126, 0.108, 0.107, 0.119, 0.201, 0.062, 0.024, 0.003, 0.004, 0.003, 0.003, 0.0001, 0.226, 0.167, 0.122, 0.086, 0.058, 0.057, 0.065, 0.116, 0.13, 0.09, 0.638, 0.08, 0.504, 0.137, 0.044, 0.113, 0.006, 0.074, 0.173, 0.147, 0.165, 0.059, 0.218, 0.013, 0.04, 0.023, 0.04, 0.0001, 0.04, 0.001, 0.001, 0.001, 16.478, 1.326, 0.611, 1.343, 3.374, 0.678, 1.131, 2.383, 9.629, 0.827, 4.598, 2.609, 3.253, 5.284, 3.187, 0.805, 0.008, 1.616, 2.094, 2.468, 4.443, 0.427, 3.161, 0.026, 2.095, 1.273, 0.001, 0.006, 0.001, 0.0001, 0.0001, 0.04, 0.005, 0.004, 0.002, 0.004, 0.002, 0.002, 0.002, 0.002, 0.002, 0.002, 0.001, 0.002, 0.001, 0.001, 0.001, 0.002, 0.001, 0.001, 0.013, 0.002, 0.001, 0.001, 0.001, 0.002, 0.006, 0.001, 0.001, 0.009, 0.008, 0.001, 0.004, 0.009, 0.003, 0.002, 0.002, 0.003, 0.001, 0.001, 0.005, 0.003, 0.009, 0.001, 0.002, 0.001, 0.002, 0.001, 0.002, 0.005, 0.009, 0.009, 0.004, 0.002, 0.003, 0.004, 0.001, 0.004, 0.003, 0.003, 0.003, 0.006, 0.003, 0.002, 0.003, 0.0001, 0.0001, 0.018, 0.029, 0.009, 0.005, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.014, 0.007, 0.011, 0.004, 0.0001, 0.0001, 0.0001, 0.002, 0.002, 0.005, 0.012, 0.01, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.002, 0.004, 0.038, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
|
||||
"szl": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.884, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 12.63, 0.002, 0.452, 0.0001, 0.0001, 0.012, 0.001, 0.026, 0.296, 0.296, 0.001, 0.001, 1.094, 0.318, 1.181, 0.015, 0.332, 0.469, 0.289, 0.138, 0.131, 0.151, 0.118, 0.131, 0.157, 0.273, 0.087, 0.014, 0.006, 0.003, 0.006, 0.0001, 0.0001, 0.207, 0.209, 0.155, 0.118, 0.048, 0.111, 0.139, 0.08, 0.122, 0.125, 0.213, 0.123, 0.287, 0.122, 0.062, 0.309, 0.005, 0.156, 0.329, 0.126, 0.154, 0.05, 0.233, 0.034, 0.017, 0.083, 0.004, 0.0001, 0.004, 0.0001, 0.006, 0.001, 5.741, 0.894, 2.016, 2.128, 5.35, 0.327, 1.279, 0.968, 3.438, 2.841, 2.633, 2.099, 2.293, 3.364, 5.857, 1.423, 0.012, 3.389, 2.85, 2.58, 2.277, 0.102, 3.144, 0.017, 3.623, 2.205, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.191, 0.035, 0.624, 0.044, 0.945, 0.014, 0.009, 0.333, 0.008, 0.003, 0.006, 0.005, 0.012, 0.221, 0.005, 0.196, 0.006, 0.005, 0.003, 0.168, 0.01, 0.003, 0.005, 0.005, 0.005, 0.109, 0.059, 0.562, 0.005, 0.005, 0.004, 0.006, 0.062, 0.111, 0.006, 0.016, 0.01, 0.004, 0.004, 0.012, 0.011, 0.03, 0.005, 0.012, 0.003, 0.012, 0.008, 1.67, 0.032, 0.015, 0.058, 0.035, 0.048, 0.018, 0.012, 0.004, 0.02, 0.013, 0.335, 0.026, 0.282, 0.022, 0.098, 0.006, 0.0001, 0.0001, 0.109, 0.208, 0.455, 5.073, 0.0001, 0.001, 0.0001, 0.008, 0.003, 0.003, 0.004, 0.0001, 0.015, 0.008, 0.161, 0.06, 0.003, 0.002, 0.0001, 0.003, 0.001, 0.009, 0.025, 0.019, 0.0001, 0.001, 0.0001, 0.0001, 0.002, 0.0001, 0.011, 0.01, 0.176, 0.006, 0.001, 0.005, 0.003, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
|
||||
"ta": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.357, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 3.862, 0.001, 0.077, 0.0001, 0.001, 0.006, 0.001, 0.007, 0.055, 0.056, 0.0001, 0.001, 0.234, 0.03, 0.384, 0.005, 0.084, 0.106, 0.063, 0.029, 0.028, 0.034, 0.027, 0.032, 0.031, 0.052, 0.017, 0.006, 0.002, 0.002, 0.002, 0.001, 0.0001, 0.008, 0.004, 0.008, 0.004, 0.004, 0.003, 0.005, 0.004, 0.006, 0.002, 0.003, 0.003, 0.005, 0.004, 0.003, 0.008, 0.0001, 0.004, 0.008, 0.005, 0.002, 0.002, 0.002, 0.001, 0.001, 0.0001, 0.006, 0.0001, 0.006, 0.0001, 0.002, 0.0001, 0.062, 0.006, 0.017, 0.014, 0.042, 0.007, 0.009, 0.018, 0.038, 0.001, 0.006, 0.024, 0.018, 0.035, 0.032, 0.011, 0.001, 0.036, 0.022, 0.032, 0.017, 0.005, 0.004, 0.002, 0.01, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.122, 2.149, 0.144, 0.01, 0.0001, 0.297, 0.436, 0.597, 0.764, 0.136, 0.24, 0.226, 0.005, 5.298, 0.158, 0.027, 0.013, 0.0001, 0.078, 0.014, 0.0001, 2.36, 0.0001, 0.0001, 0.001, 0.171, 0.627, 0.0001, 0.037, 0.002, 0.021, 1.319, 0.014, 0.0001, 0.001, 0.32, 2.012, 0.001, 0.001, 0.0001, 0.539, 0.989, 1.521, 0.0001, 0.0001, 0.001, 23.215, 10.185, 1.322, 0.801, 1.028, 0.757, 0.189, 0.942, 0.0001, 0.015, 0.06, 0.015, 0.0001, 0.0001, 0.0001, 0.0001, 1.18, 2.177, 0.0001, 0.0001, 0.016, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 31.245, 0.0001, 0.013, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
|
||||
"tcy": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.391, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 4.751, 0.001, 0.026, 0.0001, 0.0001, 0.002, 0.0001, 0.028, 0.048, 0.047, 0.0001, 0.001, 0.244, 0.028, 0.533, 0.012, 0.014, 0.02, 0.01, 0.005, 0.005, 0.007, 0.006, 0.004, 0.008, 0.009, 0.009, 0.003, 0.002, 0.003, 0.002, 0.002, 0.0001, 0.002, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.001, 0.0001, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.0001, 0.003, 0.0001, 0.001, 0.001, 0.02, 0.002, 0.008, 0.006, 0.018, 0.002, 0.005, 0.006, 0.017, 0.0001, 0.003, 0.009, 0.008, 0.014, 0.015, 0.008, 0.0001, 0.013, 0.012, 0.015, 0.006, 0.002, 0.005, 0.0001, 0.003, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.354, 1.789, 1.221, 0.031, 0.0001, 0.268, 1.686, 0.484, 0.152, 0.21, 0.745, 0.196, 0.087, 4.125, 0.064, 0.014, 0.014, 0.0001, 0.109, 0.011, 0.001, 1.28, 0.033, 0.613, 0.012, 0.007, 0.23, 0.003, 0.404, 0.002, 0.011, 0.433, 0.058, 1.007, 0.002, 0.198, 1.312, 0.064, 1.397, 0.124, 1.439, 0.012, 1.248, 0.035, 0.624, 0.105, 0.769, 0.62, 1.755, 0.0001, 22.872, 9.408, 0.0001, 0.629, 0.164, 0.121, 0.665, 0.124, 0.0001, 0.0001, 0.003, 0.0001, 1.377, 1.63, 0.0001, 0.0001, 0.05, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 30.955, 0.0001, 0.194, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
|
||||
"tcy": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.391, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 4.751, 0.001, 0.026, 0.0001, 0.0001, 0.002, 0.0001, 0.028, 0.048, 0.047, 0.0001, 0.001, 0.244, 0.028, 0.533, 0.012, 0.014, 0.02, 0.01, 0.005, 0.005, 0.007, 0.006, 0.004, 0.008, 0.009, 0.009, 0.003, 0.002, 0.003, 0.002, 0.002, 0.0001, 0.002, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.001, 0.0001, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.0001, 0.003, 0.0001, 0.001, 0.001, 0.02, 0.002, 0.008, 0.006, 0.018, 0.002, 0.005, 0.006, 0.017, 0.0001, 0.003, 0.009, 0.008, 0.014, 0.015, 0.008, 0.0001, 0.013, 0.012, 0.015, 0.006, 0.002, 0.005, 0.0001, 0.003, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.354, 1.789, 1.221, 0.031, 0.0001, 0.268, 1.686, 0.484, 0.152, 0.21, 0.745, 0.196, 0.087, 4.125, 0.064, 0.014, 0.014, 0.0001, 0.109, 0.011, 0.001, 1.28, 0.033, 0.613, 0.012, 0.007, 0.23, 0.003, 0.404, 0.002, 0.011, 0.433, 0.058, 1.007, 0.002, 0.198, 1.312, 0.064, 1.397, 0.124, 1.439, 0.012, 1.248, 0.035, 0.624, 0.105, 0.769, 0.62, 1.755, 0.0001, 22.872, 9.408, 0.0001, 0.629, 0.164, 0.121, 0.665, 0.124, 0.0001, 0.0001, 0.003, 0.0001, 1.377, 1.63, 0.0001, 0.0001, 0.05, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 30.955, 0.0001, 0.194, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
|
||||
"te": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.34, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 4.746, 0.003, 0.051, 0.0001, 0.001, 0.003, 0.002, 0.007, 0.042, 0.043, 0.0001, 0.001, 0.336, 0.028, 0.611, 0.018, 0.129, 0.152, 0.069, 0.038, 0.034, 0.073, 0.03, 0.032, 0.034, 0.047, 0.02, 0.007, 0.002, 0.004, 0.002, 0.002, 0.0001, 0.008, 0.004, 0.006, 0.005, 0.003, 0.003, 0.002, 0.002, 0.006, 0.001, 0.002, 0.003, 0.005, 0.003, 0.002, 0.005, 0.0001, 0.003, 0.008, 0.005, 0.003, 0.002, 0.002, 0.001, 0.0001, 0.0001, 0.006, 0.0001, 0.007, 0.0001, 0.005, 0.0001, 0.053, 0.008, 0.019, 0.022, 0.056, 0.009, 0.01, 0.021, 0.046, 0.001, 0.004, 0.022, 0.015, 0.038, 0.038, 0.014, 0.0001, 0.036, 0.036, 0.045, 0.017, 0.006, 0.006, 0.002, 0.007, 0.001, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.485, 1.801, 1.898, 0.051, 0.0001, 0.236, 0.427, 0.575, 0.238, 0.222, 0.152, 0.685, 0.105, 2.799, 0.055, 0.027, 0.006, 0.0001, 0.047, 0.007, 0.005, 1.329, 0.049, 0.668, 0.014, 0.002, 0.428, 0.004, 0.25, 0.001, 0.004, 0.537, 0.039, 0.598, 0.002, 0.137, 0.864, 0.099, 0.843, 0.149, 1.628, 0.0001, 0.909, 0.085, 0.267, 0.128, 0.942, 0.804, 25.531, 7.165, 1.487, 0.074, 0.0001, 0.877, 0.211, 0.153, 0.855, 0.145, 0.0001, 0.001, 0.0001, 0.0001, 2.169, 2.359, 0.0001, 0.0001, 0.014, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 30.736, 0.0001, 0.069, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
|
||||
"tet": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.506, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 14.056, 0.014, 0.345, 0.0001, 0.004, 0.018, 0.001, 0.455, 0.383, 0.382, 0.001, 0.004, 1.067, 0.53, 0.968, 0.029, 0.443, 0.39, 0.316, 0.132, 0.112, 0.137, 0.105, 0.106, 0.119, 0.181, 0.186, 0.018, 0.015, 0.005, 0.015, 0.003, 0.0001, 0.338, 0.226, 0.145, 0.169, 0.132, 0.156, 0.098, 0.111, 0.215, 0.061, 0.136, 0.43, 0.301, 0.181, 0.101, 0.266, 0.01, 0.137, 0.345, 0.37, 0.107, 0.065, 0.041, 0.021, 0.008, 0.014, 0.01, 0.0001, 0.01, 0.0001, 0.0001, 0.0001, 11.569, 1.502, 0.408, 2.068, 6.067, 0.587, 0.66, 2.225, 7.509, 0.16, 2.246, 2.814, 2.311, 6.307, 4.401, 1.282, 0.035, 4.022, 4.063, 3.545, 4.826, 0.518, 0.1, 0.064, 0.126, 0.341, 0.0001, 0.009, 0.0001, 0.0001, 0.0001, 0.318, 0.081, 0.003, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.001, 0.002, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.015, 0.001, 0.0001, 0.0001, 0.001, 0.004, 0.275, 0.001, 0.0001, 0.014, 0.013, 0.001, 0.002, 0.021, 0.254, 0.002, 0.025, 0.0001, 0.0001, 0.003, 0.02, 0.002, 0.389, 0.006, 0.001, 0.001, 0.167, 0.001, 0.001, 0.002, 0.048, 0.071, 0.284, 0.01, 0.003, 0.001, 0.0001, 0.001, 0.001, 0.076, 0.003, 0.014, 0.001, 0.001, 0.002, 0.0001, 0.0001, 0.1, 1.362, 0.004, 0.006, 0.0001, 0.0001, 0.0001, 0.009, 0.011, 0.0001, 0.0001, 0.0001, 0.007, 0.003, 0.006, 0.003, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.316, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
|
||||
"tg": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.272, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 7.893, 0.001, 0.026, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.324, 0.326, 0.0001, 0.001, 0.765, 0.105, 0.581, 0.006, 0.139, 0.257, 0.13, 0.073, 0.063, 0.072, 0.065, 0.068, 0.082, 0.185, 0.026, 0.048, 0.002, 0.001, 0.002, 0.001, 0.0001, 0.026, 0.01, 0.018, 0.007, 0.005, 0.01, 0.006, 0.007, 0.018, 0.002, 0.005, 0.008, 0.009, 0.006, 0.004, 0.009, 0.001, 0.007, 0.015, 0.007, 0.003, 0.006, 0.004, 0.006, 0.002, 0.002, 0.004, 0.0001, 0.004, 0.0001, 0.0001, 0.0001, 0.081, 0.01, 0.03, 0.023, 0.086, 0.012, 0.015, 0.021, 0.065, 0.002, 0.009, 0.037, 0.017, 0.055, 0.061, 0.017, 0.001, 0.07, 0.039, 0.054, 0.023, 0.01, 0.007, 0.003, 0.013, 0.003, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 2.968, 1.483, 1.764, 1.455, 0.398, 0.384, 0.008, 0.116, 0.704, 0.002, 0.17, 0.01, 0.024, 0.035, 0.045, 0.663, 0.178, 0.263, 0.119, 0.126, 0.303, 0.007, 0.009, 0.022, 0.136, 0.003, 0.143, 0.343, 0.148, 0.063, 0.071, 0.071, 0.134, 0.159, 0.101, 0.347, 0.121, 0.05, 0.002, 0.026, 0.059, 0.003, 0.003, 0.057, 0.003, 0.035, 0.012, 0.164, 5.899, 1.075, 1.071, 1.816, 2.336, 1.339, 0.082, 0.882, 4.885, 0.258, 1.014, 1.438, 1.445, 2.22, 3.885, 0.208, 0.0001, 0.0001, 0.132, 0.006, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.0001, 0.002, 0.001, 30.166, 10.131, 1.965, 0.481, 0.0001, 0.0001, 0.0001, 0.0001, 0.024, 0.016, 0.001, 0.006, 0.0001, 0.0001, 0.0001, 0.0001, 0.003, 0.001, 0.209, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
|
||||
|
|
@ -991,4 +1033,33 @@ const EXTENSIVE_LANG_FREQS = Object.assign({}, COMMON_LANG_FREQS, {
|
|||
"zu": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.261, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 10.94, 0.004, 0.267, 0.001, 0.002, 0.016, 0.003, 0.041, 0.181, 0.181, 0.001, 0.001, 0.907, 0.49, 0.797, 0.099, 0.343, 0.379, 0.279, 0.134, 0.118, 0.116, 0.102, 0.097, 0.11, 0.223, 0.065, 0.035, 0.134, 0.003, 0.135, 0.005, 0.0001, 0.296, 0.147, 0.142, 0.093, 0.11, 0.08, 0.077, 0.065, 0.3, 0.114, 0.141, 0.151, 0.361, 0.387, 0.067, 0.128, 0.012, 0.082, 0.239, 0.152, 0.188, 0.039, 0.182, 0.012, 0.045, 0.07, 0.138, 0.0001, 0.139, 0.0001, 0.001, 0.0001, 10.325, 2.215, 0.829, 1.627, 7.521, 0.687, 2.042, 3.525, 7.719, 0.199, 3.874, 4.421, 2.406, 6.494, 4.881, 0.951, 0.342, 1.361, 3.011, 2.552, 4.691, 0.394, 2.227, 0.134, 1.688, 1.779, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.08, 0.007, 0.001, 0.001, 0.002, 0.0001, 0.014, 0.002, 0.002, 0.001, 0.0001, 0.001, 0.003, 0.005, 0.001, 0.002, 0.002, 0.014, 0.001, 0.01, 0.003, 0.0001, 0.001, 0.001, 0.002, 0.06, 0.0001, 0.001, 0.003, 0.003, 0.001, 0.0001, 0.084, 0.004, 0.0001, 0.001, 0.001, 0.0001, 0.002, 0.004, 0.002, 0.005, 0.003, 0.001, 0.001, 0.002, 0.001, 0.0001, 0.004, 0.003, 0.003, 0.005, 0.002, 0.002, 0.001, 0.006, 0.005, 0.002, 0.003, 0.005, 0.002, 0.003, 0.003, 0.0001, 0.0001, 0.0001, 0.089, 0.024, 0.004, 0.007, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.0001, 0.0001, 0.002, 0.001, 0.029, 0.011, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.005, 0.002, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.002, 0.091, 0.001, 0.0001, 0.001, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
|
||||
});
|
||||
|
||||
/**
|
||||
* Determines whether two ArrayBuffers contain the same values.
|
||||
*
|
||||
* @param {ArrayBuffer} a
|
||||
* @param {ArrayBuffer} b
|
||||
* @returns {boolean}
|
||||
*/
|
||||
function _buffersEqual(a, b) {
|
||||
if (a === b) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (a.byteLength !== b.byteLength) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const ai = new Uint8Array(a),
|
||||
bi = new Uint8Array(b);
|
||||
|
||||
let i = a.byteLength;
|
||||
while (i--) {
|
||||
if (ai[i] !== bi[i]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
export default Magic;
|
||||
|
|
|
|||
Loading…
Reference in New Issue