Magic operation now brute forces character encodings. Linted.

src/core/config/OperationConfig.js

@@ -83,7 +83,7 @@ import URL_ from "../operations/URL.js";
 const OperationConfig = {
     "Magic": {
         module: "Default",
-        description: "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various encodings like XOR and bit rotates are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.",
+        description: "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.",
         inputType: "ArrayBuffer",
         outputType: "html",
         flowControl: true,
@@ -1381,7 +1381,7 @@ const OperationConfig = {
                 type: "option",
                 value: Object.keys(CharEnc.IO_FORMAT),
             },
-        ]
+        ],
     },
     "Decode text": {
         module: "CharEnc",

src/core/lib/Magic.js

@@ -87,7 +87,7 @@ class Magic {
 
     /**
      * Detects any matching file types for the input.
-     * 
+     *
      * @returns {Object} type
      * @returns {string} type.ext - File extension
      * @returns {string} type.mime - Mime type
@@ -178,7 +178,7 @@ class Magic {
      *
      * @returns {Object[]} - The encoded data and an operation config to generate it.
      */
-    bruteForce() {
+    async bruteForce() {
         const sample = new Uint8Array(this.inputBuffer).slice(0, 100);
 
         let results = [];
@@ -205,13 +205,39 @@ class Magic {
             });
         }
 
+        // Character encodings
+        const encodings = OperationConfig["Encode text"].args[0].value;
+
+        /**
+         * Test character encodings and add them if they change the data.
+         */
+        const testEnc = async op => {
+            for (let i = 0; i < encodings.length; i++) {
+                const conf = {
+                        op: "Encode text",
+                        args: [encodings[i]]
+                    },
+                    data = await this._runRecipe([conf], sample.buffer);
+
+                // Only add to the results if it changed the data
+                if (!_buffersEqual(data, sample.buffer)) {
+                    results.push({
+                        data: data,
+                        conf: conf
+                    });
+                }
+            }
+        };
+
+        await testEnc("Encode text");
+        await testEnc("Decode text");
 
         return results;
     }
 
     /**
      * Speculatively executes matching operations, recording metadata of each result.
-     * 
+     *
      * @param {number} [depth=0] - How many levels to try to execute
      * @param {boolean} [extLang=false] - Extensive language support (false = only check the most
      *                                    common Internet languages)
@@ -243,29 +269,21 @@ class Magic {
         // Execute each of the matching operations, then recursively call the speculativeExecution()
         // method on the resulting data, recording the properties of each option.
         await Promise.all(matchingOps.map(async op => {
-            const dish = new Dish(this.inputBuffer, Dish.ARRAY_BUFFER),
+            const opConfig = {
-                opConfig = {
                     op: op.op,
                     args: op.args
-                };
+                },
-
+                output = await this._runRecipe([opConfig]),
-            if (ENVIRONMENT_IS_WORKER()) self.loadRequiredModules([opConfig]);
+                magic = new Magic(output, this.opPatterns),
-
-            const recipe = new Recipe([opConfig]);
-            try {
-                await recipe.execute(dish, 0);
-            } catch (err) {} // Ignore errors
-
-            const magic = new Magic(dish.get(Dish.ARRAY_BUFFER), this.opPatterns),
                 speculativeResults = await magic.speculativeExecution(
-                        depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful);
+                    depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful);
 
             results = results.concat(speculativeResults);
         }));
 
         if (intensive) {
             // Run brute forcing of various types on the data and create a new branch for each option
-            const bfEncodings = this.bruteForce();
+            const bfEncodings = await this.bruteForce();
 
             await Promise.all(bfEncodings.map(async enc => {
                 const magic = new Magic(enc.data, this.opPatterns),
@@ -278,11 +296,11 @@ class Magic {
 
         // Prune branches that do not match anything
         results = results.filter(r =>
-                r.languageScores[0].probability > 0 ||
+            r.languageScores[0].probability > 0 ||
-                r.fileType ||
+            r.fileType ||
-                r.isUTF8 ||
+            r.isUTF8 ||
-                r.matchingOps.length ||
+            r.matchingOps.length ||
-                r.useful);
+            r.useful);
 
         // Return a sorted list of possible recipes along with their properties
         return results.sort((a, b) => {
@@ -302,10 +320,34 @@ class Magic {
             if (a.useful) aScore = 100;
             if (b.useful) bScore = 100;
 
+            // Shorter recipes are better, so we add the length of the recipe to the score
+            aScore += a.recipe.length;
+            bScore += b.recipe.length;
+
             return aScore - bScore;
         });
     }
 
+    /**
+     * Runs the given recipe over the input buffer and returns the output.
+     *
+     * @param {Object[]} recipeConfig
+     * @param {ArrayBuffer} [input=this.inputBuffer]
+     * @returns {ArrayBuffer}
+     */
+    async _runRecipe(recipeConfig, input=this.inputBuffer) {
+        const dish = new Dish(input, Dish.ARRAY_BUFFER);
+
+        if (ENVIRONMENT_IS_WORKER()) self.loadRequiredModules(recipeConfig);
+
+        const recipe = new Recipe(recipeConfig);
+        try {
+            await recipe.execute(dish, 0);
+        } catch (err) {} // Ignore errors
+
+        return dish.get(Dish.ARRAY_BUFFER);
+    }
+
     /**
      * Calculates the number of times each byte appears in the input
      *
@@ -952,7 +994,7 @@ const EXTENSIVE_LANG_FREQS = Object.assign({}, COMMON_LANG_FREQS, {
     "sw":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.454, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 14.93, 0.002, 0.217, 0.002, 0.001, 0.01, 0.003, 0.027, 0.171, 0.171, 0.001, 0.001, 0.703, 0.109, 0.942, 0.015, 0.41, 0.383, 0.266, 0.126, 0.108, 0.126, 0.108, 0.107, 0.119, 0.201, 0.062, 0.024, 0.003, 0.004, 0.003, 0.003, 0.0001, 0.226, 0.167, 0.122, 0.086, 0.058, 0.057, 0.065, 0.116, 0.13, 0.09, 0.638, 0.08, 0.504, 0.137, 0.044, 0.113, 0.006, 0.074, 0.173, 0.147, 0.165, 0.059, 0.218, 0.013, 0.04, 0.023, 0.04, 0.0001, 0.04, 0.001, 0.001, 0.001, 16.478, 1.326, 0.611, 1.343, 3.374, 0.678, 1.131, 2.383, 9.629, 0.827, 4.598, 2.609, 3.253, 5.284, 3.187, 0.805, 0.008, 1.616, 2.094, 2.468, 4.443, 0.427, 3.161, 0.026, 2.095, 1.273, 0.001, 0.006, 0.001, 0.0001, 0.0001, 0.04, 0.005, 0.004, 0.002, 0.004, 0.002, 0.002, 0.002, 0.002, 0.002, 0.002, 0.001, 0.002, 0.001, 0.001, 0.001, 0.002, 0.001, 0.001, 0.013, 0.002, 0.001, 0.001, 0.001, 0.002, 0.006, 0.001, 0.001, 0.009, 0.008, 0.001, 0.004, 0.009, 0.003, 0.002, 0.002, 0.003, 0.001, 0.001, 0.005, 0.003, 0.009, 0.001, 0.002, 0.001, 0.002, 0.001, 0.002, 0.005, 0.009, 0.009, 0.004, 0.002, 0.003, 0.004, 0.001, 0.004, 0.003, 0.003, 0.003, 0.006, 0.003, 0.002, 0.003, 0.0001, 0.0001, 0.018, 0.029, 0.009, 0.005, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.014, 0.007, 0.011, 0.004, 0.0001, 0.0001, 0.0001, 0.002, 0.002, 0.005, 0.012, 0.01, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.002, 0.004, 0.038, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
     "szl": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.884, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 12.63, 0.002, 0.452, 0.0001, 0.0001, 0.012, 0.001, 0.026, 0.296, 0.296, 0.001, 0.001, 1.094, 0.318, 1.181, 0.015, 0.332, 0.469, 0.289, 0.138, 0.131, 0.151, 0.118, 0.131, 0.157, 0.273, 0.087, 0.014, 0.006, 0.003, 0.006, 0.0001, 0.0001, 0.207, 0.209, 0.155, 0.118, 0.048, 0.111, 0.139, 0.08, 0.122, 0.125, 0.213, 0.123, 0.287, 0.122, 0.062, 0.309, 0.005, 0.156, 0.329, 0.126, 0.154, 0.05, 0.233, 0.034, 0.017, 0.083, 0.004, 0.0001, 0.004, 0.0001, 0.006, 0.001, 5.741, 0.894, 2.016, 2.128, 5.35, 0.327, 1.279, 0.968, 3.438, 2.841, 2.633, 2.099, 2.293, 3.364, 5.857, 1.423, 0.012, 3.389, 2.85, 2.58, 2.277, 0.102, 3.144, 0.017, 3.623, 2.205, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.191, 0.035, 0.624, 0.044, 0.945, 0.014, 0.009, 0.333, 0.008, 0.003, 0.006, 0.005, 0.012, 0.221, 0.005, 0.196, 0.006, 0.005, 0.003, 0.168, 0.01, 0.003, 0.005, 0.005, 0.005, 0.109, 0.059, 0.562, 0.005, 0.005, 0.004, 0.006, 0.062, 0.111, 0.006, 0.016, 0.01, 0.004, 0.004, 0.012, 0.011, 0.03, 0.005, 0.012, 0.003, 0.012, 0.008, 1.67, 0.032, 0.015, 0.058, 0.035, 0.048, 0.018, 0.012, 0.004, 0.02, 0.013, 0.335, 0.026, 0.282, 0.022, 0.098, 0.006, 0.0001, 0.0001, 0.109, 0.208, 0.455, 5.073, 0.0001, 0.001, 0.0001, 0.008, 0.003, 0.003, 0.004, 0.0001, 0.015, 0.008, 0.161, 0.06, 0.003, 0.002, 0.0001, 0.003, 0.001, 0.009, 0.025, 0.019, 0.0001, 0.001, 0.0001, 0.0001, 0.002, 0.0001, 0.011, 0.01, 0.176, 0.006, 0.001, 0.005, 0.003, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
     "ta":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.357, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 3.862, 0.001, 0.077, 0.0001, 0.001, 0.006, 0.001, 0.007, 0.055, 0.056, 0.0001, 0.001, 0.234, 0.03, 0.384, 0.005, 0.084, 0.106, 0.063, 0.029, 0.028, 0.034, 0.027, 0.032, 0.031, 0.052, 0.017, 0.006, 0.002, 0.002, 0.002, 0.001, 0.0001, 0.008, 0.004, 0.008, 0.004, 0.004, 0.003, 0.005, 0.004, 0.006, 0.002, 0.003, 0.003, 0.005, 0.004, 0.003, 0.008, 0.0001, 0.004, 0.008, 0.005, 0.002, 0.002, 0.002, 0.001, 0.001, 0.0001, 0.006, 0.0001, 0.006, 0.0001, 0.002, 0.0001, 0.062, 0.006, 0.017, 0.014, 0.042, 0.007, 0.009, 0.018, 0.038, 0.001, 0.006, 0.024, 0.018, 0.035, 0.032, 0.011, 0.001, 0.036, 0.022, 0.032, 0.017, 0.005, 0.004, 0.002, 0.01, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.122, 2.149, 0.144, 0.01, 0.0001, 0.297, 0.436, 0.597, 0.764, 0.136, 0.24, 0.226, 0.005, 5.298, 0.158, 0.027, 0.013, 0.0001, 0.078, 0.014, 0.0001, 2.36, 0.0001, 0.0001, 0.001, 0.171, 0.627, 0.0001, 0.037, 0.002, 0.021, 1.319, 0.014, 0.0001, 0.001, 0.32, 2.012, 0.001, 0.001, 0.0001, 0.539, 0.989, 1.521, 0.0001, 0.0001, 0.001, 23.215, 10.185, 1.322, 0.801, 1.028, 0.757, 0.189, 0.942, 0.0001, 0.015, 0.06, 0.015, 0.0001, 0.0001, 0.0001, 0.0001, 1.18, 2.177, 0.0001, 0.0001, 0.016, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 31.245, 0.0001, 0.013, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
-	"tcy": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.391, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 4.751, 0.001, 0.026, 0.0001, 0.0001, 0.002, 0.0001, 0.028, 0.048, 0.047, 0.0001, 0.001, 0.244, 0.028, 0.533, 0.012, 0.014, 0.02, 0.01, 0.005, 0.005, 0.007, 0.006, 0.004, 0.008, 0.009, 0.009, 0.003, 0.002, 0.003, 0.002, 0.002, 0.0001, 0.002, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.001, 0.0001, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.0001, 0.003, 0.0001, 0.001, 0.001, 0.02, 0.002, 0.008, 0.006, 0.018, 0.002, 0.005, 0.006, 0.017, 0.0001, 0.003, 0.009, 0.008, 0.014, 0.015, 0.008, 0.0001, 0.013, 0.012, 0.015, 0.006, 0.002, 0.005, 0.0001, 0.003, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.354, 1.789, 1.221, 0.031, 0.0001, 0.268, 1.686, 0.484, 0.152, 0.21, 0.745, 0.196, 0.087, 4.125, 0.064, 0.014, 0.014, 0.0001, 0.109, 0.011, 0.001, 1.28, 0.033, 0.613, 0.012, 0.007, 0.23, 0.003, 0.404, 0.002, 0.011, 0.433, 0.058, 1.007, 0.002, 0.198, 1.312, 0.064, 1.397, 0.124, 1.439, 0.012, 1.248, 0.035, 0.624, 0.105, 0.769, 0.62, 1.755, 0.0001, 22.872, 9.408, 0.0001, 0.629, 0.164, 0.121, 0.665, 0.124, 0.0001, 0.0001, 0.003, 0.0001, 1.377, 1.63, 0.0001, 0.0001, 0.05, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 30.955, 0.0001, 0.194, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
+    "tcy": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.391, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 4.751, 0.001, 0.026, 0.0001, 0.0001, 0.002, 0.0001, 0.028, 0.048, 0.047, 0.0001, 0.001, 0.244, 0.028, 0.533, 0.012, 0.014, 0.02, 0.01, 0.005, 0.005, 0.007, 0.006, 0.004, 0.008, 0.009, 0.009, 0.003, 0.002, 0.003, 0.002, 0.002, 0.0001, 0.002, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.001, 0.0001, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.0001, 0.003, 0.0001, 0.001, 0.001, 0.02, 0.002, 0.008, 0.006, 0.018, 0.002, 0.005, 0.006, 0.017, 0.0001, 0.003, 0.009, 0.008, 0.014, 0.015, 0.008, 0.0001, 0.013, 0.012, 0.015, 0.006, 0.002, 0.005, 0.0001, 0.003, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.354, 1.789, 1.221, 0.031, 0.0001, 0.268, 1.686, 0.484, 0.152, 0.21, 0.745, 0.196, 0.087, 4.125, 0.064, 0.014, 0.014, 0.0001, 0.109, 0.011, 0.001, 1.28, 0.033, 0.613, 0.012, 0.007, 0.23, 0.003, 0.404, 0.002, 0.011, 0.433, 0.058, 1.007, 0.002, 0.198, 1.312, 0.064, 1.397, 0.124, 1.439, 0.012, 1.248, 0.035, 0.624, 0.105, 0.769, 0.62, 1.755, 0.0001, 22.872, 9.408, 0.0001, 0.629, 0.164, 0.121, 0.665, 0.124, 0.0001, 0.0001, 0.003, 0.0001, 1.377, 1.63, 0.0001, 0.0001, 0.05, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 30.955, 0.0001, 0.194, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
     "te":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.34, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 4.746, 0.003, 0.051, 0.0001, 0.001, 0.003, 0.002, 0.007, 0.042, 0.043, 0.0001, 0.001, 0.336, 0.028, 0.611, 0.018, 0.129, 0.152, 0.069, 0.038, 0.034, 0.073, 0.03, 0.032, 0.034, 0.047, 0.02, 0.007, 0.002, 0.004, 0.002, 0.002, 0.0001, 0.008, 0.004, 0.006, 0.005, 0.003, 0.003, 0.002, 0.002, 0.006, 0.001, 0.002, 0.003, 0.005, 0.003, 0.002, 0.005, 0.0001, 0.003, 0.008, 0.005, 0.003, 0.002, 0.002, 0.001, 0.0001, 0.0001, 0.006, 0.0001, 0.007, 0.0001, 0.005, 0.0001, 0.053, 0.008, 0.019, 0.022, 0.056, 0.009, 0.01, 0.021, 0.046, 0.001, 0.004, 0.022, 0.015, 0.038, 0.038, 0.014, 0.0001, 0.036, 0.036, 0.045, 0.017, 0.006, 0.006, 0.002, 0.007, 0.001, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.485, 1.801, 1.898, 0.051, 0.0001, 0.236, 0.427, 0.575, 0.238, 0.222, 0.152, 0.685, 0.105, 2.799, 0.055, 0.027, 0.006, 0.0001, 0.047, 0.007, 0.005, 1.329, 0.049, 0.668, 0.014, 0.002, 0.428, 0.004, 0.25, 0.001, 0.004, 0.537, 0.039, 0.598, 0.002, 0.137, 0.864, 0.099, 0.843, 0.149, 1.628, 0.0001, 0.909, 0.085, 0.267, 0.128, 0.942, 0.804, 25.531, 7.165, 1.487, 0.074, 0.0001, 0.877, 0.211, 0.153, 0.855, 0.145, 0.0001, 0.001, 0.0001, 0.0001, 2.169, 2.359, 0.0001, 0.0001, 0.014, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 30.736, 0.0001, 0.069, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
     "tet": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.506, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 14.056, 0.014, 0.345, 0.0001, 0.004, 0.018, 0.001, 0.455, 0.383, 0.382, 0.001, 0.004, 1.067, 0.53, 0.968, 0.029, 0.443, 0.39, 0.316, 0.132, 0.112, 0.137, 0.105, 0.106, 0.119, 0.181, 0.186, 0.018, 0.015, 0.005, 0.015, 0.003, 0.0001, 0.338, 0.226, 0.145, 0.169, 0.132, 0.156, 0.098, 0.111, 0.215, 0.061, 0.136, 0.43, 0.301, 0.181, 0.101, 0.266, 0.01, 0.137, 0.345, 0.37, 0.107, 0.065, 0.041, 0.021, 0.008, 0.014, 0.01, 0.0001, 0.01, 0.0001, 0.0001, 0.0001, 11.569, 1.502, 0.408, 2.068, 6.067, 0.587, 0.66, 2.225, 7.509, 0.16, 2.246, 2.814, 2.311, 6.307, 4.401, 1.282, 0.035, 4.022, 4.063, 3.545, 4.826, 0.518, 0.1, 0.064, 0.126, 0.341, 0.0001, 0.009, 0.0001, 0.0001, 0.0001, 0.318, 0.081, 0.003, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.001, 0.002, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.015, 0.001, 0.0001, 0.0001, 0.001, 0.004, 0.275, 0.001, 0.0001, 0.014, 0.013, 0.001, 0.002, 0.021, 0.254, 0.002, 0.025, 0.0001, 0.0001, 0.003, 0.02, 0.002, 0.389, 0.006, 0.001, 0.001, 0.167, 0.001, 0.001, 0.002, 0.048, 0.071, 0.284, 0.01, 0.003, 0.001, 0.0001, 0.001, 0.001, 0.076, 0.003, 0.014, 0.001, 0.001, 0.002, 0.0001, 0.0001, 0.1, 1.362, 0.004, 0.006, 0.0001, 0.0001, 0.0001, 0.009, 0.011, 0.0001, 0.0001, 0.0001, 0.007, 0.003, 0.006, 0.003, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.316, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
     "tg":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.272, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 7.893, 0.001, 0.026, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.324, 0.326, 0.0001, 0.001, 0.765, 0.105, 0.581, 0.006, 0.139, 0.257, 0.13, 0.073, 0.063, 0.072, 0.065, 0.068, 0.082, 0.185, 0.026, 0.048, 0.002, 0.001, 0.002, 0.001, 0.0001, 0.026, 0.01, 0.018, 0.007, 0.005, 0.01, 0.006, 0.007, 0.018, 0.002, 0.005, 0.008, 0.009, 0.006, 0.004, 0.009, 0.001, 0.007, 0.015, 0.007, 0.003, 0.006, 0.004, 0.006, 0.002, 0.002, 0.004, 0.0001, 0.004, 0.0001, 0.0001, 0.0001, 0.081, 0.01, 0.03, 0.023, 0.086, 0.012, 0.015, 0.021, 0.065, 0.002, 0.009, 0.037, 0.017, 0.055, 0.061, 0.017, 0.001, 0.07, 0.039, 0.054, 0.023, 0.01, 0.007, 0.003, 0.013, 0.003, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 2.968, 1.483, 1.764, 1.455, 0.398, 0.384, 0.008, 0.116, 0.704, 0.002, 0.17, 0.01, 0.024, 0.035, 0.045, 0.663, 0.178, 0.263, 0.119, 0.126, 0.303, 0.007, 0.009, 0.022, 0.136, 0.003, 0.143, 0.343, 0.148, 0.063, 0.071, 0.071, 0.134, 0.159, 0.101, 0.347, 0.121, 0.05, 0.002, 0.026, 0.059, 0.003, 0.003, 0.057, 0.003, 0.035, 0.012, 0.164, 5.899, 1.075, 1.071, 1.816, 2.336, 1.339, 0.082, 0.882, 4.885, 0.258, 1.014, 1.438, 1.445, 2.22, 3.885, 0.208, 0.0001, 0.0001, 0.132, 0.006, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.0001, 0.002, 0.001, 30.166, 10.131, 1.965, 0.481, 0.0001, 0.0001, 0.0001, 0.0001, 0.024, 0.016, 0.001, 0.006, 0.0001, 0.0001, 0.0001, 0.0001, 0.003, 0.001, 0.209, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
@@ -991,4 +1033,33 @@ const EXTENSIVE_LANG_FREQS = Object.assign({}, COMMON_LANG_FREQS, {
     "zu":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.261, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 10.94, 0.004, 0.267, 0.001, 0.002, 0.016, 0.003, 0.041, 0.181, 0.181, 0.001, 0.001, 0.907, 0.49, 0.797, 0.099, 0.343, 0.379, 0.279, 0.134, 0.118, 0.116, 0.102, 0.097, 0.11, 0.223, 0.065, 0.035, 0.134, 0.003, 0.135, 0.005, 0.0001, 0.296, 0.147, 0.142, 0.093, 0.11, 0.08, 0.077, 0.065, 0.3, 0.114, 0.141, 0.151, 0.361, 0.387, 0.067, 0.128, 0.012, 0.082, 0.239, 0.152, 0.188, 0.039, 0.182, 0.012, 0.045, 0.07, 0.138, 0.0001, 0.139, 0.0001, 0.001, 0.0001, 10.325, 2.215, 0.829, 1.627, 7.521, 0.687, 2.042, 3.525, 7.719, 0.199, 3.874, 4.421, 2.406, 6.494, 4.881, 0.951, 0.342, 1.361, 3.011, 2.552, 4.691, 0.394, 2.227, 0.134, 1.688, 1.779, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.08, 0.007, 0.001, 0.001, 0.002, 0.0001, 0.014, 0.002, 0.002, 0.001, 0.0001, 0.001, 0.003, 0.005, 0.001, 0.002, 0.002, 0.014, 0.001, 0.01, 0.003, 0.0001, 0.001, 0.001, 0.002, 0.06, 0.0001, 0.001, 0.003, 0.003, 0.001, 0.0001, 0.084, 0.004, 0.0001, 0.001, 0.001, 0.0001, 0.002, 0.004, 0.002, 0.005, 0.003, 0.001, 0.001, 0.002, 0.001, 0.0001, 0.004, 0.003, 0.003, 0.005, 0.002, 0.002, 0.001, 0.006, 0.005, 0.002, 0.003, 0.005, 0.002, 0.003, 0.003, 0.0001, 0.0001, 0.0001, 0.089, 0.024, 0.004, 0.007, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.001, 0.0001, 0.0001, 0.002, 0.001, 0.029, 0.011, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.005, 0.002, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.002, 0.091, 0.001, 0.0001, 0.001, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
 });
 
+/**
+ * Determines whether two ArrayBuffers contain the same values.
+ *
+ * @param {ArrayBuffer} a
+ * @param {ArrayBuffer} b
+ * @returns {boolean}
+ */
+function _buffersEqual(a, b) {
+    if (a === b) {
+        return true;
+    }
+
+    if (a.byteLength !== b.byteLength) {
+        return false;
+    }
+
+    const ai = new Uint8Array(a),
+        bi = new Uint8Array(b);
+
+    let i = a.byteLength;
+    while (i--) {
+        if (ai[i] !== bi[i]) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
 export default Magic;