Magic operation now recognises useful operations such as 'Render Image' even though their output cannot be analysed

src/core/FlowControl.js

@@ -294,13 +294,14 @@ const FlowControl = {
             let language = "",
                 fileType = "",
                 matchingOps = "",
+                useful = "",
                 validUTF8 = option.isUTF8 ? "Valid UTF8\n" : "";
 
-            if (option.languageScores[0].probability > 0.00001) {
+            if (option.languageScores[0].probability > 0) {
-                let likelyLangs = option.languageScores.filter(l => l.probability > 0.2);
+                let likelyLangs = option.languageScores.filter(l => l.probability > 0);
                 if (likelyLangs.length < 1) likelyLangs = [option.languageScores[0]];
-                language = "Language:\n    " + likelyLangs.map(lang => {
+                language = "Possible languages:\n    " + likelyLangs.map(lang => {
-                    return `${Magic.codeToLanguage(lang.lang)} ${(lang.probability * 100).toFixed(2)}%`;
+                    return Magic.codeToLanguage(lang.lang);
                 }).join("\n    ") + "\n";
             }
 
@@ -312,10 +313,14 @@ const FlowControl = {
                 matchingOps = `Matching ops: ${[...new Set(option.matchingOps.map(op => op.op))].join(", ")}\n`;
             }
 
+            if (option.useful) {
+                useful = "Useful op detected\n";
+            }
+
             output += `<tr>
                 <td><a href="#${recipeURL}">${Utils.generatePrettyRecipe(option.recipe, true)}</a></td>
                 <td>${Utils.escapeHtml(Utils.printable(Utils.truncate(option.data, 99)))}</td>
-                <td>${language}${fileType}${matchingOps}${validUTF8}</td>
+                <td>${language}${fileType}${matchingOps}${useful}${validUTF8}</td>
             </tr>`;
         });
 

src/core/config/OperationConfig.js

@@ -4091,7 +4091,8 @@ const OperationConfig = {
             {
                 match: "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
                 flags: "",
-                args: ["Raw"]
+                args: ["Raw"],
+                useful: true
             },
         ]
     },

src/core/lib/Magic.js

@@ -58,6 +58,12 @@ class Magic {
      * @returns {Object[]}
      */
     detectLanguage(extLang = false) {
+        if (!this.inputBuffer.length) return [{
+            lang: "Unknown",
+            score: Math.MAX_VALUE,
+            probability: Math.MIN_VALUE
+        }];
+
         const inputFreq = this._freqDist();
         const langFreqs = extLang ? EXTENSIVE_LANG_FREQS : COMMON_LANG_FREQS;
         let chiSqrs = [];
@@ -212,9 +218,10 @@ class Magic {
      * @param {boolean} [intensive=false] - Run brute-forcing on each branch (significantly affects
      *                                      performance)
      * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point
-     * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding 
+     * @param {boolean} [useful=false] - Whether the current recipe should be scored highly
+     * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding
      */
-    async speculativeExecution(depth = 0, extLang = false, intensive = false, recipeConfig = []) {
+    async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false) {
         if (depth < 0) return [];
 
         // Find any operations that can be run on this data
@@ -229,7 +236,8 @@ class Magic {
             languageScores: this.detectLanguage(extLang),
             fileType: this.detectFileType(),
             isUTF8: this.isUTF8(),
-            matchingOps: matchingOps
+            matchingOps: matchingOps,
+            useful: useful
         });
 
         // Execute each of the matching operations, then recursively call the speculativeExecution()
@@ -250,7 +258,7 @@ class Magic {
 
             const magic = new Magic(dish.get(Dish.ARRAY_BUFFER), this.opPatterns),
                 speculativeResults = await magic.speculativeExecution(
-                        depth-1, extLang, intensive, [...recipeConfig, opConfig]);
+                        depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful);
 
             results = results.concat(speculativeResults);
         }));
@@ -273,7 +281,8 @@ class Magic {
                 r.languageScores[0].probability > 0 ||
                 r.fileType ||
                 r.isUTF8 ||
-                r.matchingOps.length);
+                r.matchingOps.length ||
+                r.useful);
 
         // Return a sorted list of possible recipes along with their properties
         return results.sort((a, b) => {
@@ -289,6 +298,10 @@ class Magic {
             if (a.isUTF8) aScore -= 100;
             if (b.isUTF8) bScore -= 100;
 
+            // If the option is marked useful, give it a good score
+            if (a.useful) aScore = 100;
+            if (b.useful) bScore = 100;
+
             return aScore - bScore;
         });
     }
@@ -332,7 +345,8 @@ class Magic {
                     op: op,
                     match: pattern.match,
                     flags: pattern.flags,
-                    args: pattern.args
+                    args: pattern.args,
+                    useful: pattern.useful || false
                 });
             });
         }
@@ -688,7 +702,7 @@ class Magic {
  * as of early 2018.
  */
 const COMMON_LANG_FREQS = {
-	"en":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.755, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 15.843, 0.004, 0.375, 0.002, 0.008, 0.019, 0.008, 0.134, 0.137, 0.137, 0.001, 0.001, 0.972, 0.19, 0.857, 0.017, 0.334, 0.421, 0.246, 0.108, 0.104, 0.112, 0.103, 0.1, 0.127, 0.237, 0.04, 0.027, 0.004, 0.003, 0.004, 0.002, 0.0001, 0.338, 0.218, 0.326, 0.163, 0.121, 0.149, 0.133, 0.192, 0.232, 0.107, 0.082, 0.148, 0.248, 0.134, 0.103, 0.195, 0.012, 0.162, 0.368, 0.366, 0.077, 0.061, 0.127, 0.009, 0.03, 0.015, 0.004, 0.0001, 0.004, 0.0001, 0.003, 0.0001, 6.614, 1.039, 2.327, 2.934, 9.162, 1.606, 1.415, 3.503, 5.718, 0.081, 0.461, 3.153, 1.793, 5.723, 5.565, 1.415, 0.066, 5.036, 4.79, 6.284, 1.992, 0.759, 1.176, 0.139, 1.162, 0.102, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.06, 0.004, 0.003, 0.002, 0.001, 0.001, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.001, 0.003, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.031, 0.006, 0.001, 0.001, 0.001, 0.002, 0.014, 0.001, 0.001, 0.005, 0.005, 0.001, 0.002, 0.017, 0.007, 0.002, 0.003, 0.004, 0.002, 0.001, 0.002, 0.002, 0.012, 0.001, 0.002, 0.001, 0.004, 0.001, 0.001, 0.003, 0.003, 0.002, 0.005, 0.001, 0.001, 0.003, 0.001, 0.003, 0.001, 0.002, 0.001, 0.004, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.02, 0.047, 0.009, 0.009, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.003, 0.001, 0.004, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.005, 0.002, 0.061, 0.001, 0.0001, 0.002, 0.001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
+    "en":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.755, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 15.843, 0.004, 0.375, 0.002, 0.008, 0.019, 0.008, 0.134, 0.137, 0.137, 0.001, 0.001, 0.972, 0.19, 0.857, 0.017, 0.334, 0.421, 0.246, 0.108, 0.104, 0.112, 0.103, 0.1, 0.127, 0.237, 0.04, 0.027, 0.004, 0.003, 0.004, 0.002, 0.0001, 0.338, 0.218, 0.326, 0.163, 0.121, 0.149, 0.133, 0.192, 0.232, 0.107, 0.082, 0.148, 0.248, 0.134, 0.103, 0.195, 0.012, 0.162, 0.368, 0.366, 0.077, 0.061, 0.127, 0.009, 0.03, 0.015, 0.004, 0.0001, 0.004, 0.0001, 0.003, 0.0001, 6.614, 1.039, 2.327, 2.934, 9.162, 1.606, 1.415, 3.503, 5.718, 0.081, 0.461, 3.153, 1.793, 5.723, 5.565, 1.415, 0.066, 5.036, 4.79, 6.284, 1.992, 0.759, 1.176, 0.139, 1.162, 0.102, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.06, 0.004, 0.003, 0.002, 0.001, 0.001, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.001, 0.003, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.031, 0.006, 0.001, 0.001, 0.001, 0.002, 0.014, 0.001, 0.001, 0.005, 0.005, 0.001, 0.002, 0.017, 0.007, 0.002, 0.003, 0.004, 0.002, 0.001, 0.002, 0.002, 0.012, 0.001, 0.002, 0.001, 0.004, 0.001, 0.001, 0.003, 0.003, 0.002, 0.005, 0.001, 0.001, 0.003, 0.001, 0.003, 0.001, 0.002, 0.001, 0.004, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.02, 0.047, 0.009, 0.009, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.003, 0.001, 0.004, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.005, 0.002, 0.061, 0.001, 0.0001, 0.002, 0.001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
     "ru":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.512, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 7.274, 0.002, 0.063, 0.0001, 0.001, 0.009, 0.001, 0.001, 0.118, 0.118, 0.0001, 0.001, 0.595, 0.135, 0.534, 0.009, 0.18, 0.281, 0.15, 0.078, 0.076, 0.077, 0.068, 0.066, 0.083, 0.16, 0.036, 0.016, 0.002, 0.001, 0.002, 0.001, 0.0001, 0.013, 0.009, 0.014, 0.009, 0.007, 0.006, 0.007, 0.006, 0.031, 0.002, 0.003, 0.007, 0.012, 0.007, 0.005, 0.01, 0.001, 0.008, 0.017, 0.011, 0.003, 0.009, 0.005, 0.012, 0.001, 0.001, 0.001, 0.0001, 0.001, 0.0001, 0.003, 0.0001, 0.065, 0.009, 0.022, 0.021, 0.074, 0.01, 0.013, 0.019, 0.054, 0.001, 0.008, 0.036, 0.02, 0.047, 0.055, 0.013, 0.001, 0.052, 0.037, 0.041, 0.026, 0.007, 0.006, 0.003, 0.011, 0.003, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 2.469, 2.363, 2.342, 0.986, 0.156, 0.422, 0.252, 0.495, 0.217, 0.136, 0.014, 0.778, 0.56, 0.097, 0.251, 0.811, 0.09, 0.184, 0.165, 0.06, 0.179, 0.021, 0.013, 0.029, 0.05, 0.005, 0.116, 0.045, 0.087, 0.073, 0.067, 0.124, 0.211, 0.16, 0.055, 0.033, 0.036, 0.024, 0.013, 0.02, 0.022, 0.002, 0.0001, 0.1, 0.0001, 0.025, 0.009, 0.011, 3.536, 0.619, 1.963, 0.833, 1.275, 3.452, 0.323, 0.635, 3.408, 0.642, 1.486, 1.967, 1.26, 2.857, 4.587, 1.082, 0.0001, 0.0001, 0.339, 0.003, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.013, 0.0001, 0.002, 0.001, 31.356, 12.318, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.131, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
     "de":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.726, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 13.303, 0.002, 0.278, 0.0001, 0.0001, 0.007, 0.003, 0.005, 0.149, 0.149, 0.015, 0.001, 0.636, 0.237, 0.922, 0.023, 0.305, 0.472, 0.225, 0.115, 0.11, 0.121, 0.108, 0.11, 0.145, 0.271, 0.049, 0.022, 0.002, 0.002, 0.002, 0.001, 0.0001, 0.413, 0.383, 0.144, 0.412, 0.275, 0.258, 0.273, 0.218, 0.18, 0.167, 0.277, 0.201, 0.328, 0.179, 0.111, 0.254, 0.012, 0.219, 0.602, 0.209, 0.1, 0.185, 0.206, 0.005, 0.01, 0.112, 0.002, 0.0001, 0.002, 0.0001, 0.006, 0.0001, 4.417, 1.306, 1.99, 3.615, 12.382, 1.106, 2.0, 2.958, 6.179, 0.082, 0.866, 2.842, 1.869, 7.338, 2.27, 0.606, 0.016, 6.056, 4.424, 4.731, 3.002, 0.609, 0.918, 0.053, 0.169, 0.824, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.147, 0.002, 0.003, 0.001, 0.006, 0.001, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.0001, 0.001, 0.004, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.03, 0.0001, 0.0001, 0.009, 0.001, 0.002, 0.009, 0.002, 0.001, 0.061, 0.0001, 0.048, 0.122, 0.057, 0.009, 0.001, 0.001, 0.4, 0.001, 0.002, 0.003, 0.003, 0.017, 0.001, 0.003, 0.001, 0.005, 0.0001, 0.001, 0.003, 0.002, 0.003, 0.005, 0.001, 0.001, 0.203, 0.0001, 0.002, 0.001, 0.002, 0.002, 0.438, 0.002, 0.002, 0.001, 0.0001, 0.0001, 0.056, 1.237, 0.01, 0.013, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.003, 0.001, 0.005, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.148, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
     "ja":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.834, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.258, 0.007, 0.036, 0.001, 0.0001, 0.005, 0.002, 0.003, 0.033, 0.033, 0.0001, 0.002, 0.019, 0.052, 0.026, 0.009, 0.281, 0.407, 0.259, 0.126, 0.108, 0.109, 0.095, 0.092, 0.104, 0.184, 0.008, 0.001, 0.002, 0.002, 0.002, 0.001, 0.0001, 0.048, 0.026, 0.039, 0.027, 0.028, 0.022, 0.018, 0.016, 0.03, 0.012, 0.014, 0.02, 0.03, 0.025, 0.025, 0.026, 0.002, 0.026, 0.045, 0.031, 0.013, 0.014, 0.014, 0.006, 0.006, 0.003, 0.001, 0.0001, 0.001, 0.0001, 0.002, 0.0001, 0.077, 0.012, 0.03, 0.026, 0.088, 0.012, 0.017, 0.025, 0.067, 0.002, 0.016, 0.041, 0.039, 0.059, 0.066, 0.016, 0.001, 0.06, 0.043, 0.051, 0.028, 0.009, 0.007, 0.004, 0.015, 0.004, 0.0001, 0.011, 0.0001, 0.0001, 0.0001, 2.555, 10.322, 5.875, 4.462, 0.784, 0.468, 0.442, 0.409, 1.173, 0.96, 0.657, 1.448, 1.442, 0.636, 0.341, 0.685, 0.495, 0.342, 0.651, 0.536, 0.435, 0.657, 0.51, 0.978, 0.31, 0.563, 0.439, 0.514, 0.668, 0.438, 0.29, 1.039, 0.423, 0.532, 0.407, 0.691, 0.677, 0.555, 0.911, 0.887, 1.086, 0.531, 0.836, 1.345, 0.438, 0.666, 1.528, 0.959, 0.535, 0.379, 0.302, 0.822, 0.614, 0.308, 0.253, 0.467, 0.807, 0.807, 0.777, 0.809, 1.292, 0.546, 0.524, 0.425, 0.0001, 0.0001, 0.002, 0.004, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.0001, 0.015, 19.387, 1.167, 4.022, 2.518, 1.734, 1.339, 1.229, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.409, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
@@ -726,7 +740,7 @@ const COMMON_LANG_FREQS = {
     "sl":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.06, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 14.437, 0.024, 0.237, 0.001, 0.001, 0.007, 0.002, 0.011, 0.174, 0.174, 0.021, 0.002, 1.072, 0.17, 1.037, 0.022, 0.277, 0.429, 0.215, 0.122, 0.124, 0.121, 0.109, 0.108, 0.134, 0.239, 0.061, 0.025, 0.005, 0.006, 0.005, 0.002, 0.0001, 0.162, 0.141, 0.1, 0.122, 0.063, 0.075, 0.091, 0.086, 0.111, 0.082, 0.154, 0.138, 0.185, 0.145, 0.099, 0.224, 0.004, 0.106, 0.263, 0.133, 0.042, 0.163, 0.031, 0.007, 0.007, 0.087, 0.013, 0.0001, 0.014, 0.0001, 0.006, 0.0001, 7.7, 1.204, 0.709, 2.364, 7.782, 0.229, 1.139, 0.879, 6.985, 3.327, 2.701, 3.64, 2.037, 5.283, 6.653, 2.232, 0.006, 4.152, 3.513, 3.409, 1.654, 3.049, 0.039, 0.016, 0.079, 1.473, 0.0001, 0.01, 0.0001, 0.0001, 0.0001, 0.054, 0.004, 0.003, 0.002, 0.002, 0.001, 0.001, 0.011, 0.002, 0.002, 0.0001, 0.001, 0.021, 0.847, 0.001, 0.0001, 0.001, 0.002, 0.002, 0.027, 0.001, 0.0001, 0.001, 0.001, 0.001, 0.002, 0.001, 0.001, 0.002, 0.001, 0.001, 0.001, 0.056, 0.644, 0.007, 0.001, 0.003, 0.001, 0.001, 0.002, 0.003, 0.013, 0.001, 0.027, 0.001, 0.005, 0.001, 0.001, 0.007, 0.003, 0.004, 0.005, 0.002, 0.003, 0.006, 0.001, 0.004, 0.002, 0.004, 0.028, 0.008, 0.018, 0.391, 0.002, 0.0001, 0.0001, 0.071, 0.059, 0.881, 1.071, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.01, 0.005, 0.024, 0.008, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.002, 0.054, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
     "lv":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.879, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 12.099, 0.004, 0.432, 0.0001, 0.0001, 0.013, 0.002, 0.007, 0.207, 0.208, 0.0001, 0.003, 0.965, 0.082, 1.276, 0.01, 0.332, 0.476, 0.254, 0.122, 0.117, 0.123, 0.105, 0.106, 0.127, 0.271, 0.045, 0.023, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.208, 0.134, 0.062, 0.128, 0.074, 0.067, 0.074, 0.058, 0.112, 0.068, 0.189, 0.194, 0.144, 0.089, 0.055, 0.234, 0.002, 0.136, 0.249, 0.163, 0.042, 0.182, 0.012, 0.007, 0.003, 0.051, 0.001, 0.0001, 0.001, 0.0001, 0.003, 0.0001, 8.58, 1.078, 0.806, 2.221, 4.451, 0.231, 1.228, 0.175, 6.667, 1.704, 2.603, 2.424, 2.389, 3.209, 2.883, 1.908, 0.003, 4.056, 5.825, 4.121, 3.633, 1.801, 0.012, 0.009, 0.029, 1.289, 0.0001, 0.005, 0.0001, 0.0001, 0.0001, 0.124, 2.988, 0.003, 0.002, 0.001, 0.006, 0.331, 0.001, 0.002, 0.001, 0.0001, 0.001, 0.015, 0.083, 0.0001, 0.001, 0.001, 0.001, 0.007, 1.174, 0.07, 0.0001, 0.001, 0.001, 0.002, 0.003, 0.001, 0.001, 0.005, 0.012, 0.009, 0.001, 0.06, 0.627, 0.004, 0.097, 0.002, 0.001, 0.001, 0.001, 0.001, 0.002, 0.006, 1.565, 0.0001, 0.002, 0.0001, 0.0001, 0.01, 0.002, 0.005, 0.002, 0.002, 0.005, 0.01, 0.106, 0.006, 0.002, 0.003, 0.01, 0.298, 0.012, 0.176, 0.002, 0.0001, 0.0001, 0.03, 0.013, 6.068, 1.452, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.004, 0.002, 0.051, 0.018, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.11, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
     "et":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.183, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 11.759, 0.003, 0.281, 0.0001, 0.0001, 0.013, 0.001, 0.037, 0.198, 0.199, 0.001, 0.003, 0.786, 0.203, 1.175, 0.017, 0.35, 0.548, 0.272, 0.142, 0.137, 0.143, 0.127, 0.129, 0.154, 0.323, 0.059, 0.022, 0.017, 0.003, 0.017, 0.003, 0.0001, 0.235, 0.096, 0.074, 0.061, 0.173, 0.056, 0.064, 0.105, 0.122, 0.088, 0.255, 0.166, 0.186, 0.114, 0.065, 0.208, 0.003, 0.138, 0.296, 0.251, 0.046, 0.167, 0.033, 0.011, 0.008, 0.01, 0.008, 0.0001, 0.008, 0.0001, 0.004, 0.0001, 9.665, 0.664, 0.152, 2.822, 7.678, 0.189, 1.393, 1.095, 7.816, 1.25, 3.234, 4.738, 2.585, 4.03, 3.549, 1.167, 0.005, 3.003, 6.68, 5.333, 4.153, 1.613, 0.043, 0.017, 0.074, 0.045, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.13, 0.015, 0.01, 0.006, 0.004, 0.003, 0.003, 0.004, 0.002, 0.002, 0.001, 0.002, 0.003, 0.005, 0.001, 0.003, 0.002, 0.002, 0.003, 0.102, 0.002, 0.008, 0.003, 0.003, 0.002, 0.004, 0.002, 0.001, 0.044, 0.005, 0.006, 0.003, 0.016, 0.035, 0.003, 0.002, 0.833, 0.002, 0.001, 0.002, 0.002, 0.01, 0.001, 0.006, 0.001, 0.005, 0.001, 0.001, 0.017, 0.004, 0.012, 0.007, 0.005, 0.763, 0.179, 0.003, 0.015, 0.005, 0.008, 0.007, 0.518, 0.012, 0.028, 0.003, 0.0001, 0.0001, 0.02, 2.358, 0.019, 0.061, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.009, 0.004, 0.104, 0.037, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.002, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.004, 0.123, 0.001, 0.0001, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
-	"hi":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.374, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 7.123, 0.002, 0.071, 0.0001, 0.001, 0.004, 0.0001, 0.023, 0.08, 0.08, 0.0001, 0.001, 0.255, 0.072, 0.052, 0.006, 0.068, 0.07, 0.044, 0.02, 0.019, 0.023, 0.019, 0.019, 0.021, 0.04, 0.021, 0.006, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.008, 0.004, 0.007, 0.004, 0.005, 0.003, 0.004, 0.003, 0.006, 0.001, 0.002, 0.003, 0.005, 0.004, 0.003, 0.005, 0.0001, 0.003, 0.008, 0.005, 0.002, 0.002, 0.002, 0.001, 0.001, 0.001, 0.007, 0.0001, 0.008, 0.0001, 0.001, 0.0001, 0.049, 0.007, 0.017, 0.016, 0.052, 0.008, 0.01, 0.017, 0.038, 0.001, 0.004, 0.024, 0.015, 0.034, 0.035, 0.012, 0.001, 0.033, 0.03, 0.034, 0.015, 0.005, 0.005, 0.002, 0.008, 0.001, 0.0001, 0.005, 0.0001, 0.0001, 0.0001, 1.039, 0.443, 1.278, 0.061, 0.0001, 0.273, 0.146, 1.879, 0.535, 0.214, 0.013, 0.729, 0.054, 1.826, 0.0001, 0.253, 0.014, 0.012, 0.0001, 0.042, 0.14, 2.07, 0.133, 0.43, 0.035, 0.004, 0.215, 0.046, 0.503, 0.014, 0.016, 0.269, 0.037, 0.213, 0.023, 0.155, 24.777, 7.162, 0.554, 0.224, 1.23, 0.009, 0.8, 0.117, 0.393, 0.245, 0.995, 0.828, 2.018, 0.001, 0.771, 0.001, 0.001, 0.707, 0.299, 0.18, 1.226, 0.94, 0.0001, 0.0001, 0.133, 0.001, 2.558, 1.303, 0.0001, 0.0001, 0.008, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 30.261, 0.0001, 0.024, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
+    "hi":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.374, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 7.123, 0.002, 0.071, 0.0001, 0.001, 0.004, 0.0001, 0.023, 0.08, 0.08, 0.0001, 0.001, 0.255, 0.072, 0.052, 0.006, 0.068, 0.07, 0.044, 0.02, 0.019, 0.023, 0.019, 0.019, 0.021, 0.04, 0.021, 0.006, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.008, 0.004, 0.007, 0.004, 0.005, 0.003, 0.004, 0.003, 0.006, 0.001, 0.002, 0.003, 0.005, 0.004, 0.003, 0.005, 0.0001, 0.003, 0.008, 0.005, 0.002, 0.002, 0.002, 0.001, 0.001, 0.001, 0.007, 0.0001, 0.008, 0.0001, 0.001, 0.0001, 0.049, 0.007, 0.017, 0.016, 0.052, 0.008, 0.01, 0.017, 0.038, 0.001, 0.004, 0.024, 0.015, 0.034, 0.035, 0.012, 0.001, 0.033, 0.03, 0.034, 0.015, 0.005, 0.005, 0.002, 0.008, 0.001, 0.0001, 0.005, 0.0001, 0.0001, 0.0001, 1.039, 0.443, 1.278, 0.061, 0.0001, 0.273, 0.146, 1.879, 0.535, 0.214, 0.013, 0.729, 0.054, 1.826, 0.0001, 0.253, 0.014, 0.012, 0.0001, 0.042, 0.14, 2.07, 0.133, 0.43, 0.035, 0.004, 0.215, 0.046, 0.503, 0.014, 0.016, 0.269, 0.037, 0.213, 0.023, 0.155, 24.777, 7.162, 0.554, 0.224, 1.23, 0.009, 0.8, 0.117, 0.393, 0.245, 0.995, 0.828, 2.018, 0.001, 0.771, 0.001, 0.001, 0.707, 0.299, 0.18, 1.226, 0.94, 0.0001, 0.0001, 0.133, 0.001, 2.558, 1.303, 0.0001, 0.0001, 0.008, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 30.261, 0.0001, 0.024, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],
 };
 const EXTENSIVE_LANG_FREQS = Object.assign({}, COMMON_LANG_FREQS, {
     "aa":  [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 5.161, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 13.548, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.29, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.29, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 6.452, 0.645, 0.645, 2.581, 9.032, 0.0001, 5.161, 3.871, 5.806, 0.0001, 1.935, 2.581, 1.29, 5.161, 2.581, 1.29, 0.0001, 4.516, 0.645, 3.226, 0.645, 0.0001, 1.29, 0.0001, 0.645, 1.29, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 2.581, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.29, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 1.29, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 3.871, 0.645, 2.581, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],