diff --git a/src/core/FlowControl.js b/src/core/FlowControl.js index c4f717a8..e886be6c 100755 --- a/src/core/FlowControl.js +++ b/src/core/FlowControl.js @@ -294,13 +294,14 @@ const FlowControl = { let language = "", fileType = "", matchingOps = "", + useful = "", validUTF8 = option.isUTF8 ? "Valid UTF8\n" : ""; - if (option.languageScores[0].probability > 0.00001) { - let likelyLangs = option.languageScores.filter(l => l.probability > 0.2); + if (option.languageScores[0].probability > 0) { + let likelyLangs = option.languageScores.filter(l => l.probability > 0); if (likelyLangs.length < 1) likelyLangs = [option.languageScores[0]]; - language = "Language:\n " + likelyLangs.map(lang => { - return `${Magic.codeToLanguage(lang.lang)} ${(lang.probability * 100).toFixed(2)}%`; + language = "Possible languages:\n " + likelyLangs.map(lang => { + return Magic.codeToLanguage(lang.lang); }).join("\n ") + "\n"; } @@ -312,10 +313,14 @@ const FlowControl = { matchingOps = `Matching ops: ${[...new Set(option.matchingOps.map(op => op.op))].join(", ")}\n`; } + if (option.useful) { + useful = "Useful op detected\n"; + } + output += ` ${Utils.generatePrettyRecipe(option.recipe, true)} ${Utils.escapeHtml(Utils.printable(Utils.truncate(option.data, 99)))} - ${language}${fileType}${matchingOps}${validUTF8} + ${language}${fileType}${matchingOps}${useful}${validUTF8} `; }); diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index 8579e93b..9be287c3 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -4091,7 +4091,8 @@ const OperationConfig = { { match: "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)", flags: "", - args: ["Raw"] + args: ["Raw"], + useful: true }, ] }, diff --git a/src/core/lib/Magic.js b/src/core/lib/Magic.js index 2e288064..9f93f286 100644 --- a/src/core/lib/Magic.js +++ b/src/core/lib/Magic.js @@ -58,6 +58,12 @@ class Magic { * @returns {Object[]} */ detectLanguage(extLang = false) { + if (!this.inputBuffer.length) return [{ + lang: "Unknown", + score: Math.MAX_VALUE, + probability: Math.MIN_VALUE + }]; + const inputFreq = this._freqDist(); const langFreqs = extLang ? EXTENSIVE_LANG_FREQS : COMMON_LANG_FREQS; let chiSqrs = []; @@ -212,9 +218,10 @@ class Magic { * @param {boolean} [intensive=false] - Run brute-forcing on each branch (significantly affects * performance) * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point - * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding + * @param {boolean} [useful=false] - Whether the current recipe should be scored highly + * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding */ - async speculativeExecution(depth = 0, extLang = false, intensive = false, recipeConfig = []) { + async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false) { if (depth < 0) return []; // Find any operations that can be run on this data @@ -229,7 +236,8 @@ class Magic { languageScores: this.detectLanguage(extLang), fileType: this.detectFileType(), isUTF8: this.isUTF8(), - matchingOps: matchingOps + matchingOps: matchingOps, + useful: useful }); // Execute each of the matching operations, then recursively call the speculativeExecution() @@ -250,7 +258,7 @@ class Magic { const magic = new Magic(dish.get(Dish.ARRAY_BUFFER), this.opPatterns), speculativeResults = await magic.speculativeExecution( - depth-1, extLang, intensive, [...recipeConfig, opConfig]); + depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful); results = results.concat(speculativeResults); })); @@ -273,7 +281,8 @@ class Magic { r.languageScores[0].probability > 0 || r.fileType || r.isUTF8 || - r.matchingOps.length); + r.matchingOps.length || + r.useful); // Return a sorted list of possible recipes along with their properties return results.sort((a, b) => { @@ -289,6 +298,10 @@ class Magic { if (a.isUTF8) aScore -= 100; if (b.isUTF8) bScore -= 100; + // If the option is marked useful, give it a good score + if (a.useful) aScore = 100; + if (b.useful) bScore = 100; + return aScore - bScore; }); } @@ -332,7 +345,8 @@ class Magic { op: op, match: pattern.match, flags: pattern.flags, - args: pattern.args + args: pattern.args, + useful: pattern.useful || false }); }); } @@ -688,7 +702,7 @@ class Magic { * as of early 2018. */ const COMMON_LANG_FREQS = { - "en": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.755, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 15.843, 0.004, 0.375, 0.002, 0.008, 0.019, 0.008, 0.134, 0.137, 0.137, 0.001, 0.001, 0.972, 0.19, 0.857, 0.017, 0.334, 0.421, 0.246, 0.108, 0.104, 0.112, 0.103, 0.1, 0.127, 0.237, 0.04, 0.027, 0.004, 0.003, 0.004, 0.002, 0.0001, 0.338, 0.218, 0.326, 0.163, 0.121, 0.149, 0.133, 0.192, 0.232, 0.107, 0.082, 0.148, 0.248, 0.134, 0.103, 0.195, 0.012, 0.162, 0.368, 0.366, 0.077, 0.061, 0.127, 0.009, 0.03, 0.015, 0.004, 0.0001, 0.004, 0.0001, 0.003, 0.0001, 6.614, 1.039, 2.327, 2.934, 9.162, 1.606, 1.415, 3.503, 5.718, 0.081, 0.461, 3.153, 1.793, 5.723, 5.565, 1.415, 0.066, 5.036, 4.79, 6.284, 1.992, 0.759, 1.176, 0.139, 1.162, 0.102, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.06, 0.004, 0.003, 0.002, 0.001, 0.001, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.001, 0.003, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.031, 0.006, 0.001, 0.001, 0.001, 0.002, 0.014, 0.001, 0.001, 0.005, 0.005, 0.001, 0.002, 0.017, 0.007, 0.002, 0.003, 0.004, 0.002, 0.001, 0.002, 0.002, 0.012, 0.001, 0.002, 0.001, 0.004, 0.001, 0.001, 0.003, 0.003, 0.002, 0.005, 0.001, 0.001, 0.003, 0.001, 0.003, 0.001, 0.002, 0.001, 0.004, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.02, 0.047, 0.009, 0.009, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.003, 0.001, 0.004, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.005, 0.002, 0.061, 0.001, 0.0001, 0.002, 0.001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001], + "en": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.755, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 15.843, 0.004, 0.375, 0.002, 0.008, 0.019, 0.008, 0.134, 0.137, 0.137, 0.001, 0.001, 0.972, 0.19, 0.857, 0.017, 0.334, 0.421, 0.246, 0.108, 0.104, 0.112, 0.103, 0.1, 0.127, 0.237, 0.04, 0.027, 0.004, 0.003, 0.004, 0.002, 0.0001, 0.338, 0.218, 0.326, 0.163, 0.121, 0.149, 0.133, 0.192, 0.232, 0.107, 0.082, 0.148, 0.248, 0.134, 0.103, 0.195, 0.012, 0.162, 0.368, 0.366, 0.077, 0.061, 0.127, 0.009, 0.03, 0.015, 0.004, 0.0001, 0.004, 0.0001, 0.003, 0.0001, 6.614, 1.039, 2.327, 2.934, 9.162, 1.606, 1.415, 3.503, 5.718, 0.081, 0.461, 3.153, 1.793, 5.723, 5.565, 1.415, 0.066, 5.036, 4.79, 6.284, 1.992, 0.759, 1.176, 0.139, 1.162, 0.102, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.06, 0.004, 0.003, 0.002, 0.001, 0.001, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.001, 0.003, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.031, 0.006, 0.001, 0.001, 0.001, 0.002, 0.014, 0.001, 0.001, 0.005, 0.005, 0.001, 0.002, 0.017, 0.007, 0.002, 0.003, 0.004, 0.002, 0.001, 0.002, 0.002, 0.012, 0.001, 0.002, 0.001, 0.004, 0.001, 0.001, 0.003, 0.003, 0.002, 0.005, 0.001, 0.001, 0.003, 0.001, 0.003, 0.001, 0.002, 0.001, 0.004, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.02, 0.047, 0.009, 0.009, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.003, 0.001, 0.004, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.005, 0.002, 0.061, 0.001, 0.0001, 0.002, 0.001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001], "ru": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.512, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 7.274, 0.002, 0.063, 0.0001, 0.001, 0.009, 0.001, 0.001, 0.118, 0.118, 0.0001, 0.001, 0.595, 0.135, 0.534, 0.009, 0.18, 0.281, 0.15, 0.078, 0.076, 0.077, 0.068, 0.066, 0.083, 0.16, 0.036, 0.016, 0.002, 0.001, 0.002, 0.001, 0.0001, 0.013, 0.009, 0.014, 0.009, 0.007, 0.006, 0.007, 0.006, 0.031, 0.002, 0.003, 0.007, 0.012, 0.007, 0.005, 0.01, 0.001, 0.008, 0.017, 0.011, 0.003, 0.009, 0.005, 0.012, 0.001, 0.001, 0.001, 0.0001, 0.001, 0.0001, 0.003, 0.0001, 0.065, 0.009, 0.022, 0.021, 0.074, 0.01, 0.013, 0.019, 0.054, 0.001, 0.008, 0.036, 0.02, 0.047, 0.055, 0.013, 0.001, 0.052, 0.037, 0.041, 0.026, 0.007, 0.006, 0.003, 0.011, 0.003, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 2.469, 2.363, 2.342, 0.986, 0.156, 0.422, 0.252, 0.495, 0.217, 0.136, 0.014, 0.778, 0.56, 0.097, 0.251, 0.811, 0.09, 0.184, 0.165, 0.06, 0.179, 0.021, 0.013, 0.029, 0.05, 0.005, 0.116, 0.045, 0.087, 0.073, 0.067, 0.124, 0.211, 0.16, 0.055, 0.033, 0.036, 0.024, 0.013, 0.02, 0.022, 0.002, 0.0001, 0.1, 0.0001, 0.025, 0.009, 0.011, 3.536, 0.619, 1.963, 0.833, 1.275, 3.452, 0.323, 0.635, 3.408, 0.642, 1.486, 1.967, 1.26, 2.857, 4.587, 1.082, 0.0001, 0.0001, 0.339, 0.003, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.013, 0.0001, 0.002, 0.001, 31.356, 12.318, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.131, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001], "de": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.726, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 13.303, 0.002, 0.278, 0.0001, 0.0001, 0.007, 0.003, 0.005, 0.149, 0.149, 0.015, 0.001, 0.636, 0.237, 0.922, 0.023, 0.305, 0.472, 0.225, 0.115, 0.11, 0.121, 0.108, 0.11, 0.145, 0.271, 0.049, 0.022, 0.002, 0.002, 0.002, 0.001, 0.0001, 0.413, 0.383, 0.144, 0.412, 0.275, 0.258, 0.273, 0.218, 0.18, 0.167, 0.277, 0.201, 0.328, 0.179, 0.111, 0.254, 0.012, 0.219, 0.602, 0.209, 0.1, 0.185, 0.206, 0.005, 0.01, 0.112, 0.002, 0.0001, 0.002, 0.0001, 0.006, 0.0001, 4.417, 1.306, 1.99, 3.615, 12.382, 1.106, 2.0, 2.958, 6.179, 0.082, 0.866, 2.842, 1.869, 7.338, 2.27, 0.606, 0.016, 6.056, 4.424, 4.731, 3.002, 0.609, 0.918, 0.053, 0.169, 0.824, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.147, 0.002, 0.003, 0.001, 0.006, 0.001, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.0001, 0.001, 0.004, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.03, 0.0001, 0.0001, 0.009, 0.001, 0.002, 0.009, 0.002, 0.001, 0.061, 0.0001, 0.048, 0.122, 0.057, 0.009, 0.001, 0.001, 0.4, 0.001, 0.002, 0.003, 0.003, 0.017, 0.001, 0.003, 0.001, 0.005, 0.0001, 0.001, 0.003, 0.002, 0.003, 0.005, 0.001, 0.001, 0.203, 0.0001, 0.002, 0.001, 0.002, 0.002, 0.438, 0.002, 0.002, 0.001, 0.0001, 0.0001, 0.056, 1.237, 0.01, 0.013, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.003, 0.001, 0.005, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.148, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001], "ja": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.834, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.258, 0.007, 0.036, 0.001, 0.0001, 0.005, 0.002, 0.003, 0.033, 0.033, 0.0001, 0.002, 0.019, 0.052, 0.026, 0.009, 0.281, 0.407, 0.259, 0.126, 0.108, 0.109, 0.095, 0.092, 0.104, 0.184, 0.008, 0.001, 0.002, 0.002, 0.002, 0.001, 0.0001, 0.048, 0.026, 0.039, 0.027, 0.028, 0.022, 0.018, 0.016, 0.03, 0.012, 0.014, 0.02, 0.03, 0.025, 0.025, 0.026, 0.002, 0.026, 0.045, 0.031, 0.013, 0.014, 0.014, 0.006, 0.006, 0.003, 0.001, 0.0001, 0.001, 0.0001, 0.002, 0.0001, 0.077, 0.012, 0.03, 0.026, 0.088, 0.012, 0.017, 0.025, 0.067, 0.002, 0.016, 0.041, 0.039, 0.059, 0.066, 0.016, 0.001, 0.06, 0.043, 0.051, 0.028, 0.009, 0.007, 0.004, 0.015, 0.004, 0.0001, 0.011, 0.0001, 0.0001, 0.0001, 2.555, 10.322, 5.875, 4.462, 0.784, 0.468, 0.442, 0.409, 1.173, 0.96, 0.657, 1.448, 1.442, 0.636, 0.341, 0.685, 0.495, 0.342, 0.651, 0.536, 0.435, 0.657, 0.51, 0.978, 0.31, 0.563, 0.439, 0.514, 0.668, 0.438, 0.29, 1.039, 0.423, 0.532, 0.407, 0.691, 0.677, 0.555, 0.911, 0.887, 1.086, 0.531, 0.836, 1.345, 0.438, 0.666, 1.528, 0.959, 0.535, 0.379, 0.302, 0.822, 0.614, 0.308, 0.253, 0.467, 0.807, 0.807, 0.777, 0.809, 1.292, 0.546, 0.524, 0.425, 0.0001, 0.0001, 0.002, 0.004, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.0001, 0.015, 19.387, 1.167, 4.022, 2.518, 1.734, 1.339, 1.229, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.409, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001], @@ -726,7 +740,7 @@ const COMMON_LANG_FREQS = { "sl": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.06, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 14.437, 0.024, 0.237, 0.001, 0.001, 0.007, 0.002, 0.011, 0.174, 0.174, 0.021, 0.002, 1.072, 0.17, 1.037, 0.022, 0.277, 0.429, 0.215, 0.122, 0.124, 0.121, 0.109, 0.108, 0.134, 0.239, 0.061, 0.025, 0.005, 0.006, 0.005, 0.002, 0.0001, 0.162, 0.141, 0.1, 0.122, 0.063, 0.075, 0.091, 0.086, 0.111, 0.082, 0.154, 0.138, 0.185, 0.145, 0.099, 0.224, 0.004, 0.106, 0.263, 0.133, 0.042, 0.163, 0.031, 0.007, 0.007, 0.087, 0.013, 0.0001, 0.014, 0.0001, 0.006, 0.0001, 7.7, 1.204, 0.709, 2.364, 7.782, 0.229, 1.139, 0.879, 6.985, 3.327, 2.701, 3.64, 2.037, 5.283, 6.653, 2.232, 0.006, 4.152, 3.513, 3.409, 1.654, 3.049, 0.039, 0.016, 0.079, 1.473, 0.0001, 0.01, 0.0001, 0.0001, 0.0001, 0.054, 0.004, 0.003, 0.002, 0.002, 0.001, 0.001, 0.011, 0.002, 0.002, 0.0001, 0.001, 0.021, 0.847, 0.001, 0.0001, 0.001, 0.002, 0.002, 0.027, 0.001, 0.0001, 0.001, 0.001, 0.001, 0.002, 0.001, 0.001, 0.002, 0.001, 0.001, 0.001, 0.056, 0.644, 0.007, 0.001, 0.003, 0.001, 0.001, 0.002, 0.003, 0.013, 0.001, 0.027, 0.001, 0.005, 0.001, 0.001, 0.007, 0.003, 0.004, 0.005, 0.002, 0.003, 0.006, 0.001, 0.004, 0.002, 0.004, 0.028, 0.008, 0.018, 0.391, 0.002, 0.0001, 0.0001, 0.071, 0.059, 0.881, 1.071, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.01, 0.005, 0.024, 0.008, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.002, 0.054, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001], "lv": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.879, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 12.099, 0.004, 0.432, 0.0001, 0.0001, 0.013, 0.002, 0.007, 0.207, 0.208, 0.0001, 0.003, 0.965, 0.082, 1.276, 0.01, 0.332, 0.476, 0.254, 0.122, 0.117, 0.123, 0.105, 0.106, 0.127, 0.271, 0.045, 0.023, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.208, 0.134, 0.062, 0.128, 0.074, 0.067, 0.074, 0.058, 0.112, 0.068, 0.189, 0.194, 0.144, 0.089, 0.055, 0.234, 0.002, 0.136, 0.249, 0.163, 0.042, 0.182, 0.012, 0.007, 0.003, 0.051, 0.001, 0.0001, 0.001, 0.0001, 0.003, 0.0001, 8.58, 1.078, 0.806, 2.221, 4.451, 0.231, 1.228, 0.175, 6.667, 1.704, 2.603, 2.424, 2.389, 3.209, 2.883, 1.908, 0.003, 4.056, 5.825, 4.121, 3.633, 1.801, 0.012, 0.009, 0.029, 1.289, 0.0001, 0.005, 0.0001, 0.0001, 0.0001, 0.124, 2.988, 0.003, 0.002, 0.001, 0.006, 0.331, 0.001, 0.002, 0.001, 0.0001, 0.001, 0.015, 0.083, 0.0001, 0.001, 0.001, 0.001, 0.007, 1.174, 0.07, 0.0001, 0.001, 0.001, 0.002, 0.003, 0.001, 0.001, 0.005, 0.012, 0.009, 0.001, 0.06, 0.627, 0.004, 0.097, 0.002, 0.001, 0.001, 0.001, 0.001, 0.002, 0.006, 1.565, 0.0001, 0.002, 0.0001, 0.0001, 0.01, 0.002, 0.005, 0.002, 0.002, 0.005, 0.01, 0.106, 0.006, 0.002, 0.003, 0.01, 0.298, 0.012, 0.176, 0.002, 0.0001, 0.0001, 0.03, 0.013, 6.068, 1.452, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.004, 0.002, 0.051, 0.018, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.001, 0.11, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001], "et": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.183, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 11.759, 0.003, 0.281, 0.0001, 0.0001, 0.013, 0.001, 0.037, 0.198, 0.199, 0.001, 0.003, 0.786, 0.203, 1.175, 0.017, 0.35, 0.548, 0.272, 0.142, 0.137, 0.143, 0.127, 0.129, 0.154, 0.323, 0.059, 0.022, 0.017, 0.003, 0.017, 0.003, 0.0001, 0.235, 0.096, 0.074, 0.061, 0.173, 0.056, 0.064, 0.105, 0.122, 0.088, 0.255, 0.166, 0.186, 0.114, 0.065, 0.208, 0.003, 0.138, 0.296, 0.251, 0.046, 0.167, 0.033, 0.011, 0.008, 0.01, 0.008, 0.0001, 0.008, 0.0001, 0.004, 0.0001, 9.665, 0.664, 0.152, 2.822, 7.678, 0.189, 1.393, 1.095, 7.816, 1.25, 3.234, 4.738, 2.585, 4.03, 3.549, 1.167, 0.005, 3.003, 6.68, 5.333, 4.153, 1.613, 0.043, 0.017, 0.074, 0.045, 0.0001, 0.002, 0.0001, 0.0001, 0.0001, 0.13, 0.015, 0.01, 0.006, 0.004, 0.003, 0.003, 0.004, 0.002, 0.002, 0.001, 0.002, 0.003, 0.005, 0.001, 0.003, 0.002, 0.002, 0.003, 0.102, 0.002, 0.008, 0.003, 0.003, 0.002, 0.004, 0.002, 0.001, 0.044, 0.005, 0.006, 0.003, 0.016, 0.035, 0.003, 0.002, 0.833, 0.002, 0.001, 0.002, 0.002, 0.01, 0.001, 0.006, 0.001, 0.005, 0.001, 0.001, 0.017, 0.004, 0.012, 0.007, 0.005, 0.763, 0.179, 0.003, 0.015, 0.005, 0.008, 0.007, 0.518, 0.012, 0.028, 0.003, 0.0001, 0.0001, 0.02, 2.358, 0.019, 0.061, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.009, 0.004, 0.104, 0.037, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.002, 0.002, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.004, 0.123, 0.001, 0.0001, 0.002, 0.001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001], - "hi": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.374, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 7.123, 0.002, 0.071, 0.0001, 0.001, 0.004, 0.0001, 0.023, 0.08, 0.08, 0.0001, 0.001, 0.255, 0.072, 0.052, 0.006, 0.068, 0.07, 0.044, 0.02, 0.019, 0.023, 0.019, 0.019, 0.021, 0.04, 0.021, 0.006, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.008, 0.004, 0.007, 0.004, 0.005, 0.003, 0.004, 0.003, 0.006, 0.001, 0.002, 0.003, 0.005, 0.004, 0.003, 0.005, 0.0001, 0.003, 0.008, 0.005, 0.002, 0.002, 0.002, 0.001, 0.001, 0.001, 0.007, 0.0001, 0.008, 0.0001, 0.001, 0.0001, 0.049, 0.007, 0.017, 0.016, 0.052, 0.008, 0.01, 0.017, 0.038, 0.001, 0.004, 0.024, 0.015, 0.034, 0.035, 0.012, 0.001, 0.033, 0.03, 0.034, 0.015, 0.005, 0.005, 0.002, 0.008, 0.001, 0.0001, 0.005, 0.0001, 0.0001, 0.0001, 1.039, 0.443, 1.278, 0.061, 0.0001, 0.273, 0.146, 1.879, 0.535, 0.214, 0.013, 0.729, 0.054, 1.826, 0.0001, 0.253, 0.014, 0.012, 0.0001, 0.042, 0.14, 2.07, 0.133, 0.43, 0.035, 0.004, 0.215, 0.046, 0.503, 0.014, 0.016, 0.269, 0.037, 0.213, 0.023, 0.155, 24.777, 7.162, 0.554, 0.224, 1.23, 0.009, 0.8, 0.117, 0.393, 0.245, 0.995, 0.828, 2.018, 0.001, 0.771, 0.001, 0.001, 0.707, 0.299, 0.18, 1.226, 0.94, 0.0001, 0.0001, 0.133, 0.001, 2.558, 1.303, 0.0001, 0.0001, 0.008, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 30.261, 0.0001, 0.024, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001], + "hi": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.374, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 7.123, 0.002, 0.071, 0.0001, 0.001, 0.004, 0.0001, 0.023, 0.08, 0.08, 0.0001, 0.001, 0.255, 0.072, 0.052, 0.006, 0.068, 0.07, 0.044, 0.02, 0.019, 0.023, 0.019, 0.019, 0.021, 0.04, 0.021, 0.006, 0.001, 0.002, 0.001, 0.001, 0.0001, 0.008, 0.004, 0.007, 0.004, 0.005, 0.003, 0.004, 0.003, 0.006, 0.001, 0.002, 0.003, 0.005, 0.004, 0.003, 0.005, 0.0001, 0.003, 0.008, 0.005, 0.002, 0.002, 0.002, 0.001, 0.001, 0.001, 0.007, 0.0001, 0.008, 0.0001, 0.001, 0.0001, 0.049, 0.007, 0.017, 0.016, 0.052, 0.008, 0.01, 0.017, 0.038, 0.001, 0.004, 0.024, 0.015, 0.034, 0.035, 0.012, 0.001, 0.033, 0.03, 0.034, 0.015, 0.005, 0.005, 0.002, 0.008, 0.001, 0.0001, 0.005, 0.0001, 0.0001, 0.0001, 1.039, 0.443, 1.278, 0.061, 0.0001, 0.273, 0.146, 1.879, 0.535, 0.214, 0.013, 0.729, 0.054, 1.826, 0.0001, 0.253, 0.014, 0.012, 0.0001, 0.042, 0.14, 2.07, 0.133, 0.43, 0.035, 0.004, 0.215, 0.046, 0.503, 0.014, 0.016, 0.269, 0.037, 0.213, 0.023, 0.155, 24.777, 7.162, 0.554, 0.224, 1.23, 0.009, 0.8, 0.117, 0.393, 0.245, 0.995, 0.828, 2.018, 0.001, 0.771, 0.001, 0.001, 0.707, 0.299, 0.18, 1.226, 0.94, 0.0001, 0.0001, 0.133, 0.001, 2.558, 1.303, 0.0001, 0.0001, 0.008, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.001, 0.0001, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.002, 0.001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 30.261, 0.0001, 0.024, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001], }; const EXTENSIVE_LANG_FREQS = Object.assign({}, COMMON_LANG_FREQS, { "aa": [0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 5.161, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 13.548, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.29, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.29, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 6.452, 0.645, 0.645, 2.581, 9.032, 0.0001, 5.161, 3.871, 5.806, 0.0001, 1.935, 2.581, 1.29, 5.161, 2.581, 1.29, 0.0001, 4.516, 0.645, 3.226, 0.645, 0.0001, 1.29, 0.0001, 0.645, 1.29, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 2.581, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 1.29, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 1.29, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 3.871, 0.645, 2.581, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.645, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001, 0.0001],