Mirror/CyberChef
1
0
Fork 0
mirror of https://github.com/gchq/CyberChef.git synced 2024-11-02 14:11:02 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'mattnotmitt-features/unicode-format'

Browse Source
This commit is contained in:
n1474335 2021-02-01 15:51:29 +00:00
commit 0550aedd54
6 changed files with 153 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/core/config/Categories.json
View File

@ -200,6 +200,7 @@
"ops": [
"Encode text",
"Decode text",
"Unicode Text Format",
"Remove Diacritics",
"Unescape Unicode Characters",
"Convert to NATO alphabet"

2
src/core/operations/RemoveDiacritics.mjs
View File

@ -19,7 +19,7 @@ class RemoveDiacritics extends Operation {
this.name = "Remove Diacritics";
this.module = "Default";
this.description = "Replaces accented characters with their latin character equivalent.";
this.description = "Replaces accented characters with their latin character equivalent. Accented characters are made up of Unicode combining characters, so unicode text formatting such as strikethroughs and underlines will also be removed.";
this.infoURL = "https://wikipedia.org/wiki/Diacritic";
this.inputType = "string";
this.outputType = "string";

67
src/core/operations/UnicodeTextFormat.mjs Normal file
View File

@ -0,0 +1,67 @@
/**
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2020
* @license Apache-2.0
*/
import Operation from "../Operation.mjs";
import Utils from "../Utils.mjs";
/**
* Unicode Text Format operation
*/
class UnicodeTextFormat extends Operation {
/**
* UnicodeTextFormat constructor
*/
constructor() {
super();
this.name = "Unicode Text Format";
this.module = "Default";
this.description = "Adds Unicode combining characters to change formatting of plaintext.";
this.infoURL = "https://wikipedia.org/wiki/Combining_character";
this.inputType = "byteArray";
this.outputType = "byteArray";
this.args = [
{
name: "Underline",
type: "boolean",
value: "false"
},
{
name: "Strikethrough",
type: "boolean",
value: "false"
}
];
}
/**
* @param {byteArray} input
* @param {Object[]} args
* @returns {byteArray}
*/
run(input, args) {
const [underline, strikethrough] = args;
let output = input.map(char => [char]);
if (strikethrough) {
output = output.map(charFormat => {
charFormat.push(...Utils.strToUtf8ByteArray("\u0336"));
return charFormat;
});
}
if (underline) {
output = output.map(charFormat => {
charFormat.push(...Utils.strToUtf8ByteArray("\u0332"));
return charFormat;
});
}
// return output.flat(); - Not supported in Node 10, polyfilled
return [].concat(...output);
}
}
export default UnicodeTextFormat;

2
tests/operations/index.mjs
View File

@ -69,7 +69,6 @@ import "./tests/ParseQRCode.mjs";
import "./tests/PowerSet.mjs";
import "./tests/Regex.mjs";
import "./tests/Register.mjs";
import "./tests/RemoveDiacritics.mjs";
import "./tests/Rotate.mjs";
import "./tests/SeqUtils.mjs";
import "./tests/SetDifference.mjs";
@ -101,6 +100,7 @@ import "./tests/LuhnChecksum.mjs";
import "./tests/CipherSaber2.mjs";
import "./tests/Colossus.mjs";
import "./tests/ParseObjectIDTimestamp.mjs";
import "./tests/Unicode.mjs";
// Cannot test operations that use the File type yet

22
tests/operations/tests/RemoveDiacritics.mjs
View File

@ -1,22 +0,0 @@
/**
* Remove Diacritics tests.
*
* @author Klaxon [klaxon@veyr.com]
* @copyright Crown Copyright 2017
* @license Apache-2.0
*/
import TestRegister from "../../lib/TestRegister.mjs";
TestRegister.addTests([
{
name: "Remove Diacritics",
input: "\xe0, \xe8, \xec, \xf2, \xf9 \xc0, \xc8, \xcc, \xd2, \xd9\n\xe1, \xe9, \xed, \xf3, \xfa, \xfd \xc1, \xc9, \xcd, \xd3, \xda, \xdd\n\xe2, \xea, \xee, \xf4, \xfb \xc2, \xca, \xce, \xd4, \xdb\n\xe3, \xf1, \xf5 \xc3, \xd1, \xd5\n\xe4, \xeb, \xef, \xf6, \xfc, \xff \xc4, \xcb, \xcf, \xd6, \xdc, \u0178\n\xe5, \xc5",
expectedOutput: "a, e, i, o, u A, E, I, O, U\na, e, i, o, u, y A, E, I, O, U, Y\na, e, i, o, u A, E, I, O, U\na, n, o A, N, O\na, e, i, o, u, y A, E, I, O, U, Y\na, A",
recipeConfig: [
{
"op": "Remove Diacritics",
"args": []
},
],
},
]);

83
tests/operations/tests/Unicode.mjs Normal file
View File

@ -0,0 +1,83 @@
/**
* Unicode operation tests.
*
* @author Matt C [me@mitt.dev]
* @author Klaxon [klaxon@veyr.com]
*
* @copyright Crown Copyright 2020
* @license Apache-2.0
*/
import TestRegister from "../../lib/TestRegister.mjs";
TestRegister.addTests([
{
name: "Unicode Text Format: underline",
input: "a",
expectedOutput: "a\u0332",
recipeConfig: [
{
"op": "Unicode Text Format",
"args": [true, false],
}
],
},
{
name: "Unicode Text Format: strikethrough",
input: "a",
expectedOutput: "a\u0336",
recipeConfig: [
{
"op": "Unicode Text Format",
"args": [false, true],
}
],
},
{
name: "Unicode Text Format: both",
input: "a",
expectedOutput: "a\u0336\u0332",
recipeConfig: [
{
"op": "Unicode Text Format",
"args": [true, true],
}
],
},
{
name: "Remove Diacritics: text formatting",
input: "a",
expectedOutput: "a",
recipeConfig: [
{
"op": "Unicode Text Format",
"args": [true, true],
},
{
"op": "Remove Diacritics",
"args": []
}
],
},
{
name: "Remove Diacritics: all diacritical marks one char",
input: "à̴̵̶̷̸̡̢̧̨̛̖̗̘̙̜̝̞̟̠̣̤̥̦̩̪̫̬̭̮̯̰̱̲̳̹̺̻̼́̂̃̄̅̆̇̈̉̊̋̌̍̎̏̐̑̒̓̔̽̾̿̀́͂̓̈́̕̚͠͡ͅ", // sorry about this line lol
expectedOutput: "a",
recipeConfig: [
{
"op": "Remove Diacritics",
"args": []
}
],
},
{
name: "Remove Diacritics: default",
input: "\xe0, \xe8, \xec, \xf2, \xf9 \xc0, \xc8, \xcc, \xd2, \xd9\n\xe1, \xe9, \xed, \xf3, \xfa, \xfd \xc1, \xc9, \xcd, \xd3, \xda, \xdd\n\xe2, \xea, \xee, \xf4, \xfb \xc2, \xca, \xce, \xd4, \xdb\n\xe3, \xf1, \xf5 \xc3, \xd1, \xd5\n\xe4, \xeb, \xef, \xf6, \xfc, \xff \xc4, \xcb, \xcf, \xd6, \xdc, \u0178\n\xe5, \xc5",
expectedOutput: "a, e, i, o, u A, E, I, O, U\na, e, i, o, u, y A, E, I, O, U, Y\na, e, i, o, u A, E, I, O, U\na, n, o A, N, O\na, e, i, o, u, y A, E, I, O, U, Y\na, A",
recipeConfig: [
{
"op": "Remove Diacritics",
"args": []
},
],
},
]);