From a6fa0628f2c003f7aa93a94e06abdab66156174a Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 25 Nov 2019 22:53:31 +0100 Subject: [PATCH 1/2] Add operation to normalise unicode --- package-lock.json | 5 ++ package.json | 1 + src/core/config/Categories.json | 1 + src/core/lib/ChrEnc.mjs | 12 +++++ src/core/operations/NormaliseUnicode.mjs | 60 +++++++++++++++++++++ tests/operations/index.mjs | 1 + tests/operations/tests/NormaliseUnicode.mjs | 54 +++++++++++++++++++ 7 files changed, 134 insertions(+) create mode 100644 src/core/operations/NormaliseUnicode.mjs create mode 100644 tests/operations/tests/NormaliseUnicode.mjs diff --git a/package-lock.json b/package-lock.json index 6c765b4e..15aa5451 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14403,6 +14403,11 @@ "normalize-path": "^2.1.1" } }, + "unorm": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/unorm/-/unorm-1.6.0.tgz", + "integrity": "sha512-b2/KCUlYZUeA7JFUuRJZPUtr4gZvBh7tavtv4fvk4+KV9pfGiR6CQAQAWl49ZpR3ts2dk4FYkP7EIgDJoiOLDA==" + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/package.json b/package.json index 7bdea20b..61136169 100644 --- a/package.json +++ b/package.json @@ -147,6 +147,7 @@ "ssdeep.js": "0.0.2", "tesseract.js": "^2.0.0-alpha.15", "ua-parser-js": "^0.7.20", + "unorm": "^1.6.0", "utf8": "^3.0.0", "vkbeautify": "^0.99.3", "xmldom": "^0.1.27", diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index f663e16d..53ca796d 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -39,6 +39,7 @@ "URL Decode", "Escape Unicode Characters", "Unescape Unicode Characters", + "Normalise Unicode", "To Quoted Printable", "From Quoted Printable", "To Punycode", diff --git a/src/core/lib/ChrEnc.mjs b/src/core/lib/ChrEnc.mjs index a472706b..d2d0625f 100644 --- a/src/core/lib/ChrEnc.mjs +++ b/src/core/lib/ChrEnc.mjs @@ -164,3 +164,15 @@ export const IO_FORMAT = { "Simplified Chinese GB18030 (54936)": 54936, }; +/** + * Unicode Normalisation Forms + * + * @author Matthieu [m@tthieu.xyz] + * @copyright Crown Copyright 2016 + * @license Apache-2.0 + */ + +/** + * Character encoding format mappings. + */ +export const UNICODE_NORMALISATION_FORMS = ["NFD", "NFC", "NFKD", "NFKC"]; diff --git a/src/core/operations/NormaliseUnicode.mjs b/src/core/operations/NormaliseUnicode.mjs new file mode 100644 index 00000000..731a493f --- /dev/null +++ b/src/core/operations/NormaliseUnicode.mjs @@ -0,0 +1,60 @@ +/** + * @author Matthieu [m@tthieu.xyz] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import unorm from "unorm"; +import {UNICODE_NORMALISATION_FORMS} from "../lib/ChrEnc"; + +/** + * Normalise Unicode operation + */ +class NormaliseUnicode extends Operation { + + /** + * NormaliseUnicode constructor + */ + constructor() { + super(); + + this.name = "Normalise Unicode"; + this.module = "UnicodeNormalisation"; + this.description = "Transform Unicode to one of the Normalisation Form"; + this.infoURL = "http://www.unicode.org/reports/tr15/"; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + name: "Normal Form", + type: "option", + value: UNICODE_NORMALISATION_FORMS + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const [normalForm] = args; + if (normalForm === "NFD") { + return unorm.nfd(input); + } else if (normalForm === "NFC") { + return unorm.nfc(input); + } else if (normalForm === "NFKD") { + return unorm.nfkd(input); + } else if (normalForm === "NFKC") { + return unorm.nfc(input); + } + + throw new OperationError("Unknown Normalisation Form"); + } + +} + +export default NormaliseUnicode; diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index c54fa7ef..97745878 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -57,6 +57,7 @@ import "./tests/MS.mjs"; import "./tests/Magic.mjs"; import "./tests/MorseCode.mjs"; import "./tests/NetBIOS.mjs"; +import "./tests/NormaliseUnicode.mjs"; import "./tests/OTP.mjs"; import "./tests/PGP.mjs"; import "./tests/PHP.mjs"; diff --git a/tests/operations/tests/NormaliseUnicode.mjs b/tests/operations/tests/NormaliseUnicode.mjs new file mode 100644 index 00000000..d903e865 --- /dev/null +++ b/tests/operations/tests/NormaliseUnicode.mjs @@ -0,0 +1,54 @@ +/** + * Text Encoding Brute Force tests. + * + * @author Matthieu [m@tthieux.xyz] + * + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ +import TestRegister from "../../lib/TestRegister.mjs"; + +TestRegister.addTests([ + { + name: "Normalise Unicode - NFD", + input: "\u00c7\u0043\u0327\u2160", + expectedMatch: /C\u0327C\u0327\u2160/, + recipeConfig: [ + { + op: "Normalise Unicode", + args: ["NFD"], + }, + ], + }, { + name: "Normalise Unicode - NFC", + input: "\u00c7\u0043\u0327\u2160", + expectedMatch: /\u00C7\u00C7\u2160/, + recipeConfig: [ + { + op: "Normalise Unicode", + args: ["NFC"], + }, + ], + }, { + name: "Normalise Unicode - NFKD", + input: "\u00c7\u0043\u0327\u2160", + expectedMatch: /C\u0327C\u0327I/, + recipeConfig: [ + { + op: "Normalise Unicode", + args: ["NFKD"], + }, + ], + }, { + name: "Normalise Unicode - NFKC", + input: "\u00c7\u0043\u0327\u2160", + expectedMatch: /\u00C7\u00C7\u2160/, + recipeConfig: [ + { + op: "Normalise Unicode", + args: ["NFKC"], + }, + ], + }, +]); + From 23a228bbd9f3a76d055c378f50bb3cf66d32fe84 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 20 Dec 2019 16:05:24 +0000 Subject: [PATCH 2/2] Tidied up Normalise Unicode operation --- package-lock.json | 132 ++++++++++---------- src/core/lib/ChrEnc.mjs | 2 +- src/core/operations/NormaliseUnicode.mjs | 30 ++--- tests/operations/tests/NormaliseUnicode.mjs | 2 +- 4 files changed, 84 insertions(+), 82 deletions(-) diff --git a/package-lock.json b/package-lock.json index c595374d..d0f39b9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7118,28 +7118,28 @@ "dependencies": { "abbrev": { "version": "1.1.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true, "optional": true }, "ansi-regex": { "version": "2.1.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", "dev": true, "optional": true }, "aproba": { "version": "1.2.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "dev": true, "optional": true }, "are-we-there-yet": { "version": "1.1.5", - "resolved": false, + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", "dev": true, "optional": true, @@ -7150,14 +7150,14 @@ }, "balanced-match": { "version": "1.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true, "optional": true }, "brace-expansion": { "version": "1.1.11", - "resolved": false, + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "dev": true, "optional": true, @@ -7168,42 +7168,42 @@ }, "chownr": { "version": "1.1.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==", "dev": true, "optional": true }, "code-point-at": { "version": "1.1.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", "dev": true, "optional": true }, "concat-map": { "version": "0.0.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true, "optional": true }, "console-control-strings": { "version": "1.1.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", "dev": true, "optional": true }, "core-util-is": { "version": "1.0.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "dev": true, "optional": true }, "debug": { "version": "4.1.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "dev": true, "optional": true, @@ -7213,28 +7213,28 @@ }, "deep-extend": { "version": "0.6.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", "dev": true, "optional": true }, "delegates": { "version": "1.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", "dev": true, "optional": true }, "detect-libc": { "version": "1.0.3", - "resolved": false, + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", "dev": true, "optional": true }, "fs-minipass": { "version": "1.2.5", - "resolved": false, + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz", "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==", "dev": true, "optional": true, @@ -7244,14 +7244,14 @@ }, "fs.realpath": { "version": "1.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "dev": true, "optional": true }, "gauge": { "version": "2.7.4", - "resolved": false, + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", "dev": true, "optional": true, @@ -7268,7 +7268,7 @@ }, "glob": { "version": "7.1.3", - "resolved": false, + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", "dev": true, "optional": true, @@ -7283,14 +7283,14 @@ }, "has-unicode": { "version": "2.0.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", "dev": true, "optional": true }, "iconv-lite": { "version": "0.4.24", - "resolved": false, + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "dev": true, "optional": true, @@ -7300,7 +7300,7 @@ }, "ignore-walk": { "version": "3.0.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.1.tgz", "integrity": "sha512-DTVlMx3IYPe0/JJcYP7Gxg7ttZZu3IInhuEhbchuqneY9wWe5Ojy2mXLBaQFUQmo0AW2r3qG7m1mg86js+gnlQ==", "dev": true, "optional": true, @@ -7310,7 +7310,7 @@ }, "inflight": { "version": "1.0.6", - "resolved": false, + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "dev": true, "optional": true, @@ -7321,21 +7321,21 @@ }, "inherits": { "version": "2.0.3", - "resolved": false, + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", "dev": true, "optional": true }, "ini": { "version": "1.3.5", - "resolved": false, + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "dev": true, "optional": true }, "is-fullwidth-code-point": { "version": "1.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", "dev": true, "optional": true, @@ -7345,14 +7345,14 @@ }, "isarray": { "version": "1.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", "dev": true, "optional": true }, "minimatch": { "version": "3.0.4", - "resolved": false, + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "dev": true, "optional": true, @@ -7362,14 +7362,14 @@ }, "minimist": { "version": "0.0.8", - "resolved": false, + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", "dev": true, "optional": true }, "minipass": { "version": "2.3.5", - "resolved": false, + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", "dev": true, "optional": true, @@ -7380,7 +7380,7 @@ }, "minizlib": { "version": "1.2.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz", "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==", "dev": true, "optional": true, @@ -7390,7 +7390,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, "optional": true, @@ -7400,14 +7400,14 @@ }, "ms": { "version": "2.1.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", "dev": true, "optional": true }, "needle": { "version": "2.3.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/needle/-/needle-2.3.0.tgz", "integrity": "sha512-QBZu7aAFR0522EyaXZM0FZ9GLpq6lvQ3uq8gteiDUp7wKdy0lSd2hPlgFwVuW1CBkfEs9PfDQsQzZghLs/psdg==", "dev": true, "optional": true, @@ -7419,7 +7419,7 @@ }, "node-pre-gyp": { "version": "0.12.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.12.0.tgz", "integrity": "sha512-4KghwV8vH5k+g2ylT+sLTjy5wmUOb9vPhnM8NHvRf9dHmnW/CndrFXy2aRPaPST6dugXSdHXfeaHQm77PIz/1A==", "dev": true, "optional": true, @@ -7438,7 +7438,7 @@ }, "nopt": { "version": "4.0.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", "dev": true, "optional": true, @@ -7449,14 +7449,14 @@ }, "npm-bundled": { "version": "1.0.6", - "resolved": false, + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.6.tgz", "integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g==", "dev": true, "optional": true }, "npm-packlist": { "version": "1.4.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.1.tgz", "integrity": "sha512-+TcdO7HJJ8peiiYhvPxsEDhF3PJFGUGRcFsGve3vxvxdcpO2Z4Z7rkosRM0kWj6LfbK/P0gu3dzk5RU1ffvFcw==", "dev": true, "optional": true, @@ -7467,7 +7467,7 @@ }, "npmlog": { "version": "4.1.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", "dev": true, "optional": true, @@ -7480,21 +7480,21 @@ }, "number-is-nan": { "version": "1.0.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", "dev": true, "optional": true }, "object-assign": { "version": "4.1.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", "dev": true, "optional": true }, "once": { "version": "1.4.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "dev": true, "optional": true, @@ -7504,21 +7504,21 @@ }, "os-homedir": { "version": "1.0.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true, "optional": true }, "os-tmpdir": { "version": "1.0.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true, "optional": true }, "osenv": { "version": "0.1.5", - "resolved": false, + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", "dev": true, "optional": true, @@ -7529,21 +7529,21 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true, "optional": true }, "process-nextick-args": { "version": "2.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", "dev": true, "optional": true }, "rc": { "version": "1.2.8", - "resolved": false, + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", "dev": true, "optional": true, @@ -7556,7 +7556,7 @@ "dependencies": { "minimist": { "version": "1.2.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true, "optional": true @@ -7565,7 +7565,7 @@ }, "readable-stream": { "version": "2.3.6", - "resolved": false, + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", "dev": true, "optional": true, @@ -7581,7 +7581,7 @@ }, "rimraf": { "version": "2.6.3", - "resolved": false, + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", "dev": true, "optional": true, @@ -7591,49 +7591,49 @@ }, "safe-buffer": { "version": "5.1.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "dev": true, "optional": true }, "safer-buffer": { "version": "2.1.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "dev": true, "optional": true }, "sax": { "version": "1.2.4", - "resolved": false, + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", "dev": true, "optional": true }, "semver": { "version": "5.7.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz", "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA==", "dev": true, "optional": true }, "set-blocking": { "version": "2.0.0", - "resolved": false, + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "dev": true, "optional": true }, "signal-exit": { "version": "3.0.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "dev": true, "optional": true }, "string-width": { "version": "1.0.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "optional": true, @@ -7645,7 +7645,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "optional": true, @@ -7655,7 +7655,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "optional": true, @@ -7665,14 +7665,14 @@ }, "strip-json-comments": { "version": "2.0.1", - "resolved": false, + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", "dev": true, "optional": true }, "tar": { "version": "4.4.8", - "resolved": false, + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz", "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==", "dev": true, "optional": true, @@ -7688,14 +7688,14 @@ }, "util-deprecate": { "version": "1.0.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", "dev": true, "optional": true }, "wide-align": { "version": "1.1.3", - "resolved": false, + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", "dev": true, "optional": true, @@ -7705,14 +7705,14 @@ }, "wrappy": { "version": "1.0.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true, "optional": true }, "yallist": { "version": "3.0.3", - "resolved": false, + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", "dev": true, "optional": true diff --git a/src/core/lib/ChrEnc.mjs b/src/core/lib/ChrEnc.mjs index d2d0625f..c5cb5605 100644 --- a/src/core/lib/ChrEnc.mjs +++ b/src/core/lib/ChrEnc.mjs @@ -168,7 +168,7 @@ export const IO_FORMAT = { * Unicode Normalisation Forms * * @author Matthieu [m@tthieu.xyz] - * @copyright Crown Copyright 2016 + * @copyright Crown Copyright 2019 * @license Apache-2.0 */ diff --git a/src/core/operations/NormaliseUnicode.mjs b/src/core/operations/NormaliseUnicode.mjs index 731a493f..7de8625f 100644 --- a/src/core/operations/NormaliseUnicode.mjs +++ b/src/core/operations/NormaliseUnicode.mjs @@ -6,8 +6,8 @@ import Operation from "../Operation.mjs"; import OperationError from "../errors/OperationError.mjs"; +import {UNICODE_NORMALISATION_FORMS} from "../lib/ChrEnc.mjs"; import unorm from "unorm"; -import {UNICODE_NORMALISATION_FORMS} from "../lib/ChrEnc"; /** * Normalise Unicode operation @@ -21,9 +21,9 @@ class NormaliseUnicode extends Operation { super(); this.name = "Normalise Unicode"; - this.module = "UnicodeNormalisation"; - this.description = "Transform Unicode to one of the Normalisation Form"; - this.infoURL = "http://www.unicode.org/reports/tr15/"; + this.module = "Encodings"; + this.description = "Transform Unicode characters to one of the Normalisation Forms"; + this.infoURL = "https://wikipedia.org/wiki/Unicode_equivalence#Normal_forms"; this.inputType = "string"; this.outputType = "string"; this.args = [ @@ -42,17 +42,19 @@ class NormaliseUnicode extends Operation { */ run(input, args) { const [normalForm] = args; - if (normalForm === "NFD") { - return unorm.nfd(input); - } else if (normalForm === "NFC") { - return unorm.nfc(input); - } else if (normalForm === "NFKD") { - return unorm.nfkd(input); - } else if (normalForm === "NFKC") { - return unorm.nfc(input); - } - throw new OperationError("Unknown Normalisation Form"); + switch (normalForm) { + case "NFD": + return unorm.nfd(input); + case "NFC": + return unorm.nfc(input); + case "NFKD": + return unorm.nfkd(input); + case "NFKC": + return unorm.nfc(input); + default: + throw new OperationError("Unknown Normalisation Form"); + } } } diff --git a/tests/operations/tests/NormaliseUnicode.mjs b/tests/operations/tests/NormaliseUnicode.mjs index d903e865..78a41845 100644 --- a/tests/operations/tests/NormaliseUnicode.mjs +++ b/tests/operations/tests/NormaliseUnicode.mjs @@ -3,7 +3,7 @@ * * @author Matthieu [m@tthieux.xyz] * - * @copyright Crown Copyright 2018 + * @copyright Crown Copyright 2019 * @license Apache-2.0 */ import TestRegister from "../../lib/TestRegister.mjs";