Merge branch 'master' of github.com:gchq/CyberChef into node-lib

This commit is contained in:
d98762625 2019-04-04 15:21:52 +01:00
commit e4ee0fc397
52 changed files with 6038 additions and 1539 deletions

View File

@ -1 +1,2 @@
src/core/vendor/**
src/web/static/clippy_assets/**

View File

@ -2,6 +2,12 @@
All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
### [8.29.0] - 2019-03-31
- 'BLAKE2s' and 'BLAKE2b' hashing operations added [@h345983745] | [#525]
### [8.28.0] - 2019-03-31
- 'Heatmap Chart', 'Hex Density Chart', 'Scatter Chart' and 'Series Chart' operation added [@artemisbot] [@tlwr] | [#496] [#143]
### [8.27.0] - 2019-03-14
- 'Enigma', 'Typex', 'Bombe' and 'Multiple Bombe' operations added [@s2224834] | [#516]
- See [this wiki article](https://github.com/gchq/CyberChef/wiki/Enigma,-the-Bombe,-and-Typex) for a full explanation of these operations.
@ -118,6 +124,8 @@ All major and minor version changes will be documented in this file. Details of
[8.29.0]: https://github.com/gchq/CyberChef/releases/tag/v8.29.0
[8.28.0]: https://github.com/gchq/CyberChef/releases/tag/v8.28.0
[8.27.0]: https://github.com/gchq/CyberChef/releases/tag/v8.27.0
[8.26.0]: https://github.com/gchq/CyberChef/releases/tag/v8.26.0
[8.25.0]: https://github.com/gchq/CyberChef/releases/tag/v8.25.0
@ -159,6 +167,7 @@ All major and minor version changes will be documented in this file. Details of
[@h345983745]: https://github.com/h345983745
[@s2224834]: https://github.com/s2224834
[@artemisbot]: https://github.com/artemisbot
[@tlwr]: https://github.com/tlwr
[@picapi]: https://github.com/picapi
[@Dachande663]: https://github.com/Dachande663
[@JustAnotherMark]: https://github.com/JustAnotherMark
@ -175,6 +184,7 @@ All major and minor version changes will be documented in this file. Details of
[#95]: https://github.com/gchq/CyberChef/pull/299
[#173]: https://github.com/gchq/CyberChef/pull/173
[#143]: https://github.com/gchq/CyberChef/pull/143
[#224]: https://github.com/gchq/CyberChef/pull/224
[#239]: https://github.com/gchq/CyberChef/pull/239
[#248]: https://github.com/gchq/CyberChef/pull/248
@ -209,5 +219,7 @@ All major and minor version changes will be documented in this file. Details of
[#468]: https://github.com/gchq/CyberChef/pull/468
[#476]: https://github.com/gchq/CyberChef/pull/476
[#489]: https://github.com/gchq/CyberChef/pull/489
[#496]: https://github.com/gchq/CyberChef/pull/496
[#506]: https://github.com/gchq/CyberChef/pull/506
[#516]: https://github.com/gchq/CyberChef/pull/516
[#525]: https://github.com/gchq/CyberChef/pull/525

View File

@ -158,7 +158,7 @@ module.exports = function (grunt) {
},
configs: ["*.{js,mjs}"],
core: ["src/core/**/*.{js,mjs}", "!src/core/vendor/**/*", "!src/core/operations/legacy/**/*"],
web: ["src/web/**/*.{js,mjs}"],
web: ["src/web/**/*.{js,mjs}", "!src/web/static/**/*"],
node: ["src/node/**/*.{js,mjs}"],
tests: ["tests/**/*.{js,mjs}"],
},
@ -351,7 +351,8 @@ module.exports = function (grunt) {
warningsFilter: [
/source-map/,
/dependency is an expression/,
/export 'default'/
/export 'default'/,
/Can't resolve 'sodium'/
],
}
},

View File

@ -11,14 +11,22 @@ module.exports = function(api) {
"node": "6.5"
},
"modules": false,
"useBuiltIns": "entry"
"useBuiltIns": "entry",
"corejs": 3
}]
],
"plugins": [
"babel-plugin-syntax-dynamic-import",
["babel-plugin-transform-builtin-extend", {
"globals": ["Error"]
}]
[
"babel-plugin-transform-builtin-extend", {
"globals": ["Error"]
}
],
[
"@babel/plugin-transform-runtime", {
"regenerator": true
}
]
]
};
};

3544
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,6 @@
{
"name": "cyberchef",
"version": "8.27.0",
"version": "8.29.1",
"description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
"author": "n1474335 <n1474335@gmail.com>",
"homepage": "https://gchq.github.io/CyberChef",
@ -31,20 +31,21 @@
"module": "src/node/index.mjs",
"bugs": "https://github.com/gchq/CyberChef/issues",
"devDependencies": {
"@babel/core": "^7.2.2",
"@babel/preset-env": "^7.2.3",
"autoprefixer": "^9.4.3",
"@babel/core": "^7.4.0",
"@babel/plugin-transform-runtime": "^7.4.0",
"@babel/preset-env": "^7.4.2",
"autoprefixer": "^9.5.0",
"babel-eslint": "^10.0.1",
"babel-loader": "^8.0.4",
"babel-loader": "^8.0.5",
"babel-plugin-syntax-dynamic-import": "^6.18.0",
"bootstrap": "^4.2.1",
"chromedriver": "^2.45.0",
"babel-polyfill": "^6.26.0",
"chromedriver": "^2.46.0",
"colors": "^1.3.3",
"css-loader": "^2.1.0",
"eslint": "^5.12.1",
"css-loader": "^2.1.1",
"eslint": "^5.15.3",
"exports-loader": "^0.7.0",
"file-loader": "^3.0.1",
"grunt": "^1.0.3",
"grunt": "^1.0.4",
"grunt-accessibility": "~6.0.0",
"grunt-chmod": "~1.1.1",
"grunt-concurrent": "^2.3.1",
@ -61,74 +62,82 @@
"ink-docstrap": "^1.3.2",
"jsdoc-babel": "^0.5.0",
"mini-css-extract-plugin": "^0.5.0",
"nightwatch": "^1.0.18",
"nightwatch": "^1.0.19",
"node-sass": "^4.11.0",
"postcss-css-variables": "^0.11.0",
"postcss-css-variables": "^0.12.0",
"postcss-import": "^12.0.1",
"postcss-loader": "^3.0.0",
"prompt": "^1.0.0",
"sass-loader": "^7.1.0",
"sitemap": "^2.1.0",
"style-loader": "^0.23.1",
"uglifyjs-webpack-plugin": "^2.0.1",
"svg-url-loader": "^2.3.2",
"uglifyjs-webpack-plugin": "^2.0.1",
"url-loader": "^1.1.2",
"web-resource-inliner": "^4.2.1",
"webpack": "^4.28.3",
"webpack-bundle-analyzer": "^3.0.3",
"webpack-dev-server": "^3.1.14",
"web-resource-inliner": "^4.3.1",
"webpack": "^4.29.6",
"webpack-bundle-analyzer": "^3.1.0",
"webpack-dev-server": "^3.2.1",
"webpack-node-externals": "^1.7.2",
"worker-loader": "^2.0.0"
},
"dependencies": {
"@babel/polyfill": "^7.4.0",
"@babel/runtime": "^7.4.2",
"arrive": "^2.4.1",
"babel-plugin-transform-builtin-extend": "1.1.2",
"babel-polyfill": "^6.26.0",
"bcryptjs": "^2.4.3",
"bignumber.js": "^8.0.2",
"bignumber.js": "^8.1.1",
"blakejs": "^1.1.0",
"bootstrap": "4.2.1",
"bootstrap-colorpicker": "^2.5.3",
"bootstrap-material-design": "^4.1.1",
"bson": "^4.0.1",
"bson": "^4.0.2",
"chi-squared": "^1.1.0",
"clippyjs": "0.0.3",
"core-js": "^3.0.0",
"crypto-api": "^0.8.3",
"crypto-js": "^3.1.9-1",
"ctph.js": "0.0.5",
"diff": "^3.5.0",
"d3": "^4.9.1",
"d3-hexbin": "^0.2.2",
"diff": "^4.0.1",
"es6-promisify": "^6.0.1",
"escodegen": "^1.11.0",
"escodegen": "^1.11.1",
"esmangle": "^1.0.1",
"esprima": "^4.0.1",
"exif-parser": "^0.1.12",
"file-saver": "^2.0.0",
"file-saver": "^2.0.1",
"geodesy": "^1.1.3",
"highlight.js": "^9.13.1",
"highlight.js": "^9.15.6",
"jimp": "^0.6.0",
"jquery": "^3.3.1",
"js-crc": "^0.2.0",
"js-sha3": "^0.8.0",
"jsesc": "^2.5.2",
"jsonpath": "^1.0.0",
"jsonwebtoken": "^8.4.0",
"jsqr": "^1.1.1",
"jsonpath": "^1.0.1",
"jsonwebtoken": "^8.5.1",
"jsqr": "^1.2.0",
"jsrsasign": "8.0.12",
"kbpgp": "^2.0.82",
"kbpgp": "2.1.0",
"libyara-wasm": "0.0.12",
"lodash": "^4.17.11",
"loglevel": "^1.6.1",
"loglevel-message-prefix": "^3.0.0",
"moment": "^2.23.0",
"moment": "^2.24.0",
"moment-timezone": "^0.5.23",
"ngeohash": "^0.6.3",
"node-forge": "^0.7.6",
"node-forge": "^0.8.2",
"node-md6": "^0.1.0",
"nodom": "^2.2.0",
"notepack.io": "^2.2.0",
"nwmatcher": "^1.4.4",
"otp": "^0.1.3",
"popper.js": "^1.14.6",
"popper.js": "^1.14.7",
"qr-image": "^3.2.0",
"scryptsy": "^2.0.0",
"snackbarjs": "^1.1.0",
"sortablejs": "^1.8.0-rc1",
"sortablejs": "^1.8.4",
"split.js": "^1.5.10",
"ssdeep.js": "0.0.2",
"ua-parser-js": "^0.7.19",

View File

@ -6,7 +6,6 @@
* @license Apache-2.0
*/
import "babel-polyfill";
import Chef from "./Chef";
import OperationConfig from "./config/OperationConfig.json";
import OpModules from "./config/modules/OpModules";

View File

@ -11,7 +11,7 @@ import BigNumber from "bignumber.js";
import log from "loglevel";
import {
DishArrayBuffer,
DishByteArray,
DishBigNumber,
DishFile,
DishHTML,
@ -199,7 +199,6 @@ class Dish {
return clone.get(type, notUTF8);
}
/**
* Validates that the value is the type that has been specified.
* May have to disable parts of BYTE_ARRAY validation if it effects performance.
@ -351,16 +350,17 @@ class Dish {
// Node environment => translate is sync
if (Utils.isNode()) {
this._toByteArray();
this._fromByteArray(toType, notUTF8);
this._toArrayBuffer();
this.type = Dish.ARRAY_BUFFER;
this._fromArrayBuffer(toType, notUTF8);
// Browser environment => translate is async
} else {
return new Promise((resolve, reject) => {
this._toByteArray()
.then(() => this.type = Dish.BYTE_ARRAY)
this._toArrayBuffer()
.then(() => this.type = Dish.ARRAY_BUFFER)
.then(() => {
this._fromByteArray(toType);
this._fromArrayBuffer(toType);
resolve();
})
.catch(reject);
@ -376,37 +376,37 @@ class Dish {
*
* @returns {Promise || undefined}
*/
_toByteArray() {
_toArrayBuffer() {
// Using 'bind' here to allow this.value to be mutated within translation functions
const toByteArrayFuncs = {
browser: {
[Dish.STRING]: () => Promise.resolve(DishString.toByteArray.bind(this)()),
[Dish.NUMBER]: () => Promise.resolve(DishNumber.toByteArray.bind(this)()),
[Dish.HTML]: () => Promise.resolve(DishHTML.toByteArray.bind(this)()),
[Dish.ARRAY_BUFFER]: () => Promise.resolve(DishArrayBuffer.toByteArray.bind(this)()),
[Dish.BIG_NUMBER]: () => Promise.resolve(DishBigNumber.toByteArray.bind(this)()),
[Dish.JSON]: () => Promise.resolve(DishJSON.toByteArray.bind(this)()),
[Dish.FILE]: () => DishFile.toByteArray.bind(this)(),
[Dish.LIST_FILE]: () => DishListFile.toByteArray.bind(this)(),
[Dish.BYTE_ARRAY]: () => Promise.resolve(),
[Dish.STRING]: () => Promise.resolve(DishString.toArrayBuffer.bind(this)()),
[Dish.NUMBER]: () => Promise.resolve(DishNumber.toArrayBuffer.bind(this)()),
[Dish.HTML]: () => Promise.resolve(DishHTML.toArrayBuffer.bind(this)()),
[Dish.ARRAY_BUFFER]: () => Promise.resolve(),
[Dish.BIG_NUMBER]: () => Promise.resolve(DishBigNumber.toArrayBuffer.bind(this)()),
[Dish.JSON]: () => Promise.resolve(DishJSON.toArrayBuffer.bind(this)()),
[Dish.FILE]: () => DishFile.toArrayBuffer.bind(this)(),
[Dish.LIST_FILE]: () => DishListFile.toArrayBuffer.bind(this)(),
[Dish.BYTE_ARRAY]: () => Promise.resolve(DishByteArray.toArrayBuffer.bind(this)()),
},
node: {
[Dish.STRING]: () => DishString.toByteArray.bind(this)(),
[Dish.NUMBER]: () => DishNumber.toByteArray.bind(this)(),
[Dish.HTML]: () => DishHTML.toByteArray.bind(this)(),
[Dish.ARRAY_BUFFER]: () => DishArrayBuffer.toByteArray.bind(this)(),
[Dish.BIG_NUMBER]: () => DishBigNumber.toByteArray.bind(this)(),
[Dish.JSON]: () => DishJSON.toByteArray.bind(this)(),
[Dish.FILE]: () => DishFile.toByteArray.bind(this)(),
[Dish.LIST_FILE]: () => DishListFile.toByteArray.bind(this)(),
[Dish.BYTE_ARRAY]: () => {},
[Dish.STRING]: () => DishString.toArrayBuffer.bind(this)(),
[Dish.NUMBER]: () => DishNumber.toArrayBuffer.bind(this)(),
[Dish.HTML]: () => DishHTML.toArrayBuffer.bind(this)(),
[Dish.ARRAY_BUFFER]: () => {},
[Dish.BIG_NUMBER]: () => DishBigNumber.toArrayBuffer.bind(this)(),
[Dish.JSON]: () => DishJSON.toArrayBuffer.bind(this)(),
[Dish.FILE]: () => DishFile.toArrayBuffer.bind(this)(),
[Dish.LIST_FILE]: () => DishListFile.toArrayBuffer.bind(this)(),
[Dish.BYTE_ARRAY]: () => DishByteArray.toArrayBuffer.bind(this)(),
}
};
try {
return toByteArrayFuncs[Utils.isNode() && "node" || "browser"][this.type]();
} catch (err) {
throw new DishError(`Error translating from ${Dish.enumLookup(this.type)} to byteArray: ${err}`);
throw new DishError(`Error translating from ${Dish.enumLookup(this.type)} to ArrayBuffer: ${err}`);
}
}
@ -416,33 +416,34 @@ class Dish {
* @param {number} toType - the Dish enum to convert to
* @param {boolean} [notUTF8=false] - Do not treat strings as UTF8.
*/
_fromByteArray(toType, notUTF8) {
const byteArrayToStr = notUTF8 ? Utils.byteArrayToChars : Utils.byteArrayToUtf8;
_fromArrayBuffer(toType, notUTF8) {
// Using 'bind' here to allow this.value to be mutated within translation functions
const toTypeFunctions = {
[Dish.STRING]: () => DishString.fromByteArray.bind(this)(byteArrayToStr),
[Dish.NUMBER]: () => DishNumber.fromByteArray.bind(this)(byteArrayToStr),
[Dish.HTML]: () => DishHTML.fromByteArray.bind(this)(byteArrayToStr),
[Dish.ARRAY_BUFFER]: () => DishArrayBuffer.fromByteArray.bind(this)(),
[Dish.BIG_NUMBER]: () => DishBigNumber.fromByteArray.bind(this)(byteArrayToStr),
[Dish.JSON]: () => DishJSON.fromByteArray.bind(this)(byteArrayToStr),
[Dish.FILE]: () => DishFile.fromByteArray.bind(this)(),
[Dish.LIST_FILE]: () => DishListFile.fromByteArray.bind(this)(),
[Dish.BYTE_ARRAY]: () => {},
[Dish.STRING]: () => DishString.fromArrayBuffer.bind(this)(notUTF8),
[Dish.NUMBER]: () => DishNumber.fromArrayBuffer.bind(this)(notUTF8),
[Dish.HTML]: () => DishHTML.fromArrayBuffer.bind(this)(notUTF8),
[Dish.ARRAY_BUFFER]: () => {},
[Dish.BIG_NUMBER]: () => DishBigNumber.fromArrayBuffer.bind(this)(notUTF8),
[Dish.JSON]: () => DishJSON.fromArrayBuffer.bind(this)(notUTF8),
[Dish.FILE]: () => DishFile.fromArrayBuffer.bind(this)(),
[Dish.LIST_FILE]: () => DishListFile.fromArrayBuffer.bind(this)(),
[Dish.BYTE_ARRAY]: () => DishByteArray.fromArrayBuffer.bind(this)(),
};
try {
toTypeFunctions[toType]();
this.type = toType;
} catch (err) {
throw new DishError(`Error translating from byteArray to ${Dish.enumLookup(toType)}: ${err}`);
throw new DishError(`Error translating from ArrayBuffer to ${Dish.enumLookup(toType)}: ${err}`);
}
}
}
/**
* Dish data type enum for byte arrays.
* @readonly

View File

@ -366,6 +366,61 @@ class Utils {
}
/**
* Converts a string to an ArrayBuffer.
* Treats the string as UTF-8 if any values are over 255.
*
* @param {string} str
* @returns {ArrayBuffer}
*
* @example
* // returns [72,101,108,108,111]
* Utils.strToArrayBuffer("Hello");
*
* // returns [228,189,160,229,165,189]
* Utils.strToArrayBuffer("你好");
*/
static strToArrayBuffer(str) {
const arr = new Uint8Array(str.length);
let i = str.length, b;
while (i--) {
b = str.charCodeAt(i);
arr[i] = b;
// If any of the bytes are over 255, read as UTF-8
if (b > 255) return Utils.strToUtf8ArrayBuffer(str);
}
return arr.buffer;
}
/**
* Converts a string to a UTF-8 ArrayBuffer.
*
* @param {string} str
* @returns {ArrayBuffer}
*
* @example
* // returns [72,101,108,108,111]
* Utils.strToUtf8ArrayBuffer("Hello");
*
* // returns [228,189,160,229,165,189]
* Utils.strToUtf8ArrayBuffer("你好");
*/
static strToUtf8ArrayBuffer(str) {
const utf8Str = utf8.encode(str);
if (str.length !== utf8Str.length) {
if (ENVIRONMENT_IS_WORKER()) {
self.setOption("attemptHighlight", false);
} else if (ENVIRONMENT_IS_WEB()) {
window.app.options.attemptHighlight = false;
}
}
return Utils.strToArrayBuffer(utf8Str);
}
/**
* Converts a string to a byte array.
* Treats the string as UTF-8 if any values are over 255.
@ -458,7 +513,7 @@ class Utils {
/**
* Attempts to convert a byte array to a UTF-8 string.
*
* @param {byteArray} byteArray
* @param {byteArray|Uint8Array} byteArray
* @returns {string}
*
* @example
@ -503,6 +558,7 @@ class Utils {
static byteArrayToChars(byteArray) {
if (!byteArray) return "";
let str = "";
// String concatenation appears to be faster than an array join
for (let i = 0; i < byteArray.length;) {
str += String.fromCharCode(byteArray[i++]);
}
@ -522,8 +578,8 @@ class Utils {
* Utils.arrayBufferToStr(Uint8Array.from([104,101,108,108,111]).buffer);
*/
static arrayBufferToStr(arrayBuffer, utf8=true) {
const byteArray = Array.prototype.slice.call(new Uint8Array(arrayBuffer));
return utf8 ? Utils.byteArrayToUtf8(byteArray) : Utils.byteArrayToChars(byteArray);
const arr = new Uint8Array(arrayBuffer);
return utf8 ? Utils.byteArrayToUtf8(arr) : Utils.byteArrayToChars(arr);
}
@ -970,17 +1026,15 @@ class Utils {
}
}
/** */
/**
*
*/
static readFileSync(file) {
if (!Utils.isNode()) {
throw new TypeError("Browser environment cannot support readFileSync");
}
let bytes = [];
for (const byte of file.data.values()) {
bytes = bytes.concat(byte);
}
return bytes;
return file.data.buffer;
}
@ -1040,9 +1094,11 @@ class Utils {
static charRep(token) {
return {
"Space": " ",
"Percent": "%",
"Comma": ",",
"Semi-colon": ";",
"Colon": ":",
"Tab": "\t",
"Line feed": "\n",
"CRLF": "\r\n",
"Forward slash": "/",
@ -1064,6 +1120,7 @@ class Utils {
static regexRep(token) {
return {
"Space": /\s+/g,
"Percent": /%/g,
"Comma": /,/g,
"Semi-colon": /;/g,
"Colon": /:/g,

View File

@ -297,6 +297,8 @@
"HAS-160",
"Whirlpool",
"Snefru",
"BLAKE2b",
"BLAKE2s",
"SSDEEP",
"CTPH",
"Compare SSDEEP hashes",
@ -378,7 +380,11 @@
"Image Filter",
"Contain Image",
"Cover Image",
"Image Hue/Saturation/Lightness"
"Image Hue/Saturation/Lightness",
"Hex Density chart",
"Scatter chart",
"Series chart",
"Heatmap chart"
]
},
{
@ -395,6 +401,7 @@
"Generate QR Code",
"Parse QR Code",
"Haversine distance",
"HTML To Text",
"Generate Lorem Ipsum",
"Numberwang",
"XKCD Random Number"

View File

@ -14,23 +14,22 @@ import BigNumber from "bignumber.js";
class DishBigNumber extends DishTranslationType {
/**
* convert the given value to a ByteArray
* convert the given value to a ArrayBuffer
* @param {BigNumber} value
*/
static toByteArray() {
static toArrayBuffer() {
DishBigNumber.checkForValue(this.value);
this.value = BigNumber.isBigNumber(this.value) ? Utils.strToByteArray(this.value.toFixed()) : [];
this.value = BigNumber.isBigNumber(this.value) ? Utils.strToArrayBuffer(this.value.toFixed()) : new ArrayBuffer;
}
/**
* convert the given value from a ByteArray
* @param {ByteArray} value
* @param {function} byteArrayToStr
* convert the given value from a ArrayBuffer
* @param {boolean} notUTF8
*/
static fromByteArray(byteArrayToStr) {
static fromArrayBuffer(notUTF8) {
DishBigNumber.checkForValue(this.value);
try {
this.value = new BigNumber(byteArrayToStr(this.value));
this.value = new BigNumber(Utils.arrayBufferToStr(this.value, !notUTF8));
} catch (err) {
this.value = new BigNumber(NaN);
}

View File

@ -9,24 +9,23 @@ import DishTranslationType from "./DishTranslationType";
/**
* Translation methods for ArrayBuffer Dishes
*/
class DishArrayBuffer extends DishTranslationType {
class DishByteArray extends DishTranslationType {
/**
* convert the given value to a ByteArray
* convert the given value to a ArrayBuffer
*/
static toByteArray() {
DishArrayBuffer.checkForValue(this.value);
this.value = Array.prototype.slice.call(new Uint8Array(this.value));
static toArrayBuffer() {
DishByteArray.checkForValue(this.value);
this.value = new Uint8Array(this.value).buffer;
}
/**
* convert the given value from a ByteArray
* @param {function} byteArrayToStr
* convert the given value from a ArrayBuffer
*/
static fromByteArray() {
DishArrayBuffer.checkForValue(this.value);
this.value = new Uint8Array(this.value).buffer;
static fromArrayBuffer() {
DishByteArray.checkForValue(this.value);
this.value = Array.prototype.slice.call(new Uint8Array(this.value));
}
}
export default DishArrayBuffer;
export default DishByteArray;

View File

@ -13,17 +13,18 @@ import Utils from "../Utils";
class DishFile extends DishTranslationType {
/**
* convert the given value to a ByteArray
* convert the given value to an ArrayBuffer
* @param {File} value
*/
static toByteArray() {
static toArrayBuffer() {
DishFile.checkForValue(this.value);
if (Utils.isNode()) {
// TODO
this.value = Utils.readFileSync(this.value);
} else {
return new Promise((resolve, reject) => {
Utils.readFile(this.value)
.then(v => this.value = Array.prototype.slice.call(v))
.then(v => this.value = v.buffer)
.then(resolve)
.catch(reject);
});
@ -31,13 +32,11 @@ class DishFile extends DishTranslationType {
}
/**
* convert the given value from a ByteArray
* @param {ByteArray} value
* @param {function} byteArrayToStr
* convert the given value from an ArrayBuffer
*/
static fromByteArray() {
static fromArrayBuffer() {
DishFile.checkForValue(this.value);
this.value = new File(this.value, "file.txt");
this.value = new File(this.value, "unknown");
}
}

View File

@ -14,21 +14,20 @@ import DishString from "./DishString";
class DishHTML extends DishTranslationType {
/**
* convert the given value to a ByteArray
* convert the given value to a ArrayBuffer
* @param {String} value
*/
static toByteArray() {
static toArrayBuffer() {
DishHTML.checkForValue(this.value);
this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : [];
this.value = this.value ? Utils.strToArrayBuffer(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : new ArrayBuffer;
}
/**
* convert the given value from a ByteArray
* @param {function} byteArrayToStr
* convert the given value from a ArrayBuffer
* @param {boolean} notUTF8
*/
static fromByteArray(byteArrayToStr) {
DishHTML.checkForValue(this.value);
DishString.fromByteArray(this.value, byteArrayToStr);
static fromArrayBuffer(notUTF8) {
DishString.fromByteArray(this.value, notUTF8);
}
}

View File

@ -13,21 +13,20 @@ import Utils from "../Utils";
class DishJSON extends DishTranslationType {
/**
* convert the given value to a ByteArray
* convert the given value to a ArrayBuffer
*/
static toByteArray() {
static toArrayBuffer() {
DishJSON.checkForValue(this.value);
this.value = this.value ? Utils.strToByteArray(JSON.stringify(this.value, null, 4)) : [];
this.value = this.value ? Utils.strToArrayBuffer(JSON.stringify(this.value, null, 4)) : new ArrayBuffer;
}
/**
* convert the given value from a ByteArray
* @param {ByteArray} value
* @param {function} byteArrayToStr
* convert the given value from a ArrayBuffer
* @param {boolean} notUTF8
*/
static fromByteArray(byteArrayToStr) {
static fromArrayBuffer(notUTF8) {
DishJSON.checkForValue(this.value);
this.value = JSON.parse(byteArrayToStr(this.value));
this.value = JSON.parse(Utils.arrayBufferToStr(this.value, !notUTF8));
}
}

View File

@ -13,30 +13,48 @@ import Utils from "../Utils";
class DishListFile extends DishTranslationType {
/**
* convert the given value to a ByteArray
* convert the given value to a ArrayBuffer
*/
static toByteArray() {
static toArrayBuffer() {
DishListFile.checkForValue(this.value);
if (Utils.isNode()) {
// TODO
this.value = [].concat.apply([], this.value.map(f => Utils.readFileSync(f)).map(b => Array.prototype.slice.call(b)));
} else {
return new Promise((resolve, reject) => {
Promise.all(this.value.map(async f => Utils.readFile(f)))
.then(values => this.value = values.map(b => [].concat.apply([], Array.prototype.slice.call(b))))
.then(resolve)
.catch(reject);
resolve(DishListFile.concatenateTypedArrays(...this.value).buffer);
});
}
}
/**
* convert the given value from a ByteArray
* @param {function} byteArrayToStr
* convert the given value from a ArrayBuffer
*/
static fromByteArray() {
static fromArrayBuffer() {
DishListFile.checkForValue(this.value);
this.value = [new File(this.value, "unknown")];
}
/**
* Concatenates a list of Uint8Arrays together
*
* @param {Uint8Array[]} arrays
* @returns {Uint8Array}
*/
static concatenateTypedArrays(...arrays) {
let totalLength = 0;
for (const arr of arrays) {
totalLength += arr.length;
}
const result = new Uint8Array(totalLength);
let offset = 0;
for (const arr of arrays) {
result.set(arr, offset);
offset += arr.length;
}
return result;
}
}
export default DishListFile;

View File

@ -14,20 +14,20 @@ import Utils from "../Utils";
class DishNumber extends DishTranslationType {
/**
* convert the given value to a ByteArray
* convert the given value to a ArrayBuffer
*/
static toByteArray() {
static toArrayBuffer() {
DishNumber.checkForValue(this.value);
this.value = typeof this.value === "number" ? Utils.strToByteArray(this.value.toString()) : [];
this.value = typeof this.value === "number" ? Utils.strToArrayBuffer(this.value.toString()) : new ArrayBuffer;
}
/**
* convert the given value from a ByteArray
* @param {function} byteArrayToStr
* convert the given value from a ArrayBuffer
* @param {boolean} notUTF8
*/
static fromByteArray(byteArrayToStr) {
static fromArrayBuffer(notUTF8) {
DishNumber.checkForValue(this.value);
this.value = this.value ? parseFloat(byteArrayToStr(this.value)) : 0;
this.value = this.value ? parseFloat(Utils.arrayBufferToStr(this.value, !notUTF8)) : 0;
}
}

View File

@ -14,20 +14,20 @@ import Utils from "../Utils";
class DishString extends DishTranslationType {
/**
* convert the given value to a ByteArray
* convert the given value to a ArrayBuffer
*/
static toByteArray() {
static toArrayBuffer() {
DishString.checkForValue(this.value);
this.value = this.value ? Utils.strToByteArray(this.value) : [];
this.value = this.value ? Utils.strToArrayBuffer(this.value) : new ArrayBuffer;
}
/**
* convert the given value from a ByteArray
* @param {function} byteArrayToStr
* convert the given value from a ArrayBuffer
* @param {boolean} notUTF8
*/
static fromByteArray(byteArrayToStr) {
static fromArrayBuffer(notUTF8) {
DishString.checkForValue(this.value);
this.value = this.value ? byteArrayToStr(this.value) : "";
this.value = this.value ? Utils.arrayBufferToStr(this.value, !notUTF8) : "";
}
}

View File

@ -20,18 +20,18 @@ class DishTranslationType {
}
/**
* convert the given value to a ByteArray
* convert the given value to a ArrayBuffer
* @param {*} value
*/
static toByteArray() {
static toArrayBuffer() {
throw new Error("toByteArray has not been implemented");
}
/**
* convert the given value from a ByteArray
* @param {function} byteArrayToStr
* convert the given value from a ArrayBuffer
* @param {boolean} notUTF8
*/
static fromByteArray(byteArrayToStr=undefined) {
static fromArrayBuffer(notUTF8=undefined) {
throw new Error("toType has not been implemented");
}
}

View File

@ -5,7 +5,7 @@
*/
import DishArrayBuffer from "./DishArrayBuffer";
import DishByteArray from "./DishByteArray";
import DishBigNumber from "./DishBigNumber";
import DishFile from "./DishFile";
import DishHTML from "./DishHTML";
@ -15,7 +15,7 @@ import DishNumber from "./DishNumber";
import DishString from "./DishString";
export {
DishArrayBuffer,
DishByteArray,
DishBigNumber,
DishFile,
DishHTML,

178
src/core/lib/Charts.mjs Normal file
View File

@ -0,0 +1,178 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import OperationError from "../errors/OperationError";
/**
* @constant
* @default
*/
export const RECORD_DELIMITER_OPTIONS = ["Line feed", "CRLF"];
/**
* @constant
* @default
*/
export const FIELD_DELIMITER_OPTIONS = ["Space", "Comma", "Semi-colon", "Colon", "Tab"];
/**
* Default from colour
*
* @constant
* @default
*/
export const COLOURS = {
min: "white",
max: "black"
};
/**
* Gets values from input for a plot.
*
* @param {string} input
* @param {string} recordDelimiter
* @param {string} fieldDelimiter
* @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
* @param {number} length
* @returns {Object[]}
*/
export function getValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded, length) {
let headings;
const values = [];
input
.split(recordDelimiter)
.forEach((row, rowIndex) => {
const split = row.split(fieldDelimiter);
if (split.length !== length) throw new OperationError(`Each row must have length ${length}.`);
if (columnHeadingsAreIncluded && rowIndex === 0) {
headings = split;
} else {
values.push(split);
}
});
return { headings, values };
}
/**
* Gets values from input for a scatter plot.
*
* @param {string} input
* @param {string} recordDelimiter
* @param {string} fieldDelimiter
* @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
* @returns {Object[]}
*/
export function getScatterValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
let { headings, values } = getValues(
input,
recordDelimiter,
fieldDelimiter,
columnHeadingsAreIncluded,
2
);
if (headings) {
headings = {x: headings[0], y: headings[1]};
}
values = values.map(row => {
const x = parseFloat(row[0], 10),
y = parseFloat(row[1], 10);
if (Number.isNaN(x)) throw new OperationError("Values must be numbers in base 10.");
if (Number.isNaN(y)) throw new OperationError("Values must be numbers in base 10.");
return [x, y];
});
return { headings, values };
}
/**
* Gets values from input for a scatter plot with colour from the third column.
*
* @param {string} input
* @param {string} recordDelimiter
* @param {string} fieldDelimiter
* @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
* @returns {Object[]}
*/
export function getScatterValuesWithColour(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
let { headings, values } = getValues(
input,
recordDelimiter, fieldDelimiter,
columnHeadingsAreIncluded,
3
);
if (headings) {
headings = {x: headings[0], y: headings[1]};
}
values = values.map(row => {
const x = parseFloat(row[0], 10),
y = parseFloat(row[1], 10),
colour = row[2];
if (Number.isNaN(x)) throw new OperationError("Values must be numbers in base 10.");
if (Number.isNaN(y)) throw new OperationError("Values must be numbers in base 10.");
return [x, y, colour];
});
return { headings, values };
}
/**
* Gets values from input for a time series plot.
*
* @param {string} input
* @param {string} recordDelimiter
* @param {string} fieldDelimiter
* @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
* @returns {Object[]}
*/
export function getSeriesValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
const { values } = getValues(
input,
recordDelimiter, fieldDelimiter,
false,
3
);
let xValues = new Set();
const series = {};
values.forEach(row => {
const serie = row[0],
xVal = row[1],
val = parseFloat(row[2], 10);
if (Number.isNaN(val)) throw new OperationError("Values must be numbers in base 10.");
xValues.add(xVal);
if (typeof series[serie] === "undefined") series[serie] = {};
series[serie][xVal] = val;
});
xValues = new Array(...xValues);
const seriesList = [];
for (const seriesName in series) {
const serie = series[seriesName];
seriesList.push({name: seriesName, data: serie});
}
return { xValues, series: seriesList };
}

View File

@ -100,7 +100,7 @@ export function fromHex(data, delim="Auto", byteLen=2) {
/**
* To Hexadecimal delimiters.
*/
export const TO_HEX_DELIM_OPTIONS = ["Space", "Comma", "Semi-colon", "Colon", "Line feed", "CRLF", "0x", "\\x", "None"];
export const TO_HEX_DELIM_OPTIONS = ["Space", "Percent", "Comma", "Semi-colon", "Colon", "Line feed", "CRLF", "0x", "\\x", "None"];
/**

View File

@ -0,0 +1,79 @@
/**
* @author h345983745
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
import blakejs from "blakejs";
import OperationError from "../errors/OperationError";
import Utils from "../Utils";
import { toBase64 } from "../lib/Base64";
/**
* BLAKE2b operation
*/
class BLAKE2b extends Operation {
/**
* BLAKE2b constructor
*/
constructor() {
super();
this.name = "BLAKE2b";
this.module = "Hashing";
this.description = `Performs BLAKE2b hashing on the input.
<br><br> BLAKE2b is a flavour of the BLAKE cryptographic hash function that is optimized for 64-bit platforms and produces digests of any size between 1 and 64 bytes.
<br><br> Supports the use of an optional key.`;
this.infoURL = "https://wikipedia.org/wiki/BLAKE_(hash_function)#BLAKE2b_algorithm";
this.inputType = "ArrayBuffer";
this.outputType = "string";
this.args = [
{
"name": "Size",
"type": "option",
"value": ["512", "384", "256", "160", "128"]
}, {
"name": "Output Encoding",
"type": "option",
"value": ["Hex", "Base64", "Raw"]
}, {
"name": "Key",
"type": "toggleString",
"value": "",
"toggleValues": ["UTF8", "Decimal", "Base64", "Hex", "Latin1"]
}
];
}
/**
* @param {ArrayBuffer} input
* @param {Object[]} args
* @returns {string} The input having been hashed with BLAKE2b in the encoding format speicifed.
*/
run(input, args) {
const [outSize, outFormat] = args;
let key = Utils.convertToByteArray(args[2].string || "", args[2].option);
if (key.length === 0) {
key = null;
} else if (key.length > 64) {
throw new OperationError(["Key cannot be greater than 64 bytes", "It is currently " + key.length + " bytes."].join("\n"));
}
input = new Uint8Array(input);
switch (outFormat) {
case "Hex":
return blakejs.blake2bHex(input, key, outSize / 8);
case "Base64":
return toBase64(blakejs.blake2b(input, key, outSize / 8));
case "Raw":
return Utils.arrayBufferToStr(blakejs.blake2b(input, key, outSize / 8).buffer);
default:
return new OperationError("Unsupported Output Type");
}
}
}
export default BLAKE2b;

View File

@ -0,0 +1,80 @@
/**
* @author h345983745
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
import blakejs from "blakejs";
import OperationError from "../errors/OperationError";
import Utils from "../Utils";
import { toBase64 } from "../lib/Base64";
/**
* BLAKE2s Operation
*/
class BLAKE2s extends Operation {
/**
* BLAKE2s constructor
*/
constructor() {
super();
this.name = "BLAKE2s";
this.module = "Hashing";
this.description = `Performs BLAKE2s hashing on the input.
<br><br>BLAKE2s is a flavour of the BLAKE cryptographic hash function that is optimized for 8- to 32-bit platforms and produces digests of any size between 1 and 32 bytes.
<br><br>Supports the use of an optional key.`;
this.infoURL = "https://wikipedia.org/wiki/BLAKE_(hash_function)#BLAKE2";
this.inputType = "ArrayBuffer";
this.outputType = "string";
this.args = [
{
"name": "Size",
"type": "option",
"value": ["256", "160", "128"]
}, {
"name": "Output Encoding",
"type": "option",
"value": ["Hex", "Base64", "Raw"]
},
{
"name": "Key",
"type": "toggleString",
"value": "",
"toggleValues": ["UTF8", "Decimal", "Base64", "Hex", "Latin1"]
}
];
}
/**
* @param {ArrayBuffer} input
* @param {Object[]} args
* @returns {string} The input having been hashed with BLAKE2s in the encoding format speicifed.
*/
run(input, args) {
const [outSize, outFormat] = args;
let key = Utils.convertToByteArray(args[2].string || "", args[2].option);
if (key.length === 0) {
key = null;
} else if (key.length > 32) {
throw new OperationError(["Key cannot be greater than 32 bytes", "It is currently " + key.length + " bytes."].join("\n"));
}
input = new Uint8Array(input);
switch (outFormat) {
case "Hex":
return blakejs.blake2sHex(input, key, outSize / 8);
case "Base64":
return toBase64(blakejs.blake2s(input, key, outSize / 8));
case "Raw":
return Utils.arrayBufferToStr(blakejs.blake2s(input, key, outSize / 8).buffer);
default:
return new OperationError("Unsupported Output Type");
}
}
}
export default BLAKE2s;

View File

@ -23,7 +23,7 @@ class ExtractFiles extends Operation {
this.name = "Extract Files";
this.module = "Default";
this.description = "TODO";
this.description = "Performs file carving to attempt to extract files from the input.<br><br>This operation is currently capable of carving out the following formats:<ul><li>JPG</li><li>EXE</li><li>ZIP</li><li>PDF</li><li>PNG</li><li>BMP</li><li>FLV</li><li>RTF</li><li>DOCX, PPTX, XLSX</li><li>EPUB</li><li>GZIP</li><li>ZLIB</li><li>ELF, BIN, AXF, O, PRX, SO</li></ul>";
this.infoURL = "https://forensicswiki.org/wiki/File_Carving";
this.inputType = "ArrayBuffer";
this.outputType = "List<File>";

View File

@ -28,6 +28,8 @@ import Fletcher64Checksum from "./Fletcher64Checksum";
import Adler32Checksum from "./Adler32Checksum";
import CRC16Checksum from "./CRC16Checksum";
import CRC32Checksum from "./CRC32Checksum";
import BLAKE2b from "./BLAKE2b";
import BLAKE2s from "./BLAKE2s";
/**
* Generate all hashes operation
@ -86,6 +88,14 @@ class GenerateAllHashes extends Operation {
"\nWhirlpool-0: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool-0"]) +
"\nWhirlpool-T: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool-T"]) +
"\nWhirlpool: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool"]) +
"\nBLAKE2b-128: " + (new BLAKE2b).run(arrayBuffer, ["128", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2b-160: " + (new BLAKE2b).run(arrayBuffer, ["160", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2b-256: " + (new BLAKE2b).run(arrayBuffer, ["256", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2b-384: " + (new BLAKE2b).run(arrayBuffer, ["384", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2b-512: " + (new BLAKE2b).run(arrayBuffer, ["512", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2s-128: " + (new BLAKE2s).run(arrayBuffer, ["128", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2s-160: " + (new BLAKE2s).run(arrayBuffer, ["160", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2s-256: " + (new BLAKE2s).run(arrayBuffer, ["256", "Hex", {string: "", option: "UTF8"}]) +
"\nSSDEEP: " + (new SSDEEP()).run(str) +
"\nCTPH: " + (new CTPH()).run(str) +
"\n\nChecksums:" +

View File

@ -0,0 +1,41 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
/**
* HTML To Text operation
*/
class HTMLToText extends Operation {
/**
* HTMLToText constructor
*/
constructor() {
super();
this.name = "HTML To Text";
this.module = "Default";
this.description = "Converts an HTML output from an operation to a readable string instead of being rendered in the DOM.";
this.infoURL = "";
this.inputType = "html";
this.outputType = "string";
this.args = [];
}
/**
* @param {html} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
return input;
}
}
export default HTMLToText;

View File

@ -0,0 +1,266 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import * as d3temp from "d3";
import * as nodomtemp from "nodom";
import { getScatterValues, RECORD_DELIMITER_OPTIONS, COLOURS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Utils from "../Utils";
const d3 = d3temp.default ? d3temp.default : d3temp;
const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
/**
* Heatmap chart operation
*/
class HeatmapChart extends Operation {
/**
* HeatmapChart constructor
*/
constructor() {
super();
this.name = "Heatmap chart";
this.module = "Charts";
this.description = "A heatmap is a graphical representation of data where the individual values contained in a matrix are represented as colors.";
this.infoURL = "https://wikipedia.org/wiki/Heat_map";
this.inputType = "string";
this.outputType = "html";
this.args = [
{
name: "Record delimiter",
type: "option",
value: RECORD_DELIMITER_OPTIONS,
},
{
name: "Field delimiter",
type: "option",
value: FIELD_DELIMITER_OPTIONS,
},
{
name: "Number of vertical bins",
type: "number",
value: 25,
},
{
name: "Number of horizontal bins",
type: "number",
value: 25,
},
{
name: "Use column headers as labels",
type: "boolean",
value: true,
},
{
name: "X label",
type: "string",
value: "",
},
{
name: "Y label",
type: "string",
value: "",
},
{
name: "Draw bin edges",
type: "boolean",
value: false,
},
{
name: "Min colour value",
type: "string",
value: COLOURS.min,
},
{
name: "Max colour value",
type: "string",
value: COLOURS.max,
},
];
}
/**
* Heatmap chart operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {html}
*/
run(input, args) {
const recordDelimiter = Utils.charRep(args[0]),
fieldDelimiter = Utils.charRep(args[1]),
vBins = args[2],
hBins = args[3],
columnHeadingsAreIncluded = args[4],
drawEdges = args[7],
minColour = args[8],
maxColour = args[9],
dimension = 500;
if (vBins <= 0) throw new OperationError("Number of vertical bins must be greater than 0");
if (hBins <= 0) throw new OperationError("Number of horizontal bins must be greater than 0");
let xLabel = args[5],
yLabel = args[6];
const { headings, values } = getScatterValues(
input,
recordDelimiter,
fieldDelimiter,
columnHeadingsAreIncluded
);
if (headings) {
xLabel = headings.x;
yLabel = headings.y;
}
const document = new nodom.Document();
let svg = document.createElement("svg");
svg = d3.select(svg)
.attr("width", "100%")
.attr("height", "100%")
.attr("viewBox", `0 0 ${dimension} ${dimension}`);
const margin = {
top: 10,
right: 0,
bottom: 40,
left: 30,
},
width = dimension - margin.left - margin.right,
height = dimension - margin.top - margin.bottom,
binWidth = width / hBins,
binHeight = height/ vBins,
marginedSpace = svg.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
const bins = this.getHeatmapPacking(values, vBins, hBins),
maxCount = Math.max(...bins.map(row => {
const lengths = row.map(cell => cell.length);
return Math.max(...lengths);
}));
const xExtent = d3.extent(values, d => d[0]),
yExtent = d3.extent(values, d => d[1]);
const xAxis = d3.scaleLinear()
.domain(xExtent)
.range([0, width]);
const yAxis = d3.scaleLinear()
.domain(yExtent)
.range([height, 0]);
const colour = d3.scaleSequential(d3.interpolateLab(minColour, maxColour))
.domain([0, maxCount]);
marginedSpace.append("clipPath")
.attr("id", "clip")
.append("rect")
.attr("width", width)
.attr("height", height);
marginedSpace.append("g")
.attr("class", "bins")
.attr("clip-path", "url(#clip)")
.selectAll("g")
.data(bins)
.enter()
.append("g")
.selectAll("rect")
.data(d => d)
.enter()
.append("rect")
.attr("x", (d) => binWidth * d.x)
.attr("y", (d) => (height - binHeight * (d.y + 1)))
.attr("width", binWidth)
.attr("height", binHeight)
.attr("fill", (d) => colour(d.length))
.attr("stroke", drawEdges ? "rgba(0, 0, 0, 0.5)" : "none")
.attr("stroke-width", drawEdges ? "0.5" : "none")
.append("title")
.text(d => {
const count = d.length,
perc = 100.0 * d.length / values.length,
tooltip = `Count: ${count}\n
Percentage: ${perc.toFixed(2)}%\n
`.replace(/\s{2,}/g, "\n");
return tooltip;
});
marginedSpace.append("g")
.attr("class", "axis axis--y")
.call(d3.axisLeft(yAxis).tickSizeOuter(-width));
svg.append("text")
.attr("transform", "rotate(-90)")
.attr("y", -margin.left)
.attr("x", -(height / 2))
.attr("dy", "1em")
.style("text-anchor", "middle")
.text(yLabel);
marginedSpace.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(xAxis).tickSizeOuter(-height));
svg.append("text")
.attr("x", width / 2)
.attr("y", dimension)
.style("text-anchor", "middle")
.text(xLabel);
return svg._groups[0][0].outerHTML;
}
/**
* Packs a list of x, y coordinates into a number of bins for use in a heatmap.
*
* @param {Object[]} points
* @param {number} number of vertical bins
* @param {number} number of horizontal bins
* @returns {Object[]} a list of bins (each bin is an Array) with x y coordinates, filled with the points
*/
getHeatmapPacking(values, vBins, hBins) {
const xBounds = d3.extent(values, d => d[0]),
yBounds = d3.extent(values, d => d[1]),
bins = [];
if (xBounds[0] === xBounds[1]) throw "Cannot pack points. There is no difference between the minimum and maximum X coordinate.";
if (yBounds[0] === yBounds[1]) throw "Cannot pack points. There is no difference between the minimum and maximum Y coordinate.";
for (let y = 0; y < vBins; y++) {
bins.push([]);
for (let x = 0; x < hBins; x++) {
const item = [];
item.y = y;
item.x = x;
bins[y].push(item);
} // x
} // y
const epsilon = 0.000000001; // This is to clamp values that are exactly the maximum;
values.forEach(v => {
const fractionOfY = (v[1] - yBounds[0]) / ((yBounds[1] + epsilon) - yBounds[0]),
fractionOfX = (v[0] - xBounds[0]) / ((xBounds[1] + epsilon) - xBounds[0]),
y = Math.floor(vBins * fractionOfY),
x = Math.floor(hBins * fractionOfX);
bins[y][x].push({x: v[0], y: v[1]});
});
return bins;
}
}
export default HeatmapChart;

View File

@ -0,0 +1,296 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import * as d3temp from "d3";
import * as d3hexbintemp from "d3-hexbin";
import * as nodomtemp from "nodom";
import { getScatterValues, RECORD_DELIMITER_OPTIONS, COLOURS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
import Operation from "../Operation";
import Utils from "../Utils";
const d3 = d3temp.default ? d3temp.default : d3temp;
const d3hexbin = d3hexbintemp.default ? d3hexbintemp.default : d3hexbintemp;
const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
/**
* Hex Density chart operation
*/
class HexDensityChart extends Operation {
/**
* HexDensityChart constructor
*/
constructor() {
super();
this.name = "Hex Density chart";
this.module = "Charts";
this.description = "Hex density charts are used in a similar way to scatter charts, however rather than rendering tens of thousands of points, it groups the points into a few hundred hexagons to show the distribution.";
this.inputType = "string";
this.outputType = "html";
this.args = [
{
name: "Record delimiter",
type: "option",
value: RECORD_DELIMITER_OPTIONS,
},
{
name: "Field delimiter",
type: "option",
value: FIELD_DELIMITER_OPTIONS,
},
{
name: "Pack radius",
type: "number",
value: 25,
},
{
name: "Draw radius",
type: "number",
value: 15,
},
{
name: "Use column headers as labels",
type: "boolean",
value: true,
},
{
name: "X label",
type: "string",
value: "",
},
{
name: "Y label",
type: "string",
value: "",
},
{
name: "Draw hexagon edges",
type: "boolean",
value: false,
},
{
name: "Min colour value",
type: "string",
value: COLOURS.min,
},
{
name: "Max colour value",
type: "string",
value: COLOURS.max,
},
{
name: "Draw empty hexagons within data boundaries",
type: "boolean",
value: false,
}
];
}
/**
* Hex Bin chart operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {html}
*/
run(input, args) {
const recordDelimiter = Utils.charRep(args[0]),
fieldDelimiter = Utils.charRep(args[1]),
packRadius = args[2],
drawRadius = args[3],
columnHeadingsAreIncluded = args[4],
drawEdges = args[7],
minColour = args[8],
maxColour = args[9],
drawEmptyHexagons = args[10],
dimension = 500;
let xLabel = args[5],
yLabel = args[6];
const { headings, values } = getScatterValues(
input,
recordDelimiter,
fieldDelimiter,
columnHeadingsAreIncluded
);
if (headings) {
xLabel = headings.x;
yLabel = headings.y;
}
const document = new nodom.Document();
let svg = document.createElement("svg");
svg = d3.select(svg)
.attr("width", "100%")
.attr("height", "100%")
.attr("viewBox", `0 0 ${dimension} ${dimension}`);
const margin = {
top: 10,
right: 0,
bottom: 40,
left: 30,
},
width = dimension - margin.left - margin.right,
height = dimension - margin.top - margin.bottom,
marginedSpace = svg.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
const hexbin = d3hexbin.hexbin()
.radius(packRadius)
.extent([0, 0], [width, height]);
const hexPoints = hexbin(values),
maxCount = Math.max(...hexPoints.map(b => b.length));
const xExtent = d3.extent(hexPoints, d => d.x),
yExtent = d3.extent(hexPoints, d => d.y);
xExtent[0] -= 2 * packRadius;
xExtent[1] += 3 * packRadius;
yExtent[0] -= 2 * packRadius;
yExtent[1] += 2 * packRadius;
const xAxis = d3.scaleLinear()
.domain(xExtent)
.range([0, width]);
const yAxis = d3.scaleLinear()
.domain(yExtent)
.range([height, 0]);
const colour = d3.scaleSequential(d3.interpolateLab(minColour, maxColour))
.domain([0, maxCount]);
marginedSpace.append("clipPath")
.attr("id", "clip")
.append("rect")
.attr("width", width)
.attr("height", height);
if (drawEmptyHexagons) {
marginedSpace.append("g")
.attr("class", "empty-hexagon")
.selectAll("path")
.data(this.getEmptyHexagons(hexPoints, packRadius))
.enter()
.append("path")
.attr("d", d => {
return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(drawRadius)}`;
})
.attr("fill", (d) => colour(0))
.attr("stroke", drawEdges ? "black" : "none")
.attr("stroke-width", drawEdges ? "0.5" : "none")
.append("title")
.text(d => {
const count = 0,
perc = 0,
tooltip = `Count: ${count}\n
Percentage: ${perc.toFixed(2)}%\n
Center: ${d.x.toFixed(2)}, ${d.y.toFixed(2)}\n
`.replace(/\s{2,}/g, "\n");
return tooltip;
});
}
marginedSpace.append("g")
.attr("class", "hexagon")
.attr("clip-path", "url(#clip)")
.selectAll("path")
.data(hexPoints)
.enter()
.append("path")
.attr("d", d => {
return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(drawRadius)}`;
})
.attr("fill", (d) => colour(d.length))
.attr("stroke", drawEdges ? "black" : "none")
.attr("stroke-width", drawEdges ? "0.5" : "none")
.append("title")
.text(d => {
const count = d.length,
perc = 100.0 * d.length / values.length,
CX = d.x,
CY = d.y,
xMin = Math.min(...d.map(d => d[0])),
xMax = Math.max(...d.map(d => d[0])),
yMin = Math.min(...d.map(d => d[1])),
yMax = Math.max(...d.map(d => d[1])),
tooltip = `Count: ${count}\n
Percentage: ${perc.toFixed(2)}%\n
Center: ${CX.toFixed(2)}, ${CY.toFixed(2)}\n
Min X: ${xMin.toFixed(2)}\n
Max X: ${xMax.toFixed(2)}\n
Min Y: ${yMin.toFixed(2)}\n
Max Y: ${yMax.toFixed(2)}
`.replace(/\s{2,}/g, "\n");
return tooltip;
});
marginedSpace.append("g")
.attr("class", "axis axis--y")
.call(d3.axisLeft(yAxis).tickSizeOuter(-width));
svg.append("text")
.attr("transform", "rotate(-90)")
.attr("y", -margin.left)
.attr("x", -(height / 2))
.attr("dy", "1em")
.style("text-anchor", "middle")
.text(yLabel);
marginedSpace.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(xAxis).tickSizeOuter(-height));
svg.append("text")
.attr("x", width / 2)
.attr("y", dimension)
.style("text-anchor", "middle")
.text(xLabel);
return svg._groups[0][0].outerHTML;
}
/**
* Hex Bin chart operation.
*
* @param {Object[]} - centres
* @param {number} - radius
* @returns {Object[]}
*/
getEmptyHexagons(centres, radius) {
const emptyCentres = [],
boundingRect = [d3.extent(centres, d => d.x), d3.extent(centres, d => d.y)],
hexagonCenterToEdge = Math.cos(2 * Math.PI / 12) * radius,
hexagonEdgeLength = Math.sin(2 * Math.PI / 12) * radius;
let indent = false;
for (let y = boundingRect[1][0]; y <= boundingRect[1][1] + radius; y += hexagonEdgeLength + radius) {
for (let x = boundingRect[0][0]; x <= boundingRect[0][1] + radius; x += 2 * hexagonCenterToEdge) {
let cx = x;
const cy = y;
if (indent && x >= boundingRect[0][1]) break;
if (indent) cx += hexagonCenterToEdge;
emptyCentres.push({x: cx, y: cy});
}
indent = !indent;
}
return emptyCentres;
}
}
export default HexDensityChart;

View File

@ -21,7 +21,7 @@ class JavaScriptParser extends Operation {
this.name = "JavaScript Parser";
this.module = "Code";
this.description = "Returns an Abstract Syntax Tree for valid JavaScript code.";
this.infoURL = "https://en.wikipedia.org/wiki/Abstract_syntax_tree";
this.infoURL = "https://wikipedia.org/wiki/Abstract_syntax_tree";
this.inputType = "string";
this.outputType = "string";
this.args = [

View File

@ -21,7 +21,7 @@ class PEMToHex extends Operation {
this.name = "PEM to Hex";
this.module = "PublicKey";
this.description = "Converts PEM (Privacy Enhanced Mail) format to a hexadecimal DER (Distinguished Encoding Rules) string.";
this.infoURL = "https://en.wikipedia.org/wiki/X.690#DER_encoding";
this.infoURL = "https://wikipedia.org/wiki/X.690#DER_encoding";
this.inputType = "string";
this.outputType = "string";
this.args = [];

View File

@ -0,0 +1,199 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import * as d3temp from "d3";
import * as nodomtemp from "nodom";
import { getScatterValues, getScatterValuesWithColour, RECORD_DELIMITER_OPTIONS, COLOURS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
import Operation from "../Operation";
import Utils from "../Utils";
const d3 = d3temp.default ? d3temp.default : d3temp;
const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
/**
* Scatter chart operation
*/
class ScatterChart extends Operation {
/**
* ScatterChart constructor
*/
constructor() {
super();
this.name = "Scatter chart";
this.module = "Charts";
this.description = "Plots two-variable data as single points on a graph.";
this.infoURL = "https://wikipedia.org/wiki/Scatter_plot";
this.inputType = "string";
this.outputType = "html";
this.args = [
{
name: "Record delimiter",
type: "option",
value: RECORD_DELIMITER_OPTIONS,
},
{
name: "Field delimiter",
type: "option",
value: FIELD_DELIMITER_OPTIONS,
},
{
name: "Use column headers as labels",
type: "boolean",
value: true,
},
{
name: "X label",
type: "string",
value: "",
},
{
name: "Y label",
type: "string",
value: "",
},
{
name: "Colour",
type: "string",
value: COLOURS.max,
},
{
name: "Point radius",
type: "number",
value: 10,
},
{
name: "Use colour from third column",
type: "boolean",
value: false,
}
];
}
/**
* Scatter chart operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {html}
*/
run(input, args) {
const recordDelimiter = Utils.charRep(args[0]),
fieldDelimiter = Utils.charRep(args[1]),
columnHeadingsAreIncluded = args[2],
fillColour = args[5],
radius = args[6],
colourInInput = args[7],
dimension = 500;
let xLabel = args[3],
yLabel = args[4];
const dataFunction = colourInInput ? getScatterValuesWithColour : getScatterValues;
const { headings, values } = dataFunction(
input,
recordDelimiter,
fieldDelimiter,
columnHeadingsAreIncluded
);
if (headings) {
xLabel = headings.x;
yLabel = headings.y;
}
const document = new nodom.Document();
let svg = document.createElement("svg");
svg = d3.select(svg)
.attr("width", "100%")
.attr("height", "100%")
.attr("viewBox", `0 0 ${dimension} ${dimension}`);
const margin = {
top: 10,
right: 0,
bottom: 40,
left: 30,
},
width = dimension - margin.left - margin.right,
height = dimension - margin.top - margin.bottom,
marginedSpace = svg.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
const xExtent = d3.extent(values, d => d[0]),
xDelta = xExtent[1] - xExtent[0],
yExtent = d3.extent(values, d => d[1]),
yDelta = yExtent[1] - yExtent[0],
xAxis = d3.scaleLinear()
.domain([xExtent[0] - (0.1 * xDelta), xExtent[1] + (0.1 * xDelta)])
.range([0, width]),
yAxis = d3.scaleLinear()
.domain([yExtent[0] - (0.1 * yDelta), yExtent[1] + (0.1 * yDelta)])
.range([height, 0]);
marginedSpace.append("clipPath")
.attr("id", "clip")
.append("rect")
.attr("width", width)
.attr("height", height);
marginedSpace.append("g")
.attr("class", "points")
.attr("clip-path", "url(#clip)")
.selectAll("circle")
.data(values)
.enter()
.append("circle")
.attr("cx", (d) => xAxis(d[0]))
.attr("cy", (d) => yAxis(d[1]))
.attr("r", d => radius)
.attr("fill", d => {
return colourInInput ? d[2] : fillColour;
})
.attr("stroke", "rgba(0, 0, 0, 0.5)")
.attr("stroke-width", "0.5")
.append("title")
.text(d => {
const x = d[0],
y = d[1],
tooltip = `X: ${x}\n
Y: ${y}\n
`.replace(/\s{2,}/g, "\n");
return tooltip;
});
marginedSpace.append("g")
.attr("class", "axis axis--y")
.call(d3.axisLeft(yAxis).tickSizeOuter(-width));
svg.append("text")
.attr("transform", "rotate(-90)")
.attr("y", -margin.left)
.attr("x", -(height / 2))
.attr("dy", "1em")
.style("text-anchor", "middle")
.text(yLabel);
marginedSpace.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(xAxis).tickSizeOuter(-height));
svg.append("text")
.attr("x", width / 2)
.attr("y", dimension)
.style("text-anchor", "middle")
.text(xLabel);
return svg._groups[0][0].outerHTML;
}
}
export default ScatterChart;

View File

@ -0,0 +1,227 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import * as d3temp from "d3";
import * as nodomtemp from "nodom";
import { getSeriesValues, RECORD_DELIMITER_OPTIONS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
import Operation from "../Operation";
import Utils from "../Utils";
const d3 = d3temp.default ? d3temp.default : d3temp;
const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
/**
* Series chart operation
*/
class SeriesChart extends Operation {
/**
* SeriesChart constructor
*/
constructor() {
super();
this.name = "Series chart";
this.module = "Charts";
this.description = "A time series graph is a line graph of repeated measurements taken over regular time intervals.";
this.inputType = "string";
this.outputType = "html";
this.args = [
{
name: "Record delimiter",
type: "option",
value: RECORD_DELIMITER_OPTIONS,
},
{
name: "Field delimiter",
type: "option",
value: FIELD_DELIMITER_OPTIONS,
},
{
name: "X label",
type: "string",
value: "",
},
{
name: "Point radius",
type: "number",
value: 1,
},
{
name: "Series colours",
type: "string",
value: "mediumseagreen, dodgerblue, tomato",
},
];
}
/**
* Series chart operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {html}
*/
run(input, args) {
const recordDelimiter = Utils.charRep(args[0]),
fieldDelimiter = Utils.charRep(args[1]),
xLabel = args[2],
pipRadius = args[3],
seriesColours = args[4].split(","),
svgWidth = 500,
interSeriesPadding = 20,
xAxisHeight = 50,
seriesLabelWidth = 50,
seriesHeight = 100,
seriesWidth = svgWidth - seriesLabelWidth - interSeriesPadding;
const { xValues, series } = getSeriesValues(input, recordDelimiter, fieldDelimiter),
allSeriesHeight = Object.keys(series).length * (interSeriesPadding + seriesHeight),
svgHeight = allSeriesHeight + xAxisHeight + interSeriesPadding;
const document = new nodom.Document();
let svg = document.createElement("svg");
svg = d3.select(svg)
.attr("width", "100%")
.attr("height", "100%")
.attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
const xAxis = d3.scalePoint()
.domain(xValues)
.range([0, seriesWidth]);
svg.append("g")
.attr("class", "axis axis--x")
.attr("transform", `translate(${seriesLabelWidth}, ${xAxisHeight})`)
.call(
d3.axisTop(xAxis).tickValues(xValues.filter((x, i) => {
return [0, Math.round(xValues.length / 2), xValues.length -1].indexOf(i) >= 0;
}))
);
svg.append("text")
.attr("x", svgWidth / 2)
.attr("y", xAxisHeight / 2)
.style("text-anchor", "middle")
.text(xLabel);
const tooltipText = {},
tooltipAreaWidth = seriesWidth / xValues.length;
xValues.forEach(x => {
const tooltip = [];
series.forEach(serie => {
const y = serie.data[x];
if (typeof y === "undefined") return;
tooltip.push(`${serie.name}: ${y}`);
});
tooltipText[x] = tooltip.join("\n");
});
const chartArea = svg.append("g")
.attr("transform", `translate(${seriesLabelWidth}, ${xAxisHeight})`);
chartArea
.append("g")
.selectAll("rect")
.data(xValues)
.enter()
.append("rect")
.attr("x", x => {
return xAxis(x) - (tooltipAreaWidth / 2);
})
.attr("y", 0)
.attr("width", tooltipAreaWidth)
.attr("height", allSeriesHeight)
.attr("stroke", "none")
.attr("fill", "transparent")
.append("title")
.text(x => {
return `${x}\n
--\n
${tooltipText[x]}\n
`.replace(/\s{2,}/g, "\n");
});
const yAxesArea = svg.append("g")
.attr("transform", `translate(0, ${xAxisHeight})`);
series.forEach((serie, seriesIndex) => {
const yExtent = d3.extent(Object.values(serie.data)),
yAxis = d3.scaleLinear()
.domain(yExtent)
.range([seriesHeight, 0]);
const seriesGroup = chartArea
.append("g")
.attr("transform", `translate(0, ${seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`);
let path = "";
xValues.forEach((x, xIndex) => {
let nextX = xValues[xIndex + 1],
y = serie.data[x],
nextY= serie.data[nextX];
if (typeof y === "undefined" || typeof nextY === "undefined") return;
x = xAxis(x); nextX = xAxis(nextX);
y = yAxis(y); nextY = yAxis(nextY);
path += `M ${x} ${y} L ${nextX} ${nextY} z `;
});
seriesGroup
.append("path")
.attr("d", path)
.attr("fill", "none")
.attr("stroke", seriesColours[seriesIndex % seriesColours.length])
.attr("stroke-width", "1");
xValues.forEach(x => {
const y = serie.data[x];
if (typeof y === "undefined") return;
seriesGroup
.append("circle")
.attr("cx", xAxis(x))
.attr("cy", yAxis(y))
.attr("r", pipRadius)
.attr("fill", seriesColours[seriesIndex % seriesColours.length])
.append("title")
.text(d => {
return `${x}\n
--\n
${tooltipText[x]}\n
`.replace(/\s{2,}/g, "\n");
});
});
yAxesArea
.append("g")
.attr("transform", `translate(${seriesLabelWidth - interSeriesPadding}, ${seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`)
.attr("class", "axis axis--y")
.call(d3.axisLeft(yAxis).ticks(5));
yAxesArea
.append("g")
.attr("transform", `translate(0, ${seriesHeight / 2 + seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`)
.append("text")
.style("text-anchor", "middle")
.attr("transform", "rotate(-90)")
.text(serie.name);
});
return svg._groups[0][0].outerHTML;
}
}
export default SeriesChart;

View File

@ -27,10 +27,20 @@ class File {
* @param {Object} stats (optional) - file stats e.g. lastModified
*/
constructor(data, name="", stats={}) {
if (!Array.isArray(data)) {
data = [data];
}
const buffers = data.map((d) => {
if (d instanceof File) {
return Buffer.from(d.data);
}
if (d instanceof ArrayBuffer) {
return Buffer.from(d);
}
return Buffer.from(d);
});
const totalLength = buffers.reduce((p, c) => p + c.length, 0);

1607
src/node/index.mjs Normal file

File diff suppressed because it is too large Load Diff

View File

@ -4,6 +4,10 @@
* @license Apache-2.0
*/
import clippy from "clippyjs";
import "./static/clippy_assets/agents/Clippy/agent.js";
import clippyMap from "./static/clippy_assets/agents/Clippy/map.png";
/**
* Waiter to handle seasonal events and easter eggs.
*/
@ -18,6 +22,8 @@ class SeasonalWaiter {
constructor(app, manager) {
this.app = app;
this.manager = manager;
this.clippyAgent = null;
}
@ -28,6 +34,14 @@ class SeasonalWaiter {
// Konami code
this.kkeys = [];
window.addEventListener("keydown", this.konamiCodeListener.bind(this));
// Clippy
const now = new Date();
if (now.getMonth() === 3 && now.getDate() === 1) {
this.addClippyOption();
this.manager.addDynamicListener(".option-item #clippy", "change", this.setupClippy, this);
this.setupClippy();
}
}
@ -51,6 +65,285 @@ class SeasonalWaiter {
}
}
/**
* Creates an option in the Options menu for turning Clippy on or off
*/
addClippyOption() {
const optionsBody = document.getElementById("options-body"),
optionItem = document.createElement("span");
optionItem.className = "bmd-form-group is-filled";
optionItem.innerHTML = `<div class="checkbox option-item">
<label for="clippy">
<input type="checkbox" option="clippy" id="clippy" checked="">
Use the Clippy helper
</label>
</div>`;
optionsBody.appendChild(optionItem);
if (!this.app.options.hasOwnProperty("clippy")) {
this.app.options.clippy = true;
}
this.manager.options.load();
}
/**
* Sets up Clippy for April Fools Day
*/
setupClippy() {
// Destroy any previous agents
if (this.clippyAgent) {
this.clippyAgent.closeBalloonImmediately();
this.clippyAgent.hide();
}
if (!this.app.options.clippy) {
if (this.clippyTimeouts) this.clippyTimeouts.forEach(t => clearTimeout(t));
return;
}
// Set base path to # to prevent external network requests
const clippyAssets = "#";
// Shim the library to prevent external network requests
shimClippy(clippy);
const self = this;
clippy.load("Clippy", (agent) => {
shimClippyAgent(agent);
self.clippyAgent = agent;
agent.show();
agent.speak("Hello, I'm Clippy, your personal cyber assistant!");
}, undefined, clippyAssets);
// Watch for the Auto Magic button appearing
const magic = document.getElementById("magic");
const observer = new MutationObserver((mutationsList, observer) => {
// Read in message and recipe
let msg, recipe;
for (const mutation of mutationsList) {
if (mutation.attributeName === "data-original-title") {
msg = magic.getAttribute("data-original-title");
}
if (mutation.attributeName === "data-recipe") {
recipe = magic.getAttribute("data-recipe");
}
}
// Close balloon if it is currently showing a magic hint
const balloon = self.clippyAgent._balloon._balloon;
if (balloon.is(":visible") && balloon.text().indexOf("That looks like encoded data") >= 0) {
self.clippyAgent._balloon.hide(true);
this.clippyAgent._balloon._hidden = true;
}
// If a recipe was found, get Clippy to tell the user
if (recipe) {
recipe = this.manager.controls.generateStateUrl(true, true, JSON.parse(recipe));
msg = `That looks like encoded data!<br><br>${msg}<br><br>Click <a class="clippyMagicRecipe" href="${recipe}">here</a> to load this recipe.`;
// Stop current balloon activity immediately and trigger speak again
this.clippyAgent.closeBalloonImmediately();
self.clippyAgent.speak(msg, true);
// self.clippyAgent._queue.next();
}
});
observer.observe(document.getElementById("magic"), {attributes: true});
// Play animations for various things
this.manager.addListeners("#search", "click", () => {
this.clippyAgent.play("Searching");
}, this);
this.manager.addListeners("#save,#save-to-file", "click", () => {
this.clippyAgent.play("Save");
}, this);
this.manager.addListeners("#clr-recipe,#clr-io", "click", () => {
this.clippyAgent.play("EmptyTrash");
}, this);
this.manager.addListeners("#bake", "click", e => {
if (e.target.closest("button").textContent.toLowerCase().indexOf("bake") >= 0) {
this.clippyAgent.play("Thinking");
} else {
this.clippyAgent.play("EmptyTrash");
}
this.clippyAgent._queue.clear();
}, this);
this.manager.addListeners("#input-text", "keydown", () => {
this.clippyAgent.play("Writing");
this.clippyAgent._queue.clear();
}, this);
this.manager.addDynamicListener("a.clippyMagicRecipe", "click", (e) => {
this.clippyAgent.play("Congratulate");
}, this);
this.clippyTimeouts = [];
// Show challenge after timeout
this.clippyTimeouts.push(setTimeout(() => {
const hex = "1f 8b 08 00 ae a1 9b 5c 00 ff 05 40 a1 12 00 10 0c fd 26 61 5b 76 aa 9d 26 a8 02 02 37 84 f7 fb bb c5 a4 5f 22 c6 09 e5 6e c5 4c 2d 3f e9 30 a6 ea 41 a2 f2 ac 1c 00 00 00";
self.clippyAgent.speak(`How about a fun challenge?<br><br>Try decoding this (click to load):<br><a href="#recipe=[]&input=${encodeURIComponent(btoa(hex))}">${hex}</a>`, true);
self.clippyAgent.play("GetAttention");
}, 1 * 60 * 1000));
this.clippyTimeouts.push(setTimeout(() => {
self.clippyAgent.speak("<i>Did you know?</i><br><br>You can load files into CyberChef up to around 500MB using drag and drop or the load file button.", 15000);
self.clippyAgent.play("Wave");
}, 2 * 60 * 1000));
this.clippyTimeouts.push(setTimeout(() => {
self.clippyAgent.speak("<i>Did you know?</i><br><br>You can use the 'Fork' operation to split up your input and run the recipe over each branch separately.<br><br><a class='clippyMagicRecipe' href=\"#recipe=Fork('%5C%5Cn','%5C%5Cn',false)From_UNIX_Timestamp('Seconds%20(s)')&amp;input=OTc4MzQ2ODAwCjEwMTI2NTEyMDAKMTA0NjY5NjQwMAoxMDgxMDg3MjAwCjExMTUzMDUyMDAKMTE0OTYwOTYwMA\">Here's an example</a>.", 15000);
self.clippyAgent.play("Print");
}, 3 * 60 * 1000));
this.clippyTimeouts.push(setTimeout(() => {
self.clippyAgent.speak("<i>Did you know?</i><br><br>The 'Magic' operation uses a number of methods to detect encoded data and the operations which can be used to make sense of it. A technical description of these methods can be found <a href=\"https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic\">here</a>.", 15000);
self.clippyAgent.play("Alert");
}, 4 * 60 * 1000));
this.clippyTimeouts.push(setTimeout(() => {
self.clippyAgent.speak("<i>Did you know?</i><br><br>You can use parts of the input as arguments to operations.<br><br><a class='clippyMagicRecipe' href=\"#recipe=Register('key%3D(%5B%5C%5Cda-f%5D*)',true,false)Find_/_Replace(%7B'option':'Regex','string':'.*data%3D(.*)'%7D,'$1',true,false,true)RC4(%7B'option':'Hex','string':'$R0'%7D,'Hex','Latin1')&amp;input=aHR0cDovL21hbHdhcmV6LmJpei9iZWFjb24ucGhwP2tleT0wZTkzMmE1YyZkYXRhPThkYjdkNWViZTM4NjYzYTU0ZWNiYjMzNGUzZGIxMQ\">Click here for an example</a>.", 15000);
self.clippyAgent.play("CheckingSomething");
}, 5 * 60 * 1000));
}
}
/**
* Shims various ClippyJS functions to modify behaviour.
*
* @param {Clippy} clippy - The Clippy library
*/
function shimClippy(clippy) {
// Shim _loadSounds so that it doesn't actually try to load any sounds
clippy.load._loadSounds = function _loadSounds (name, path) {
let dfd = clippy.load._sounds[name];
if (dfd) return dfd;
// set dfd if not defined
dfd = clippy.load._sounds[name] = $.Deferred();
// Resolve immediately without loading
dfd.resolve({});
return dfd.promise();
};
// Shim _loadMap so that it uses the local copy
clippy.load._loadMap = function _loadMap (path) {
let dfd = clippy.load._maps[path];
if (dfd) return dfd;
// set dfd if not defined
dfd = clippy.load._maps[path] = $.Deferred();
const src = clippyMap;
const img = new Image();
img.onload = dfd.resolve;
img.onerror = dfd.reject;
// start loading the map;
img.setAttribute("src", src);
return dfd.promise();
};
// Make sure we don't request the remote map
clippy.Animator.prototype._setupElement = function _setupElement (el) {
const frameSize = this._data.framesize;
el.css("display", "none");
el.css({ width: frameSize[0], height: frameSize[1] });
el.css("background", "url('" + clippyMap + "') no-repeat");
return el;
};
}
/**
* Shims various ClippyJS Agent functions to modify behaviour.
*
* @param {Agent} agent - The Clippy Agent
*/
function shimClippyAgent(agent) {
// Turn off all sounds
agent._animator._playSound = () => {};
// Improve speak function to support HTML markup
const self = agent._balloon;
agent._balloon.speak = (complete, text, hold) => {
self._hidden = false;
self.show();
const c = self._content;
// set height to auto
c.height("auto");
c.width("auto");
// add the text
c.html(text);
// set height
c.height(c.height());
c.width(c.width());
c.text("");
self.reposition();
self._complete = complete;
self._sayWords(text, hold, complete);
if (hold) agent._queue.next();
};
// Improve the _sayWords function to allow HTML and support timeouts
agent._balloon.WORD_SPEAK_TIME = 60;
agent._balloon._sayWords = (text, hold, complete) => {
self._active = true;
self._hold = hold;
const words = text.split(/[^\S-]/);
const time = self.WORD_SPEAK_TIME;
const el = self._content;
let idx = 1;
clearTimeout(self.holdTimeout);
self._addWord = $.proxy(function () {
if (!self._active) return;
if (idx > words.length) {
delete self._addWord;
self._active = false;
if (!self._hold) {
complete();
self.hide();
} else if (typeof hold === "number") {
self.holdTimeout = setTimeout(() => {
self._hold = false;
complete();
self.hide();
}, hold);
}
} else {
el.html(words.slice(0, idx).join(" "));
idx++;
self._loop = window.setTimeout($.proxy(self._addWord, self), time);
}
}, self);
self._addWord();
};
// Add break-word to balloon CSS
agent._balloon._balloon.css("word-break", "break-word");
// Close the balloon on click (unless it was a link)
agent._balloon._balloon.click(e => {
if (e.target.nodeName !== "A") {
agent._balloon.hide(true);
agent._balloon._hidden = true;
}
});
// Add function to immediately close the balloon even if it is currently doing something
agent.closeBalloonImmediately = () => {
agent._queue.clear();
agent._balloon.hide(true);
agent._balloon._hidden = true;
agent._queue.next();
};
}
export default SeasonalWaiter;

View File

@ -591,7 +591,7 @@
What sort of things can I do with CyberChef?
</a>
<div class="collapse" id="faq-examples">
<p>There are around 200 operations in CyberChef allowing you to carry out simple and complex tasks easily. Here are some examples:</p>
<p>There are around 300 operations in CyberChef allowing you to carry out simple and complex tasks easily. Here are some examples:</p>
<ul>
<li><a href="#recipe=From_Base64('A-Za-z0-9%2B/%3D',true)&input=VTI4Z2JHOXVaeUJoYm1RZ2RHaGhibXR6SUdadmNpQmhiR3dnZEdobElHWnBjMmd1">Decode a Base64-encoded string</a></li>
<li><a href="#recipe=Translate_DateTime_Format('Standard%20date%20and%20time','DD/MM/YYYY%20HH:mm:ss','UTC','dddd%20Do%20MMMM%20YYYY%20HH:mm:ss%20Z%20z','Australia/Queensland')&input=MTUvMDYvMjAxNSAyMDo0NTowMA">Convert a date and time to a different time zone</a></li>

View File

@ -8,7 +8,6 @@
import "./stylesheets/index.js";
// Libs
import "babel-polyfill";
import "arrive";
import "snackbarjs";
import "bootstrap-material-design";

File diff suppressed because one or more lines are too long

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

View File

@ -0,0 +1,62 @@
.clippy, .clippy-balloon {
position: fixed;
z-index: 1000;
cursor: pointer;
}
.clippy-balloon {
background: #FFC;
color: black;
padding: 8px;
border: 1px solid black;
border-radius: 5px;
}
.clippy-content {
max-width: 200px;
min-width: 120px;
font-family: "Microsoft Sans", sans-serif;
font-size: 10pt;
}
.clippy-tip {
width: 10px;
height: 16px;
background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAgCAMAAAAlvKiEAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAAlQTFRF///MAAAA////52QwgAAAAAN0Uk5T//8A18oNQQAAAGxJREFUeNqs0kEOwCAIRFHn3//QTUU6xMyyxii+jQosrTPkyPEM6IN3FtzIRk1U4dFeKWQiH6pRRowMVKEmvronEynkwj0uZJgR22+YLopPSo9P34wJSamLSU7lSIWLJU7NkNomNlhqxUeAAQC+TQLZyEuJBwAAAABJRU5ErkJggg==) no-repeat;
position: absolute;
}
.clippy-top-left .clippy-tip {
top: 100%;
margin-top: 0px;
left: 100%;
margin-left: -50px;
}
.clippy-top-right .clippy-tip {
top: 100%;
margin-top: 0px;
left: 0;
margin-left: 50px;
background-position: -10px 0;
}
.clippy-bottom-right .clippy-tip {
top: 0;
margin-top: -16px;
left: 0;
margin-left: 50px;
background-position: -10px -16px;
}
.clippy-bottom-left .clippy-tip {
top: 0;
margin-top: -16px;
left: 100%;
margin-left: -50px;
background-position: 0px -16px;
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 229 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 238 B

View File

@ -8,6 +8,7 @@
/* Libraries */
import "highlight.js/styles/vs.css";
import "../static/clippy_assets/clippy.css";
/* Frameworks */
import "./vendors/bootstrap.scss";

View File

@ -48,7 +48,6 @@ class TestRegister {
* Runs all the tests in the register.
*/
runTests () {
console.log("run tests");
return Promise.all(
this.tests.map(function(test, i) {
const chef = new Chef();

View File

@ -2,7 +2,8 @@ import assert from "assert";
import it from "../assertionHandler";
import TestRegister from "../../lib/TestRegister";
import File from "../../../src/node/File";
import {zip} from "../../../src/node/index";
import {zip, Dish} from "../../../src/node/index";
import { DishBigNumber } from "../../../src/core/dishTranslationTypes/index.mjs";
TestRegister.addApiTests([
it("File: should exist", () => {
@ -31,4 +32,52 @@ TestRegister.addApiTests([
const file = new File([uint8Array], "sample.txt");
assert.strictEqual(file.type, "application/unknown");
}),
it("File: should be able to make a dish from it", () => {
const uint8Array = new Uint8Array(Buffer.from("hello"));
const file = new File([uint8Array], "sample.txt");
try {
const dish = new Dish(file, 7);
assert.ok(dish.valid());
} catch (e) {
assert.fail(e.message);
}
}),
it("File: should allow dish to translate to ArrayBuffer", () => {
const uint8Array = new Uint8Array(Buffer.from("hello"));
const file = new File([uint8Array], "sample.txt");
try {
const dish = new Dish(file, 7);
assert.ok(dish.value);
dish.get(4);
assert.strictEqual(dish.type, 4);
assert.ok(dish.valid());
} catch (e) {
assert.fail(e.message);
}
}),
it("File: should allow dish to translate from ArrayBuffer to File", () => {
const uint8Array = new Uint8Array(Buffer.from("hello"));
const file = new File([uint8Array], "sample.txt");
try {
const dish = new Dish(file, 7);
assert.ok(dish.value);
// translate to ArrayBuffer
dish.get(4);
assert.ok(dish.valid());
// translate back to File
dish.get(7);
assert.ok(dish.valid());
} catch (e) {
assert.fail(e.message);
}
})
]);

View File

@ -400,8 +400,8 @@ TestRegister.addApiTests([
}),
it("Operation arguments: should be accessible from operation object if op has array arg", () => {
assert.ok(chef.toCharcode.argOptions);
assert.deepEqual(chef.unzip.argOptions, {
assert.ok(chef.toCharcode.args);
assert.deepEqual(chef.unzip.args, {
password: {
type: "binaryString",
value: "",
@ -414,20 +414,20 @@ TestRegister.addApiTests([
}),
it("Operation arguments: should have key for each argument in operation", () => {
assert.ok(chef.convertDistance.argOptions.inputUnits);
assert.ok(chef.convertDistance.argOptions.outputUnits);
assert.ok(chef.convertDistance.args.inputUnits);
assert.ok(chef.convertDistance.args.outputUnits);
assert.strictEqual(chef.bitShiftRight.argOptions.amount.type, "number");
assert.strictEqual(chef.bitShiftRight.argOptions.amount.value, 1);
assert.strictEqual(chef.bitShiftRight.argOptions.type.type, "option");
assert.ok(Array.isArray(chef.bitShiftRight.argOptions.type.options));
assert.strictEqual(chef.bitShiftRight.args.amount.type, "number");
assert.strictEqual(chef.bitShiftRight.args.amount.value, 1);
assert.strictEqual(chef.bitShiftRight.args.type.type, "option");
assert.ok(Array.isArray(chef.bitShiftRight.args.type.options));
}),
it("Operation arguments: should list all options excluding subheadings", () => {
// First element (subheading) removed
assert.equal(chef.convertDistance.argOptions.inputUnits.options[0], "Nanometres (nm)");
assert.equal(chef.defangURL.argOptions.process.options[1], "Only full URLs");
assert.equal(chef.convertDistance.args.inputUnits.options[0], "Nanometres (nm)");
assert.equal(chef.defangURL.args.process.options[1], "Only full URLs");
})
]);

View File

@ -10,7 +10,6 @@
* @copyright Crown Copyright 2017
* @license Apache-2.0
*/
import "babel-polyfill";
import {
setLongTestFailure,
@ -38,6 +37,7 @@ import "./tests/BitwiseOp";
import "./tests/ByteRepr";
import "./tests/CartesianProduct";
import "./tests/CharEnc";
import "./tests/Charts";
import "./tests/Checksum";
import "./tests/Ciphers";
import "./tests/Code";
@ -92,6 +92,8 @@ import "./tests/Enigma";
import "./tests/Bombe";
import "./tests/MultipleBombe";
import "./tests/Typex";
import "./tests/BLAKE2b";
import "./tests/BLAKE2s";
// Cannot test operations that use the File type yet
//import "./tests/SplitColourChannels";

View File

@ -0,0 +1,56 @@
/**
* BitwiseOp tests
*
* @author h345983745
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import TestRegister from "../TestRegister";
TestRegister.addTests([
{
name: "BLAKE2b: 512 - Hello World",
input: "Hello World",
expectedOutput: "4386a08a265111c9896f56456e2cb61a64239115c4784cf438e36cc851221972da3fb0115f73cd02486254001f878ab1fd126aac69844ef1c1ca152379d0a9bd",
recipeConfig: [
{ "op": "BLAKE2b",
"args": ["512", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2b: 384 - Hello World",
input: "Hello World",
expectedOutput: "4d388e82ca8f866e606b6f6f0be910abd62ad6e98c0adfc27cf35acf948986d5c5b9c18b6f47261e1e679eb98edf8e2d",
recipeConfig: [
{ "op": "BLAKE2b",
"args": ["384", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2b: 256 - Hello World",
input: "Hello World",
expectedOutput: "1dc01772ee0171f5f614c673e3c7fa1107a8cf727bdf5a6dadb379e93c0d1d00",
recipeConfig: [
{ "op": "BLAKE2b",
"args": ["256", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2b: 160 - Hello World",
input: "Hello World",
expectedOutput: "6a8489e6fd6e51fae12ab271ec7fc8134dd5d737",
recipeConfig: [
{ "op": "BLAKE2b",
"args": ["160", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2b: Key Test",
input: "message data",
expectedOutput: "3d363ff7401e02026f4a4687d4863ced",
recipeConfig: [
{ "op": "BLAKE2b",
"args": ["128", "Hex", {string: "pseudorandom key", option: "UTF8"}] }
]
}
]);

View File

@ -0,0 +1,47 @@
/**
* BitwiseOp tests
*
* @author h345983745
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import TestRegister from "../TestRegister";
TestRegister.addTests([
{
name: "BLAKE2s: 256 - Hello World",
input: "Hello World",
expectedOutput: "7706af019148849e516f95ba630307a2018bb7bf03803eca5ed7ed2c3c013513",
recipeConfig: [
{ "op": "BLAKE2s",
"args": ["256", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2s: 160 - Hello World",
input: "Hello World",
expectedOutput: "0e4fcfc2ee0097ac1d72d70b595a39e09a3c7c7e",
recipeConfig: [
{ "op": "BLAKE2s",
"args": ["160", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2s: 128 - Hello World",
input: "Hello World",
expectedOutput: "9964ee6f36126626bf864363edfa96f6",
recipeConfig: [
{ "op": "BLAKE2s",
"args": ["128", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2s: Key Test",
input: "Hello World",
expectedOutput: "9964ee6f36126626bf864363edfa96f6",
recipeConfig: [
{ "op": "BLAKE2s",
"args": ["128", "Hex", {string: "", option: "UTF8"}] }
]
}
]);

View File

@ -0,0 +1,55 @@
/**
* Chart tests.
*
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import TestRegister from "../TestRegister";
TestRegister.addTests([
{
name: "Scatter chart",
input: "100 100\n200 200\n300 300\n400 400\n500 500",
expectedMatch: /^<svg width/,
recipeConfig: [
{
"op": "Scatter chart",
"args": ["Line feed", "Space", false, "time", "stress", "black", 5, false]
}
],
},
{
name: "Hex density chart",
input: "100 100\n200 200\n300 300\n400 400\n500 500",
expectedMatch: /^<svg width/,
recipeConfig: [
{
"op": "Hex Density chart",
"args": ["Line feed", "Space", 25, 15, true, "", "", true, "white", "black", true]
}
],
},
{
name: "Series chart",
input: "100 100 100\n200 200 200\n300 300 300\n400 400 400\n500 500 500",
expectedMatch: /^<svg width/,
recipeConfig: [
{
"op": "Series chart",
"args": ["Line feed", "Space", "", 1, "mediumseagreen, dodgerblue, tomato"]
}
],
},
{
name: "Heatmap chart",
input: "100 100\n200 200\n300 300\n400 400\n500 500",
expectedMatch: /^<svg width/,
recipeConfig: [
{
"op": "Heatmap chart",
"args": ["Line feed", "Space", 25, 25, true, "", "", false, "white", "black"]
}
],
},
]);

View File

@ -133,11 +133,15 @@ module.exports = {
warningsFilter: [
/source-map/,
/dependency is an expression/,
/export 'default'/
/export 'default'/,
/Can't resolve 'sodium'/
],
},
node: {
fs: "empty"
fs: "empty",
"child_process": "empty",
net: "empty",
tls: "empty"
},
performance: {
hints: false