Mirror
/
CyberChef
mirror of https://github.com/gchq/CyberChef.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

This commit is contained in:
j433866 2019-05-09 09:08:01 +01:00
parent 328c0ade22 01f0625d6a
commit f9354c8cd1
48 changed files with 5443 additions and 3286 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintignore
View File

@ -1 +1,2 @@
src/core/vendor/** src/core/vendor/**
src/web/static/clippy_assets/**

3
.travis.yml
View File

@ -30,8 +30,9 @@ deploy:
skip_cleanup: true skip_cleanup: true
api_key: api_key:
secure: "HV1WSKv4l/0Y2bKKs1iBJocBcmLj08PCRUeEM/jTwA4jqJ8EiLHWiXtER/D5sEg2iibRVKd2OQjfrmS6bo4AiwdeVgAKmv0FtS2Jw+391N8Nd5AkEANHa5Om/IpHLTL2YRAjpJTsDpY72bMUTJIwjQA3TFJkgrpOw6KYfohOcgbxLpZ4XuNJRU3VL4Hsxdv5V9aOVmfFOmMOVPQlakXy7NgtW5POp1f2WJwgcZxylkR1CjwaqMyXmSoVl46pyH3tr5+dptsQoKSGdi6sIHGA60oDotFPcm+0ifa47wZw+vapuuDi4tdNxhrHGaDMG8xiE0WFDHwQUDlk2/+W7j9SEX0H3Em7us371JXRp56EDwEcDa34VpVkC6i8HGcHK55hnxVbMZXGf3qhOFD8wY7qMbjMRvIpucrMHBi86OfkDfv0vDj2LyvIl5APj/AX50BrE0tfH1MZbH26Jkx4NdlkcxQ14GumarmUqfmVvbX/fsoA6oUuAAE9ZgRRi3KHO4wci6KUcRfdm+XOeUkaBFsL86G3EEYIvrtBTuaypdz+Cx7nd1iPZyWMx5Y1gXnVzha4nBdV4+7l9JIsFggD8QVpw2uHXQiS1KXFjOeqA3DBD8tjMB7q26Fl2fD3jkOo4BTbQ2NrRIZUu/iL+fOmMPsyMt2qulB0yaSBCfkbEq8xrUA=" secure: "HV1WSKv4l/0Y2bKKs1iBJocBcmLj08PCRUeEM/jTwA4jqJ8EiLHWiXtER/D5sEg2iibRVKd2OQjfrmS6bo4AiwdeVgAKmv0FtS2Jw+391N8Nd5AkEANHa5Om/IpHLTL2YRAjpJTsDpY72bMUTJIwjQA3TFJkgrpOw6KYfohOcgbxLpZ4XuNJRU3VL4Hsxdv5V9aOVmfFOmMOVPQlakXy7NgtW5POp1f2WJwgcZxylkR1CjwaqMyXmSoVl46pyH3tr5+dptsQoKSGdi6sIHGA60oDotFPcm+0ifa47wZw+vapuuDi4tdNxhrHGaDMG8xiE0WFDHwQUDlk2/+W7j9SEX0H3Em7us371JXRp56EDwEcDa34VpVkC6i8HGcHK55hnxVbMZXGf3qhOFD8wY7qMbjMRvIpucrMHBi86OfkDfv0vDj2LyvIl5APj/AX50BrE0tfH1MZbH26Jkx4NdlkcxQ14GumarmUqfmVvbX/fsoA6oUuAAE9ZgRRi3KHO4wci6KUcRfdm+XOeUkaBFsL86G3EEYIvrtBTuaypdz+Cx7nd1iPZyWMx5Y1gXnVzha4nBdV4+7l9JIsFggD8QVpw2uHXQiS1KXFjOeqA3DBD8tjMB7q26Fl2fD3jkOo4BTbQ2NrRIZUu/iL+fOmMPsyMt2qulB0yaSBCfkbEq8xrUA="
file_glob: true
file: file:
- build/prod/cyberchef.htm - build/prod/*.zip
- build/node/CyberChef.js - build/node/CyberChef.js
on: on:
repo: gchq/CyberChef repo: gchq/CyberChef

21
CHANGELOG.md
View File

@ -2,6 +2,18 @@
All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
### [8.31.0] - 2019-04-12
- The downloadable version of CyberChef is now a .zip file containing separate modules rather than a single .htm file. It is still completely standalone and will not make any external network requests. This change reduces the complexity of the build process significantly. [@n1474335]
### [8.30.0] - 2019-04-12
- 'Decode Protobuf' operation added [@n1474335] | [#533]
### [8.29.0] - 2019-03-31
- 'BLAKE2s' and 'BLAKE2b' hashing operations added [@h345983745] | [#525]
### [8.28.0] - 2019-03-31
- 'Heatmap Chart', 'Hex Density Chart', 'Scatter Chart' and 'Series Chart' operation added [@artemisbot] [@tlwr] | [#496] [#143]
### [8.27.0] - 2019-03-14 ### [8.27.0] - 2019-03-14
- 'Enigma', 'Typex', 'Bombe' and 'Multiple Bombe' operations added [@s2224834] | [#516] - 'Enigma', 'Typex', 'Bombe' and 'Multiple Bombe' operations added [@s2224834] | [#516]
- See [this wiki article](https://github.com/gchq/CyberChef/wiki/Enigma,-the-Bombe,-and-Typex) for a full explanation of these operations. - See [this wiki article](https://github.com/gchq/CyberChef/wiki/Enigma,-the-Bombe,-and-Typex) for a full explanation of these operations.
@ -118,6 +130,10 @@ All major and minor version changes will be documented in this file. Details of
[8.31.0]: https://github.com/gchq/CyberChef/releases/tag/v8.31.0
[8.30.0]: https://github.com/gchq/CyberChef/releases/tag/v8.30.0
[8.29.0]: https://github.com/gchq/CyberChef/releases/tag/v8.29.0
[8.28.0]: https://github.com/gchq/CyberChef/releases/tag/v8.28.0
[8.27.0]: https://github.com/gchq/CyberChef/releases/tag/v8.27.0 [8.27.0]: https://github.com/gchq/CyberChef/releases/tag/v8.27.0
[8.26.0]: https://github.com/gchq/CyberChef/releases/tag/v8.26.0 [8.26.0]: https://github.com/gchq/CyberChef/releases/tag/v8.26.0
[8.25.0]: https://github.com/gchq/CyberChef/releases/tag/v8.25.0 [8.25.0]: https://github.com/gchq/CyberChef/releases/tag/v8.25.0
@ -159,6 +175,7 @@ All major and minor version changes will be documented in this file. Details of
[@h345983745]: https://github.com/h345983745 [@h345983745]: https://github.com/h345983745
[@s2224834]: https://github.com/s2224834 [@s2224834]: https://github.com/s2224834
[@artemisbot]: https://github.com/artemisbot [@artemisbot]: https://github.com/artemisbot
[@tlwr]: https://github.com/tlwr
[@picapi]: https://github.com/picapi [@picapi]: https://github.com/picapi
[@Dachande663]: https://github.com/Dachande663 [@Dachande663]: https://github.com/Dachande663
[@JustAnotherMark]: https://github.com/JustAnotherMark [@JustAnotherMark]: https://github.com/JustAnotherMark
@ -175,6 +192,7 @@ All major and minor version changes will be documented in this file. Details of
[#95]: https://github.com/gchq/CyberChef/pull/299 [#95]: https://github.com/gchq/CyberChef/pull/299
[#173]: https://github.com/gchq/CyberChef/pull/173 [#173]: https://github.com/gchq/CyberChef/pull/173
[#143]: https://github.com/gchq/CyberChef/pull/143
[#224]: https://github.com/gchq/CyberChef/pull/224 [#224]: https://github.com/gchq/CyberChef/pull/224
[#239]: https://github.com/gchq/CyberChef/pull/239 [#239]: https://github.com/gchq/CyberChef/pull/239
[#248]: https://github.com/gchq/CyberChef/pull/248 [#248]: https://github.com/gchq/CyberChef/pull/248
@ -209,5 +227,8 @@ All major and minor version changes will be documented in this file. Details of
[#468]: https://github.com/gchq/CyberChef/pull/468 [#468]: https://github.com/gchq/CyberChef/pull/468
[#476]: https://github.com/gchq/CyberChef/pull/476 [#476]: https://github.com/gchq/CyberChef/pull/476
[#489]: https://github.com/gchq/CyberChef/pull/489 [#489]: https://github.com/gchq/CyberChef/pull/489
[#496]: https://github.com/gchq/CyberChef/pull/496
[#506]: https://github.com/gchq/CyberChef/pull/506 [#506]: https://github.com/gchq/CyberChef/pull/506
[#516]: https://github.com/gchq/CyberChef/pull/516 [#516]: https://github.com/gchq/CyberChef/pull/516
[#525]: https://github.com/gchq/CyberChef/pull/525
[#533]: https://github.com/gchq/CyberChef/pull/533

121
Gruntfile.js
View File

@ -4,7 +4,6 @@ const webpack = require("webpack");
const HtmlWebpackPlugin = require("html-webpack-plugin"); const HtmlWebpackPlugin = require("html-webpack-plugin");
const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin; const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
const NodeExternals = require("webpack-node-externals"); const NodeExternals = require("webpack-node-externals");
const Inliner = require("web-resource-inliner");
const glob = require("glob"); const glob = require("glob");
const path = require("path"); const path = require("path");
@ -43,18 +42,16 @@ module.exports = function (grunt) {
grunt.registerTask("prod", grunt.registerTask("prod",
"Creates a production-ready build. Use the --msg flag to add a compile message.", "Creates a production-ready build. Use the --msg flag to add a compile message.",
["eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web", "inline", "chmod"]); [
"eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web",
"copy:standalone", "zip:standalone", "clean:standalone", "chmod"
]);
grunt.registerTask("default", grunt.registerTask("default",
"Lints the code base", "Lints the code base",
["eslint", "exec:repoSize"]); ["eslint", "exec:repoSize"]);
grunt.registerTask("inline",
"Compiles a production build of CyberChef into a single, portable web page.",
["exec:generateConfig", "webpack:webInline", "runInliner", "clean:inlineScripts"]);
grunt.registerTask("runInliner", runInliner);
grunt.registerTask("doc", "docs"); grunt.registerTask("doc", "docs");
grunt.registerTask("tests", "test"); grunt.registerTask("tests", "test");
grunt.registerTask("lint", "eslint"); grunt.registerTask("lint", "eslint");
@ -72,6 +69,7 @@ module.exports = function (grunt) {
grunt.loadNpmTasks("grunt-accessibility"); grunt.loadNpmTasks("grunt-accessibility");
grunt.loadNpmTasks("grunt-concurrent"); grunt.loadNpmTasks("grunt-concurrent");
grunt.loadNpmTasks("grunt-contrib-connect"); grunt.loadNpmTasks("grunt-contrib-connect");
grunt.loadNpmTasks("grunt-zip");
// Project configuration // Project configuration
@ -94,32 +92,6 @@ module.exports = function (grunt) {
}, },
moduleEntryPoints = listEntryModules(); moduleEntryPoints = listEntryModules();
/**
* Compiles a production build of CyberChef into a single, portable web page.
*/
function runInliner() {
const done = this.async();
Inliner.html({
relativeTo: "build/prod/",
fileContent: grunt.file.read("build/prod/cyberchef.htm"),
images: true,
svgs: true,
scripts: true,
links: true,
strict: true
}, function(error, result) {
if (error) {
if (error instanceof Error) {
done(error);
} else {
done(new Error(error));
}
} else {
grunt.file.write("build/prod/cyberchef.htm", result);
done(true);
}
});
}
/** /**
* Generates an entry list for all the modules. * Generates an entry list for all the modules.
@ -130,7 +102,7 @@ module.exports = function (grunt) {
glob.sync("./src/core/config/modules/*.mjs").forEach(file => { glob.sync("./src/core/config/modules/*.mjs").forEach(file => {
const basename = path.basename(file); const basename = path.basename(file);
if (basename !== "Default.mjs" && basename !== "OpModules.mjs") if (basename !== "Default.mjs" && basename !== "OpModules.mjs")
entryModules[basename.split(".mjs")[0]] = path.resolve(file); entryModules["modules/" + basename.split(".mjs")[0]] = path.resolve(file);
}); });
return entryModules; return entryModules;
@ -143,7 +115,7 @@ module.exports = function (grunt) {
node: ["build/node/*"], node: ["build/node/*"],
config: ["src/core/config/OperationConfig.json", "src/core/config/modules/*", "src/code/operations/index.mjs"], config: ["src/core/config/OperationConfig.json", "src/core/config/modules/*", "src/code/operations/index.mjs"],
docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico", "!docs/*.png"], docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico", "!docs/*.png"],
inlineScripts: ["build/prod/scripts.js"], standalone: ["build/prod/CyberChef*.html"]
}, },
eslint: { eslint: {
options: { options: {
@ -151,7 +123,7 @@ module.exports = function (grunt) {
}, },
configs: ["*.{js,mjs}"], configs: ["*.{js,mjs}"],
core: ["src/core/**/*.{js,mjs}", "!src/core/vendor/**/*", "!src/core/operations/legacy/**/*"], core: ["src/core/**/*.{js,mjs}", "!src/core/vendor/**/*", "!src/core/operations/legacy/**/*"],
web: ["src/web/**/*.{js,mjs}"], web: ["src/web/**/*.{js,mjs}", "!src/web/static/**/*"],
node: ["src/node/**/*.{js,mjs}"], node: ["src/node/**/*.{js,mjs}"],
tests: ["tests/**/*.{js,mjs}"], tests: ["tests/**/*.{js,mjs}"],
}, },
@ -195,6 +167,9 @@ module.exports = function (grunt) {
}, moduleEntryPoints), }, moduleEntryPoints),
output: { output: {
path: __dirname + "/build/prod", path: __dirname + "/build/prod",
filename: chunkData => {
return chunkData.chunk.name === "main" ? "assets/[name].js": "[name].js";
},
globalObject: "this" globalObject: "this"
}, },
resolve: { resolve: {
@ -225,33 +200,6 @@ module.exports = function (grunt) {
] ]
}; };
}, },
webInline: {
mode: "production",
target: "web",
entry: "./src/web/index.js",
output: {
filename: "scripts.js",
path: __dirname + "/build/prod"
},
plugins: [
new webpack.DefinePlugin(Object.assign({}, BUILD_CONSTANTS, {
INLINE: "true"
})),
new HtmlWebpackPlugin({
filename: "cyberchef.htm",
template: "./src/web/html/index.html",
compileTime: compileTime,
version: pkg.version + "s",
inline: true,
minify: {
removeComments: true,
collapseWhitespace: true,
minifyJS: true,
minifyCSS: true
}
}),
]
},
node: { node: {
mode: "production", mode: "production",
target: "node", target: "node",
@ -284,7 +232,8 @@ module.exports = function (grunt) {
warningsFilter: [ warningsFilter: [
/source-map/, /source-map/,
/dependency is an expression/, /dependency is an expression/,
/export 'default'/ /export 'default'/,
/Can't resolve 'sodium'/
], ],
} }
}, },
@ -316,6 +265,18 @@ module.exports = function (grunt) {
} }
} }
}, },
zip: {
standalone: {
cwd: "build/prod/",
src: [
"build/prod/**/*",
"!build/prod/index.html",
"!build/prod/BundleAnalyzerReport.html",
"!build/prod/sitemap.js"
],
dest: `build/prod/CyberChef_v${pkg.version}.zip`
}
},
connect: { connect: {
prod: { prod: {
options: { options: {
@ -328,10 +289,16 @@ module.exports = function (grunt) {
ghPages: { ghPages: {
options: { options: {
process: function (content, srcpath) { process: function (content, srcpath) {
// Add Google Analytics code to index.html
if (srcpath.indexOf("index.html") >= 0) { if (srcpath.indexOf("index.html") >= 0) {
// Add Google Analytics code to index.html
content = content.replace("</body></html>", content = content.replace("</body></html>",
grunt.file.read("src/web/static/ga.html") + "</body></html>"); grunt.file.read("src/web/static/ga.html") + "</body></html>");
// Add Structured Data for SEO
content = content.replace("</head>",
"<script type='application/ld+json'>" +
JSON.stringify(JSON.parse(grunt.file.read("src/web/static/structuredData.json"))) +
"</script></head>");
return grunt.template.process(content, srcpath); return grunt.template.process(content, srcpath);
} else { } else {
return content; return content;
@ -350,6 +317,28 @@ module.exports = function (grunt) {
dest: "build/prod/" dest: "build/prod/"
}, },
] ]
},
standalone: {
options: {
process: function (content, srcpath) {
if (srcpath.indexOf("index.html") >= 0) {
// Replace download link with version number
content = content.replace(/<a [^>]+>Download CyberChef.+?<\/a>/,
`<span>Version ${pkg.version}</span>`);
return grunt.template.process(content, srcpath);
} else {
return content;
}
},
noProcess: ["**", "!**/*.html"]
},
files: [
{
src: "build/prod/index.html",
dest: `build/prod/CyberChef_v${pkg.version}.html`
}
]
} }
}, },
chmod: { chmod: {
@ -405,7 +394,7 @@ module.exports = function (grunt) {
command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs" command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
}, },
browserTests: { browserTests: {
command: "./node_modules/.bin/nightwatch --env prod,inline" command: "./node_modules/.bin/nightwatch --env prod"
} }
}, },
}); });

16
babel.config.js
View File

@ -11,14 +11,22 @@ module.exports = function(api) {
"node": "6.5" "node": "6.5"
}, },
"modules": false, "modules": false,
"useBuiltIns": "entry" "useBuiltIns": "entry",
"corejs": 3
}] }]
], ],
"plugins": [ "plugins": [
"babel-plugin-syntax-dynamic-import", "babel-plugin-syntax-dynamic-import",
["babel-plugin-transform-builtin-extend", { [
"globals": ["Error"] "babel-plugin-transform-builtin-extend", {
}] "globals": ["Error"]
}
],
[
"@babel/plugin-transform-runtime", {
"regenerator": true
}
]
] ]
}; };
}; };

4
nightwatch.json
View File

@ -23,10 +23,6 @@
"prod": { "prod": {
"launch_url": "http://localhost:8000/index.html" "launch_url": "http://localhost:8000/index.html"
},
"inline": {
"launch_url": "http://localhost:8000/cyberchef.htm"
} }
} }

5909
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

78
package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "cyberchef", "name": "cyberchef",
"version": "8.27.0", "version": "8.31.1",
"description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
"author": "n1474335 <n1474335@gmail.com>", "author": "n1474335 <n1474335@gmail.com>",
"homepage": "https://gchq.github.io/CyberChef", "homepage": "https://gchq.github.io/CyberChef",
@ -30,20 +30,20 @@
"main": "build/node/CyberChef.js", "main": "build/node/CyberChef.js",
"bugs": "https://github.com/gchq/CyberChef/issues", "bugs": "https://github.com/gchq/CyberChef/issues",
"devDependencies": { "devDependencies": {
"@babel/core": "^7.2.2", "@babel/core": "^7.4.3",
"@babel/preset-env": "^7.2.3", "@babel/plugin-transform-runtime": "^7.4.3",
"autoprefixer": "^9.4.3", "@babel/preset-env": "^7.4.3",
"autoprefixer": "^9.5.1",
"babel-eslint": "^10.0.1", "babel-eslint": "^10.0.1",
"babel-loader": "^8.0.4", "babel-loader": "^8.0.5",
"babel-plugin-syntax-dynamic-import": "^6.18.0", "babel-plugin-syntax-dynamic-import": "^6.18.0",
"bootstrap": "^4.2.1", "chromedriver": "^2.46.0",
"chromedriver": "^2.45.0",
"colors": "^1.3.3", "colors": "^1.3.3",
"css-loader": "^2.1.0", "css-loader": "^2.1.1",
"eslint": "^5.12.1", "eslint": "^5.16.0",
"exports-loader": "^0.7.0", "exports-loader": "^0.7.0",
"file-loader": "^3.0.1", "file-loader": "^3.0.1",
"grunt": "^1.0.3", "grunt": "^1.0.4",
"grunt-accessibility": "~6.0.0", "grunt-accessibility": "~6.0.0",
"grunt-chmod": "~1.1.1", "grunt-chmod": "~1.1.1",
"grunt-concurrent": "^2.3.1", "grunt-concurrent": "^2.3.1",
@ -53,16 +53,17 @@
"grunt-contrib-watch": "^1.1.0", "grunt-contrib-watch": "^1.1.0",
"grunt-eslint": "^21.0.0", "grunt-eslint": "^21.0.0",
"grunt-exec": "~3.0.0", "grunt-exec": "~3.0.0",
"grunt-jsdoc": "^2.3.0", "grunt-jsdoc": "^2.3.1",
"grunt-webpack": "^3.1.3", "grunt-webpack": "^3.1.3",
"grunt-zip": "^0.18.2",
"html-webpack-plugin": "^3.2.0", "html-webpack-plugin": "^3.2.0",
"imports-loader": "^0.8.0", "imports-loader": "^0.8.0",
"ink-docstrap": "^1.3.2", "ink-docstrap": "^1.3.2",
"jsdoc-babel": "^0.5.0", "jsdoc-babel": "^0.5.0",
"mini-css-extract-plugin": "^0.5.0", "mini-css-extract-plugin": "^0.6.0",
"nightwatch": "^1.0.18", "nightwatch": "^1.0.19",
"node-sass": "^4.11.0", "node-sass": "^4.11.0",
"postcss-css-variables": "^0.11.0", "postcss-css-variables": "^0.12.0",
"postcss-import": "^12.0.1", "postcss-import": "^12.0.1",
"postcss-loader": "^3.0.0", "postcss-loader": "^3.0.0",
"prompt": "^1.0.0", "prompt": "^1.0.0",
@ -71,62 +72,69 @@
"style-loader": "^0.23.1", "style-loader": "^0.23.1",
"svg-url-loader": "^2.3.2", "svg-url-loader": "^2.3.2",
"url-loader": "^1.1.2", "url-loader": "^1.1.2",
"web-resource-inliner": "^4.2.1", "webpack": "^4.29.6",
"webpack": "^4.28.3", "webpack-bundle-analyzer": "^3.3.2",
"webpack-bundle-analyzer": "^3.0.3", "webpack-dev-server": "^3.3.1",
"webpack-dev-server": "^3.1.14",
"webpack-node-externals": "^1.7.2", "webpack-node-externals": "^1.7.2",
"worker-loader": "^2.0.0" "worker-loader": "^2.0.0"
}, },
"dependencies": { "dependencies": {
"@babel/polyfill": "^7.4.3",
"@babel/runtime": "^7.4.3",
"arrive": "^2.4.1", "arrive": "^2.4.1",
"babel-plugin-transform-builtin-extend": "1.1.2", "babel-plugin-transform-builtin-extend": "1.1.2",
"babel-polyfill": "^6.26.0",
"bcryptjs": "^2.4.3", "bcryptjs": "^2.4.3",
"bignumber.js": "^8.0.2", "bignumber.js": "^8.1.1",
"blakejs": "^1.1.0",
"bootstrap": "4.2.1",
"bootstrap-colorpicker": "^2.5.3", "bootstrap-colorpicker": "^2.5.3",
"bootstrap-material-design": "^4.1.1", "bootstrap-material-design": "^4.1.1",
"bson": "^4.0.1", "bson": "^4.0.2",
"chi-squared": "^1.1.0", "chi-squared": "^1.1.0",
"clippyjs": "0.0.3",
"core-js": "^3.0.1",
"crypto-api": "^0.8.3", "crypto-api": "^0.8.3",
"crypto-js": "^3.1.9-1", "crypto-js": "^3.1.9-1",
"ctph.js": "0.0.5", "ctph.js": "0.0.5",
"diff": "^3.5.0", "d3": "^4.9.1",
"d3-hexbin": "^0.2.2",
"diff": "^4.0.1",
"es6-promisify": "^6.0.1", "es6-promisify": "^6.0.1",
"escodegen": "^1.11.0", "escodegen": "^1.11.1",
"esmangle": "^1.0.1", "esmangle": "^1.0.1",
"esprima": "^4.0.1", "esprima": "^4.0.1",
"exif-parser": "^0.1.12", "exif-parser": "^0.1.12",
"file-saver": "^2.0.0", "file-saver": "^2.0.1",
"geodesy": "^1.1.3", "geodesy": "^1.1.3",
"highlight.js": "^9.13.1", "highlight.js": "^9.15.6",
"jimp": "^0.6.0", "jimp": "^0.6.1",
"jquery": "^3.3.1", "jquery": "3.3.1",
"js-crc": "^0.2.0", "js-crc": "^0.2.0",
"js-sha3": "^0.8.0", "js-sha3": "^0.8.0",
"jsesc": "^2.5.2", "jsesc": "^2.5.2",
"jsonpath": "^1.0.0", "jsonpath": "^1.0.1",
"jsonwebtoken": "^8.4.0", "jsonwebtoken": "^8.5.1",
"jsqr": "^1.1.1", "jsqr": "^1.2.0",
"jsrsasign": "8.0.12", "jsrsasign": "8.0.12",
"kbpgp": "^2.0.82", "kbpgp": "2.1.0",
"libyara-wasm": "0.0.12", "libyara-wasm": "0.0.12",
"lodash": "^4.17.11", "lodash": "^4.17.11",
"loglevel": "^1.6.1", "loglevel": "^1.6.1",
"loglevel-message-prefix": "^3.0.0", "loglevel-message-prefix": "^3.0.0",
"moment": "^2.23.0", "moment": "^2.24.0",
"moment-timezone": "^0.5.23", "moment-timezone": "^0.5.23",
"ngeohash": "^0.6.3", "ngeohash": "^0.6.3",
"node-forge": "^0.7.6", "node-forge": "^0.8.2",
"node-md6": "^0.1.0", "node-md6": "^0.1.0",
"nodom": "^2.2.0",
"notepack.io": "^2.2.0", "notepack.io": "^2.2.0",
"nwmatcher": "^1.4.4", "nwmatcher": "^1.4.4",
"otp": "^0.1.3", "otp": "^0.1.3",
"popper.js": "^1.14.6", "popper.js": "^1.15.0",
"qr-image": "^3.2.0", "qr-image": "^3.2.0",
"scryptsy": "^2.0.0", "scryptsy": "^2.0.0",
"snackbarjs": "^1.1.0", "snackbarjs": "^1.1.0",
"sortablejs": "^1.8.0-rc1", "sortablejs": "^1.8.4",
"split.js": "^1.5.10", "split.js": "^1.5.10",
"ssdeep.js": "0.0.2", "ssdeep.js": "0.0.2",
"ua-parser-js": "^0.7.19", "ua-parser-js": "^0.7.19",

3
src/core/ChefWorker.js
View File

@ -6,7 +6,6 @@
* @license Apache-2.0 * @license Apache-2.0
*/ */
import "babel-polyfill";
import Chef from "./Chef"; import Chef from "./Chef";
import OperationConfig from "./config/OperationConfig.json"; import OperationConfig from "./config/OperationConfig.json";
import OpModules from "./config/modules/OpModules"; import OpModules from "./config/modules/OpModules";
@ -179,7 +178,7 @@ self.loadRequiredModules = function(recipeConfig) {
if (!OpModules.hasOwnProperty(module)) { if (!OpModules.hasOwnProperty(module)) {
log.info(`Loading ${module} module`); log.info(`Loading ${module} module`);
self.sendStatusMessage(`Loading ${module} module`); self.sendStatusMessage(`Loading ${module} module`);
self.importScripts(`${self.docURL}/${module}.js`); self.importScripts(`${self.docURL}/modules/${module}.js`);
self.sendStatusMessage(""); self.sendStatusMessage("");
} }
}); });

69
src/core/Dish.mjs
View File

@ -21,8 +21,8 @@ class Dish {
* @param {Dish} [dish=null] - A dish to clone * @param {Dish} [dish=null] - A dish to clone
*/ */
constructor(dish=null) { constructor(dish=null) {
this.value = []; this.value = new ArrayBuffer(0);
this.type = Dish.BYTE_ARRAY; this.type = Dish.ARRAY_BUFFER;
if (dish && if (dish &&
dish.hasOwnProperty("value") && dish.hasOwnProperty("value") &&
@ -149,78 +149,75 @@ class Dish {
*/ */
async _translate(toType, notUTF8=false) { async _translate(toType, notUTF8=false) {
log.debug(`Translating Dish from ${Dish.enumLookup(this.type)} to ${Dish.enumLookup(toType)}`); log.debug(`Translating Dish from ${Dish.enumLookup(this.type)} to ${Dish.enumLookup(toType)}`);
const byteArrayToStr = notUTF8 ? Utils.byteArrayToChars : Utils.byteArrayToUtf8;
// Convert data to intermediate byteArray type // Convert data to intermediate ArrayBuffer type
try { try {
switch (this.type) { switch (this.type) {
case Dish.STRING: case Dish.STRING:
this.value = this.value ? Utils.strToByteArray(this.value) : []; this.value = this.value ? Utils.strToArrayBuffer(this.value) : new ArrayBuffer;
break; break;
case Dish.NUMBER: case Dish.NUMBER:
this.value = typeof this.value === "number" ? Utils.strToByteArray(this.value.toString()) : []; this.value = typeof this.value === "number" ? Utils.strToArrayBuffer(this.value.toString()) : new ArrayBuffer;
break; break;
case Dish.HTML: case Dish.HTML:
this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : []; this.value = this.value ? Utils.strToArrayBuffer(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : new ArrayBuffer;
break; break;
case Dish.ARRAY_BUFFER: case Dish.BYTE_ARRAY:
// Array.from() would be nicer here, but it's slightly slower this.value = new Uint8Array(this.value).buffer;
this.value = Array.prototype.slice.call(new Uint8Array(this.value));
break; break;
case Dish.BIG_NUMBER: case Dish.BIG_NUMBER:
this.value = BigNumber.isBigNumber(this.value) ? Utils.strToByteArray(this.value.toFixed()) : []; this.value = BigNumber.isBigNumber(this.value) ? Utils.strToArrayBuffer(this.value.toFixed()) : new ArrayBuffer;
break; break;
case Dish.JSON: case Dish.JSON:
this.value = this.value ? Utils.strToByteArray(JSON.stringify(this.value, null, 4)) : []; this.value = this.value ? Utils.strToArrayBuffer(JSON.stringify(this.value, null, 4)) : new ArrayBuffer;
break; break;
case Dish.FILE: case Dish.FILE:
this.value = await Utils.readFile(this.value); this.value = (await Utils.readFile(this.value)).buffer;
this.value = Array.prototype.slice.call(this.value);
break; break;
case Dish.LIST_FILE: case Dish.LIST_FILE:
this.value = await Promise.all(this.value.map(async f => Utils.readFile(f))); this.value = await Promise.all(this.value.map(async f => Utils.readFile(f)));
this.value = this.value.map(b => Array.prototype.slice.call(b)); this.value = concatenateTypedArrays(...this.value).buffer;
this.value = [].concat.apply([], this.value);
break; break;
default: default:
break; break;
} }
} catch (err) { } catch (err) {
throw new DishError(`Error translating from ${Dish.enumLookup(this.type)} to byteArray: ${err}`); throw new DishError(`Error translating from ${Dish.enumLookup(this.type)} to ArrayBuffer: ${err}`);
} }
this.type = Dish.BYTE_ARRAY; this.type = Dish.ARRAY_BUFFER;
// Convert from byteArray to toType // Convert from ArrayBuffer to toType
try { try {
switch (toType) { switch (toType) {
case Dish.STRING: case Dish.STRING:
case Dish.HTML: case Dish.HTML:
this.value = this.value ? byteArrayToStr(this.value) : ""; this.value = this.value ? Utils.arrayBufferToStr(this.value, !notUTF8) : "";
this.type = Dish.STRING; this.type = Dish.STRING;
break; break;
case Dish.NUMBER: case Dish.NUMBER:
this.value = this.value ? parseFloat(byteArrayToStr(this.value)) : 0; this.value = this.value ? parseFloat(Utils.arrayBufferToStr(this.value, !notUTF8)) : 0;
this.type = Dish.NUMBER; this.type = Dish.NUMBER;
break; break;
case Dish.ARRAY_BUFFER: case Dish.BYTE_ARRAY:
this.value = new Uint8Array(this.value).buffer; this.value = Array.prototype.slice.call(new Uint8Array(this.value));
this.type = Dish.ARRAY_BUFFER; this.type = Dish.ARRAY_BUFFER;
break; break;
case Dish.BIG_NUMBER: case Dish.BIG_NUMBER:
try { try {
this.value = new BigNumber(byteArrayToStr(this.value)); this.value = new BigNumber(Utils.arrayBufferToStr(this.value, !notUTF8));
} catch (err) { } catch (err) {
this.value = new BigNumber(NaN); this.value = new BigNumber(NaN);
} }
this.type = Dish.BIG_NUMBER; this.type = Dish.BIG_NUMBER;
break; break;
case Dish.JSON: case Dish.JSON:
this.value = JSON.parse(byteArrayToStr(this.value)); this.value = JSON.parse(Utils.arrayBufferToStr(this.value, !notUTF8));
this.type = Dish.JSON; this.type = Dish.JSON;
break; break;
case Dish.FILE: case Dish.FILE:
this.value = new File(this.value, "unknown"); this.value = new File(this.value, "unknown");
this.type = Dish.FILE;
break; break;
case Dish.LIST_FILE: case Dish.LIST_FILE:
this.value = [new File(this.value, "unknown")]; this.value = [new File(this.value, "unknown")];
@ -230,7 +227,7 @@ class Dish {
break; break;
} }
} catch (err) { } catch (err) {
throw new DishError(`Error translating from byteArray to ${Dish.enumLookup(toType)}: ${err}`); throw new DishError(`Error translating from ArrayBuffer to ${Dish.enumLookup(toType)}: ${err}`);
} }
} }
@ -374,6 +371,26 @@ class Dish {
} }
/**
* Concatenates a list of Uint8Arrays together
*
* @param {Uint8Array[]} arrays
* @returns {Uint8Array}
*/
function concatenateTypedArrays(...arrays) {
let totalLength = 0;
for (const arr of arrays) {
totalLength += arr.length;
}
const result = new Uint8Array(totalLength);
let offset = 0;
for (const arr of arrays) {
result.set(arr, offset);
offset += arr.length;
}
return result;
}
/** /**
* Dish data type enum for byte arrays. * Dish data type enum for byte arrays.

65
src/core/Utils.mjs
View File

@ -367,6 +367,61 @@ class Utils {
} }
/**
* Converts a string to an ArrayBuffer.
* Treats the string as UTF-8 if any values are over 255.
*
* @param {string} str
* @returns {ArrayBuffer}
*
* @example
* // returns [72,101,108,108,111]
* Utils.strToArrayBuffer("Hello");
*
* // returns [228,189,160,229,165,189]
* Utils.strToArrayBuffer("你好");
*/
static strToArrayBuffer(str) {
const arr = new Uint8Array(str.length);
let i = str.length, b;
while (i--) {
b = str.charCodeAt(i);
arr[i] = b;
// If any of the bytes are over 255, read as UTF-8
if (b > 255) return Utils.strToUtf8ArrayBuffer(str);
}
return arr.buffer;
}
/**
* Converts a string to a UTF-8 ArrayBuffer.
*
* @param {string} str
* @returns {ArrayBuffer}
*
* @example
* // returns [72,101,108,108,111]
* Utils.strToUtf8ArrayBuffer("Hello");
*
* // returns [228,189,160,229,165,189]
* Utils.strToUtf8ArrayBuffer("你好");
*/
static strToUtf8ArrayBuffer(str) {
const utf8Str = utf8.encode(str);
if (str.length !== utf8Str.length) {
if (ENVIRONMENT_IS_WORKER()) {
self.setOption("attemptHighlight", false);
} else if (ENVIRONMENT_IS_WEB()) {
window.app.options.attemptHighlight = false;
}
}
return Utils.strToArrayBuffer(utf8Str);
}
/** /**
* Converts a string to a byte array. * Converts a string to a byte array.
* Treats the string as UTF-8 if any values are over 255. * Treats the string as UTF-8 if any values are over 255.
@ -459,7 +514,7 @@ class Utils {
/** /**
* Attempts to convert a byte array to a UTF-8 string. * Attempts to convert a byte array to a UTF-8 string.
* *
* @param {byteArray} byteArray * @param {byteArray|Uint8Array} byteArray
* @returns {string} * @returns {string}
* *
* @example * @example
@ -505,6 +560,7 @@ class Utils {
static byteArrayToChars(byteArray) { static byteArrayToChars(byteArray) {
if (!byteArray) return ""; if (!byteArray) return "";
let str = ""; let str = "";
// String concatenation appears to be faster than an array join
for (let i = 0; i < byteArray.length;) { for (let i = 0; i < byteArray.length;) {
str += String.fromCharCode(byteArray[i++]); str += String.fromCharCode(byteArray[i++]);
} }
@ -524,8 +580,8 @@ class Utils {
* Utils.arrayBufferToStr(Uint8Array.from([104,101,108,108,111]).buffer); * Utils.arrayBufferToStr(Uint8Array.from([104,101,108,108,111]).buffer);
*/ */
static arrayBufferToStr(arrayBuffer, utf8=true) { static arrayBufferToStr(arrayBuffer, utf8=true) {
const byteArray = Array.prototype.slice.call(new Uint8Array(arrayBuffer)); const arr = new Uint8Array(arrayBuffer);
return utf8 ? Utils.byteArrayToUtf8(byteArray) : Utils.byteArrayToChars(byteArray); return utf8 ? Utils.byteArrayToUtf8(arr) : Utils.byteArrayToChars(arr);
} }
@ -1023,9 +1079,11 @@ class Utils {
static charRep(token) { static charRep(token) {
return { return {
"Space": " ", "Space": " ",
"Percent": "%",
"Comma": ",", "Comma": ",",
"Semi-colon": ";", "Semi-colon": ";",
"Colon": ":", "Colon": ":",
"Tab": "\t",
"Line feed": "\n", "Line feed": "\n",
"CRLF": "\r\n", "CRLF": "\r\n",
"Forward slash": "/", "Forward slash": "/",
@ -1047,6 +1105,7 @@ class Utils {
static regexRep(token) { static regexRep(token) {
return { return {
"Space": /\s+/g, "Space": /\s+/g,
"Percent": /%/g,
"Comma": /,/g, "Comma": /,/g,
"Semi-colon": /;/g, "Semi-colon": /;/g,
"Colon": /:/g, "Colon": /:/g,

12
src/core/config/Categories.json
View File

@ -169,6 +169,9 @@
"Parse URI", "Parse URI",
"URL Encode", "URL Encode",
"URL Decode", "URL Decode",
"Protobuf Decode",
"VarInt Encode",
"VarInt Decode",
"Format MAC addresses", "Format MAC addresses",
"Change IP format", "Change IP format",
"Group IP addresses", "Group IP addresses",
@ -297,6 +300,8 @@
"HAS-160", "HAS-160",
"Whirlpool", "Whirlpool",
"Snefru", "Snefru",
"BLAKE2b",
"BLAKE2s",
"SSDEEP", "SSDEEP",
"CTPH", "CTPH",
"Compare SSDEEP hashes", "Compare SSDEEP hashes",
@ -378,7 +383,11 @@
"Image Filter", "Image Filter",
"Contain Image", "Contain Image",
"Cover Image", "Cover Image",
"Image Hue/Saturation/Lightness" "Image Hue/Saturation/Lightness",
"Hex Density chart",
"Scatter chart",
"Series chart",
"Heatmap chart"
] ]
}, },
{ {
@ -395,6 +404,7 @@
"Generate QR Code", "Generate QR Code",
"Parse QR Code", "Parse QR Code",
"Haversine distance", "Haversine distance",
"HTML To Text",
"Generate Lorem Ipsum", "Generate Lorem Ipsum",
"Numberwang", "Numberwang",
"XKCD Random Number" "XKCD Random Number"

178
src/core/lib/Charts.mjs Normal file
View File

@ -0,0 +1,178 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import OperationError from "../errors/OperationError";
/**
* @constant
* @default
*/
export const RECORD_DELIMITER_OPTIONS = ["Line feed", "CRLF"];
/**
* @constant
* @default
*/
export const FIELD_DELIMITER_OPTIONS = ["Space", "Comma", "Semi-colon", "Colon", "Tab"];
/**
* Default from colour
*
* @constant
* @default
*/
export const COLOURS = {
min: "white",
max: "black"
};
/**
* Gets values from input for a plot.
*
* @param {string} input
* @param {string} recordDelimiter
* @param {string} fieldDelimiter
* @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
* @param {number} length
* @returns {Object[]}
*/
export function getValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded, length) {
let headings;
const values = [];
input
.split(recordDelimiter)
.forEach((row, rowIndex) => {
const split = row.split(fieldDelimiter);
if (split.length !== length) throw new OperationError(`Each row must have length ${length}.`);
if (columnHeadingsAreIncluded && rowIndex === 0) {
headings = split;
} else {
values.push(split);
}
});
return { headings, values };
}
/**
* Gets values from input for a scatter plot.
*
* @param {string} input
* @param {string} recordDelimiter
* @param {string} fieldDelimiter
* @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
* @returns {Object[]}
*/
export function getScatterValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
let { headings, values } = getValues(
input,
recordDelimiter,
fieldDelimiter,
columnHeadingsAreIncluded,
2
);
if (headings) {
headings = {x: headings[0], y: headings[1]};
}
values = values.map(row => {
const x = parseFloat(row[0], 10),
y = parseFloat(row[1], 10);
if (Number.isNaN(x)) throw new OperationError("Values must be numbers in base 10.");
if (Number.isNaN(y)) throw new OperationError("Values must be numbers in base 10.");
return [x, y];
});
return { headings, values };
}
/**
* Gets values from input for a scatter plot with colour from the third column.
*
* @param {string} input
* @param {string} recordDelimiter
* @param {string} fieldDelimiter
* @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
* @returns {Object[]}
*/
export function getScatterValuesWithColour(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
let { headings, values } = getValues(
input,
recordDelimiter, fieldDelimiter,
columnHeadingsAreIncluded,
3
);
if (headings) {
headings = {x: headings[0], y: headings[1]};
}
values = values.map(row => {
const x = parseFloat(row[0], 10),
y = parseFloat(row[1], 10),
colour = row[2];
if (Number.isNaN(x)) throw new OperationError("Values must be numbers in base 10.");
if (Number.isNaN(y)) throw new OperationError("Values must be numbers in base 10.");
return [x, y, colour];
});
return { headings, values };
}
/**
* Gets values from input for a time series plot.
*
* @param {string} input
* @param {string} recordDelimiter
* @param {string} fieldDelimiter
* @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
* @returns {Object[]}
*/
export function getSeriesValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
const { values } = getValues(
input,
recordDelimiter, fieldDelimiter,
false,
3
);
let xValues = new Set();
const series = {};
values.forEach(row => {
const serie = row[0],
xVal = row[1],
val = parseFloat(row[2], 10);
if (Number.isNaN(val)) throw new OperationError("Values must be numbers in base 10.");
xValues.add(xVal);
if (typeof series[serie] === "undefined") series[serie] = {};
series[serie][xVal] = val;
});
xValues = new Array(...xValues);
const seriesList = [];
for (const seriesName in series) {
const serie = series[seriesName];
seriesList.push({name: seriesName, data: serie});
}
return { xValues, series: seriesList };
}

2
src/core/lib/Hex.mjs
View File

@ -100,7 +100,7 @@ export function fromHex(data, delim="Auto", byteLen=2) {
/** /**
* To Hexadecimal delimiters. * To Hexadecimal delimiters.
*/ */
export const TO_HEX_DELIM_OPTIONS = ["Space", "Comma", "Semi-colon", "Colon", "Line feed", "CRLF", "0x", "\\x", "None"]; export const TO_HEX_DELIM_OPTIONS = ["Space", "Percent", "Comma", "Semi-colon", "Colon", "Line feed", "CRLF", "0x", "\\x", "None"];
/** /**

285
src/core/lib/Protobuf.mjs Normal file
View File

@ -0,0 +1,285 @@
import Utils from "../Utils";
/**
* Protobuf lib. Contains functions to decode protobuf serialised
* data without a schema or .proto file.
*
* Provides utility functions to encode and decode variable length
* integers (varint).
*
* @author GCHQ Contributor [3]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
class Protobuf {
/**
* Protobuf constructor
*
* @param {byteArray} data
*/
constructor(data) {
// Check we have a byteArray
if (data instanceof Array) {
this.data = data;
} else {
throw new Error("Protobuf input must be a byteArray");
}
// Set up masks
this.TYPE = 0x07;
this.NUMBER = 0x78;
this.MSB = 0x80;
this.VALUE = 0x7f;
// Declare offset and length
this.offset = 0;
this.LENGTH = data.length;
}
// Public Functions
/**
* Encode a varint from a number
*
* @param {number} number
* @returns {byteArray}
*/
static varIntEncode(number) {
const MSB = 0x80,
VALUE = 0x7f,
MSBALL = ~VALUE,
INT = Math.pow(2, 31);
const out = [];
let offset = 0;
while (number >= INT) {
out[offset++] = (number & 0xff) | MSB;
number /= 128;
}
while (number & MSBALL) {
out[offset++] = (number & 0xff) | MSB;
number >>>= 7;
}
out[offset] = number | 0;
return out;
}
/**
* Decode a varint from the byteArray
*
* @param {byteArray} input
* @returns {number}
*/
static varIntDecode(input) {
const pb = new Protobuf(input);
return pb._varInt();
}
/**
* Parse Protobuf data
*
* @param {byteArray} input
* @returns {Object}
*/
static decode(input) {
const pb = new Protobuf(input);
return pb._parse();
}
// Private Class Functions
/**
* Main private parsing function
*
* @private
* @returns {Object}
*/
_parse() {
let object = {};
// Continue reading whilst we still have data
while (this.offset < this.LENGTH) {
const field = this._parseField();
object = this._addField(field, object);
}
// Throw an error if we have gone beyond the end of the data
if (this.offset > this.LENGTH) {
throw new Error("Exhausted Buffer");
}
return object;
}
/**
* Add a field read from the protobuf data into the Object. As
* protobuf fields can appear multiple times, if the field already
* exists we need to add the new field into an array of fields
* for that key.
*
* @private
* @param {Object} field
* @param {Object} object
* @returns {Object}
*/
_addField(field, object) {
// Get the field key/values
const key = field.key;
const value = field.value;
object[key] = object.hasOwnProperty(key) ?
object[key] instanceof Array ?
object[key].concat([value]) :
[object[key], value] :
value;
return object;
}
/**
* Parse a field and return the Object read from the record
*
* @private
* @returns {Object}
*/
_parseField() {
// Get the field headers
const header = this._fieldHeader();
const type = header.type;
const key = header.key;
switch (type) {
// varint
case 0:
return { "key": key, "value": this._varInt() };
// fixed 64
case 1:
return { "key": key, "value": this._uint64() };
// length delimited
case 2:
return { "key": key, "value": this._lenDelim() };
// fixed 32
case 5:
return { "key": key, "value": this._uint32() };
// unknown type
default:
throw new Error("Unknown type 0x" + type.toString(16));
}
}
/**
* Parse the field header and return the type and key
*
* @private
* @returns {Object}
*/
_fieldHeader() {
// Make sure we call type then number to preserve offset
return { "type": this._fieldType(), "key": this._fieldNumber() };
}
/**
* Parse the field type from the field header. Type is stored in the
* lower 3 bits of the tag byte. This does not move the offset on as
* we need to read the field number from the tag byte too.
*
* @private
* @returns {number}
*/
_fieldType() {
// Field type stored in lower 3 bits of tag byte
return this.data[this.offset] & this.TYPE;
}
/**
* Parse the field number (i.e. the key) from the field header. The
* field number is stored in the upper 5 bits of the tag byte - but
* is also varint encoded so the follow on bytes may need to be read
* when field numbers are > 15.
*
* @private
* @returns {number}
*/
_fieldNumber() {
let shift = -3;
let fieldNumber = 0;
do {
fieldNumber += shift < 28 ?
shift === -3 ?
(this.data[this.offset] & this.NUMBER) >> -shift :
(this.data[this.offset] & this.VALUE) << shift :
(this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
shift += 7;
} while ((this.data[this.offset++] & this.MSD) === this.MSB);
return fieldNumber;
}
// Field Parsing Functions
/**
* Read off a varint from the data
*
* @private
* @returns {number}
*/
_varInt() {
let value = 0;
let shift = 0;
// Keep reading while upper bit set
do {
value += shift < 28 ?
(this.data[this.offset] & this.VALUE) << shift :
(this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
shift += 7;
} while ((this.data[this.offset++] & this.MSB) === this.MSB);
return value;
}
/**
* Read off a 64 bit unsigned integer from the data
*
* @private
* @returns {number}
*/
_uint64() {
// Read off a Uint64
let num = this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
num = num * 0x100000000 + this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
return num;
}
/**
* Read off a length delimited field from the data
*
* @private
* @returns {Object|string}
*/
_lenDelim() {
// Read off the field length
const length = this._varInt();
const fieldBytes = this.data.slice(this.offset, this.offset + length);
let field;
try {
// Attempt to parse as a new Protobuf Object
const pbObject = new Protobuf(fieldBytes);
field = pbObject._parse();
} catch (err) {
// Otherwise treat as bytes
field = Utils.byteArrayToChars(fieldBytes);
}
// Move the offset and return the field
this.offset += length;
return field;
}
/**
* Read a 32 bit unsigned integer from the data
*
* @private
* @returns {number}
*/
_uint32() {
// Use a dataview to read off the integer
const dataview = new DataView(new Uint8Array(this.data.slice(this.offset, this.offset + 4)).buffer);
const value = dataview.getUint32(0);
this.offset += 4;
return value;
}
}
export default Protobuf;

79
src/core/operations/BLAKE2b.mjs Normal file
View File

@ -0,0 +1,79 @@
/**
* @author h345983745
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
import blakejs from "blakejs";
import OperationError from "../errors/OperationError";
import Utils from "../Utils";
import { toBase64 } from "../lib/Base64";
/**
* BLAKE2b operation
*/
class BLAKE2b extends Operation {
/**
* BLAKE2b constructor
*/
constructor() {
super();
this.name = "BLAKE2b";
this.module = "Hashing";
this.description = `Performs BLAKE2b hashing on the input.
<br><br> BLAKE2b is a flavour of the BLAKE cryptographic hash function that is optimized for 64-bit platforms and produces digests of any size between 1 and 64 bytes.
<br><br> Supports the use of an optional key.`;
this.infoURL = "https://wikipedia.org/wiki/BLAKE_(hash_function)#BLAKE2b_algorithm";
this.inputType = "ArrayBuffer";
this.outputType = "string";
this.args = [
{
"name": "Size",
"type": "option",
"value": ["512", "384", "256", "160", "128"]
}, {
"name": "Output Encoding",
"type": "option",
"value": ["Hex", "Base64", "Raw"]
}, {
"name": "Key",
"type": "toggleString",
"value": "",
"toggleValues": ["UTF8", "Decimal", "Base64", "Hex", "Latin1"]
}
];
}
/**
* @param {ArrayBuffer} input
* @param {Object[]} args
* @returns {string} The input having been hashed with BLAKE2b in the encoding format speicifed.
*/
run(input, args) {
const [outSize, outFormat] = args;
let key = Utils.convertToByteArray(args[2].string || "", args[2].option);
if (key.length === 0) {
key = null;
} else if (key.length > 64) {
throw new OperationError(["Key cannot be greater than 64 bytes", "It is currently " + key.length + " bytes."].join("\n"));
}
input = new Uint8Array(input);
switch (outFormat) {
case "Hex":
return blakejs.blake2bHex(input, key, outSize / 8);
case "Base64":
return toBase64(blakejs.blake2b(input, key, outSize / 8));
case "Raw":
return Utils.arrayBufferToStr(blakejs.blake2b(input, key, outSize / 8).buffer);
default:
return new OperationError("Unsupported Output Type");
}
}
}
export default BLAKE2b;

80
src/core/operations/BLAKE2s.mjs Normal file
View File

@ -0,0 +1,80 @@
/**
* @author h345983745
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
import blakejs from "blakejs";
import OperationError from "../errors/OperationError";
import Utils from "../Utils";
import { toBase64 } from "../lib/Base64";
/**
* BLAKE2s Operation
*/
class BLAKE2s extends Operation {
/**
* BLAKE2s constructor
*/
constructor() {
super();
this.name = "BLAKE2s";
this.module = "Hashing";
this.description = `Performs BLAKE2s hashing on the input.
<br><br>BLAKE2s is a flavour of the BLAKE cryptographic hash function that is optimized for 8- to 32-bit platforms and produces digests of any size between 1 and 32 bytes.
<br><br>Supports the use of an optional key.`;
this.infoURL = "https://wikipedia.org/wiki/BLAKE_(hash_function)#BLAKE2";
this.inputType = "ArrayBuffer";
this.outputType = "string";
this.args = [
{
"name": "Size",
"type": "option",
"value": ["256", "160", "128"]
}, {
"name": "Output Encoding",
"type": "option",
"value": ["Hex", "Base64", "Raw"]
},
{
"name": "Key",
"type": "toggleString",
"value": "",
"toggleValues": ["UTF8", "Decimal", "Base64", "Hex", "Latin1"]
}
];
}
/**
* @param {ArrayBuffer} input
* @param {Object[]} args
* @returns {string} The input having been hashed with BLAKE2s in the encoding format speicifed.
*/
run(input, args) {
const [outSize, outFormat] = args;
let key = Utils.convertToByteArray(args[2].string || "", args[2].option);
if (key.length === 0) {
key = null;
} else if (key.length > 32) {
throw new OperationError(["Key cannot be greater than 32 bytes", "It is currently " + key.length + " bytes."].join("\n"));
}
input = new Uint8Array(input);
switch (outFormat) {
case "Hex":
return blakejs.blake2sHex(input, key, outSize / 8);
case "Base64":
return toBase64(blakejs.blake2s(input, key, outSize / 8));
case "Raw":
return Utils.arrayBufferToStr(blakejs.blake2s(input, key, outSize / 8).buffer);
default:
return new OperationError("Unsupported Output Type");
}
}
}
export default BLAKE2s;

2
src/core/operations/ExtractFiles.mjs
View File

@ -23,7 +23,7 @@ class ExtractFiles extends Operation {
this.name = "Extract Files"; this.name = "Extract Files";
this.module = "Default"; this.module = "Default";
this.description = "TODO"; this.description = "Performs file carving to attempt to extract files from the input.<br><br>This operation is currently capable of carving out the following formats:<ul><li>JPG</li><li>EXE</li><li>ZIP</li><li>PDF</li><li>PNG</li><li>BMP</li><li>FLV</li><li>RTF</li><li>DOCX, PPTX, XLSX</li><li>EPUB</li><li>GZIP</li><li>ZLIB</li><li>ELF, BIN, AXF, O, PRX, SO</li></ul>";
this.infoURL = "https://forensicswiki.org/wiki/File_Carving"; this.infoURL = "https://forensicswiki.org/wiki/File_Carving";
this.inputType = "ArrayBuffer"; this.inputType = "ArrayBuffer";
this.outputType = "List<File>"; this.outputType = "List<File>";

10
src/core/operations/GenerateAllHashes.mjs
View File

@ -28,6 +28,8 @@ import Fletcher64Checksum from "./Fletcher64Checksum";
import Adler32Checksum from "./Adler32Checksum"; import Adler32Checksum from "./Adler32Checksum";
import CRC16Checksum from "./CRC16Checksum"; import CRC16Checksum from "./CRC16Checksum";
import CRC32Checksum from "./CRC32Checksum"; import CRC32Checksum from "./CRC32Checksum";
import BLAKE2b from "./BLAKE2b";
import BLAKE2s from "./BLAKE2s";
/** /**
* Generate all hashes operation * Generate all hashes operation
@ -86,6 +88,14 @@ class GenerateAllHashes extends Operation {
"\nWhirlpool-0: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool-0"]) + "\nWhirlpool-0: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool-0"]) +
"\nWhirlpool-T: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool-T"]) + "\nWhirlpool-T: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool-T"]) +
"\nWhirlpool: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool"]) + "\nWhirlpool: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool"]) +
"\nBLAKE2b-128: " + (new BLAKE2b).run(arrayBuffer, ["128", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2b-160: " + (new BLAKE2b).run(arrayBuffer, ["160", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2b-256: " + (new BLAKE2b).run(arrayBuffer, ["256", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2b-384: " + (new BLAKE2b).run(arrayBuffer, ["384", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2b-512: " + (new BLAKE2b).run(arrayBuffer, ["512", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2s-128: " + (new BLAKE2s).run(arrayBuffer, ["128", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2s-160: " + (new BLAKE2s).run(arrayBuffer, ["160", "Hex", {string: "", option: "UTF8"}]) +
"\nBLAKE2s-256: " + (new BLAKE2s).run(arrayBuffer, ["256", "Hex", {string: "", option: "UTF8"}]) +
"\nSSDEEP: " + (new SSDEEP()).run(str) + "\nSSDEEP: " + (new SSDEEP()).run(str) +
"\nCTPH: " + (new CTPH()).run(str) + "\nCTPH: " + (new CTPH()).run(str) +
"\n\nChecksums:" + "\n\nChecksums:" +

41
src/core/operations/HTMLToText.mjs Normal file
View File

@ -0,0 +1,41 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
/**
* HTML To Text operation
*/
class HTMLToText extends Operation {
/**
* HTMLToText constructor
*/
constructor() {
super();
this.name = "HTML To Text";
this.module = "Default";
this.description = "Converts an HTML output from an operation to a readable string instead of being rendered in the DOM.";
this.infoURL = "";
this.inputType = "html";
this.outputType = "string";
this.args = [];
}
/**
* @param {html} input
* @param {Object[]} args
* @returns {string}
*/
run(input, args) {
return input;
}
}
export default HTMLToText;

266
src/core/operations/HeatmapChart.mjs Normal file
View File

@ -0,0 +1,266 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import * as d3temp from "d3";
import * as nodomtemp from "nodom";
import { getScatterValues, RECORD_DELIMITER_OPTIONS, COLOURS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Utils from "../Utils";
const d3 = d3temp.default ? d3temp.default : d3temp;
const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
/**
* Heatmap chart operation
*/
class HeatmapChart extends Operation {
/**
* HeatmapChart constructor
*/
constructor() {
super();
this.name = "Heatmap chart";
this.module = "Charts";
this.description = "A heatmap is a graphical representation of data where the individual values contained in a matrix are represented as colors.";
this.infoURL = "https://wikipedia.org/wiki/Heat_map";
this.inputType = "string";
this.outputType = "html";
this.args = [
{
name: "Record delimiter",
type: "option",
value: RECORD_DELIMITER_OPTIONS,
},
{
name: "Field delimiter",
type: "option",
value: FIELD_DELIMITER_OPTIONS,
},
{
name: "Number of vertical bins",
type: "number",
value: 25,
},
{
name: "Number of horizontal bins",
type: "number",
value: 25,
},
{
name: "Use column headers as labels",
type: "boolean",
value: true,
},
{
name: "X label",
type: "string",
value: "",
},
{
name: "Y label",
type: "string",
value: "",
},
{
name: "Draw bin edges",
type: "boolean",
value: false,
},
{
name: "Min colour value",
type: "string",
value: COLOURS.min,
},
{
name: "Max colour value",
type: "string",
value: COLOURS.max,
},
];
}
/**
* Heatmap chart operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {html}
*/
run(input, args) {
const recordDelimiter = Utils.charRep(args[0]),
fieldDelimiter = Utils.charRep(args[1]),
vBins = args[2],
hBins = args[3],
columnHeadingsAreIncluded = args[4],
drawEdges = args[7],
minColour = args[8],
maxColour = args[9],
dimension = 500;
if (vBins <= 0) throw new OperationError("Number of vertical bins must be greater than 0");
if (hBins <= 0) throw new OperationError("Number of horizontal bins must be greater than 0");
let xLabel = args[5],
yLabel = args[6];
const { headings, values } = getScatterValues(
input,
recordDelimiter,
fieldDelimiter,
columnHeadingsAreIncluded
);
if (headings) {
xLabel = headings.x;
yLabel = headings.y;
}
const document = new nodom.Document();
let svg = document.createElement("svg");
svg = d3.select(svg)
.attr("width", "100%")
.attr("height", "100%")
.attr("viewBox", `0 0 ${dimension} ${dimension}`);
const margin = {
top: 10,
right: 0,
bottom: 40,
left: 30,
},
width = dimension - margin.left - margin.right,
height = dimension - margin.top - margin.bottom,
binWidth = width / hBins,
binHeight = height/ vBins,
marginedSpace = svg.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
const bins = this.getHeatmapPacking(values, vBins, hBins),
maxCount = Math.max(...bins.map(row => {
const lengths = row.map(cell => cell.length);
return Math.max(...lengths);
}));
const xExtent = d3.extent(values, d => d[0]),
yExtent = d3.extent(values, d => d[1]);
const xAxis = d3.scaleLinear()
.domain(xExtent)
.range([0, width]);
const yAxis = d3.scaleLinear()
.domain(yExtent)
.range([height, 0]);
const colour = d3.scaleSequential(d3.interpolateLab(minColour, maxColour))
.domain([0, maxCount]);
marginedSpace.append("clipPath")
.attr("id", "clip")
.append("rect")
.attr("width", width)
.attr("height", height);
marginedSpace.append("g")
.attr("class", "bins")
.attr("clip-path", "url(#clip)")
.selectAll("g")
.data(bins)
.enter()
.append("g")
.selectAll("rect")
.data(d => d)
.enter()
.append("rect")
.attr("x", (d) => binWidth * d.x)
.attr("y", (d) => (height - binHeight * (d.y + 1)))
.attr("width", binWidth)
.attr("height", binHeight)
.attr("fill", (d) => colour(d.length))
.attr("stroke", drawEdges ? "rgba(0, 0, 0, 0.5)" : "none")
.attr("stroke-width", drawEdges ? "0.5" : "none")
.append("title")
.text(d => {
const count = d.length,
perc = 100.0 * d.length / values.length,
tooltip = `Count: ${count}\n
Percentage: ${perc.toFixed(2)}%\n
`.replace(/\s{2,}/g, "\n");
return tooltip;
});
marginedSpace.append("g")
.attr("class", "axis axis--y")
.call(d3.axisLeft(yAxis).tickSizeOuter(-width));
svg.append("text")
.attr("transform", "rotate(-90)")
.attr("y", -margin.left)
.attr("x", -(height / 2))
.attr("dy", "1em")
.style("text-anchor", "middle")
.text(yLabel);
marginedSpace.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(xAxis).tickSizeOuter(-height));
svg.append("text")
.attr("x", width / 2)
.attr("y", dimension)
.style("text-anchor", "middle")
.text(xLabel);
return svg._groups[0][0].outerHTML;
}
/**
* Packs a list of x, y coordinates into a number of bins for use in a heatmap.
*
* @param {Object[]} points
* @param {number} number of vertical bins
* @param {number} number of horizontal bins
* @returns {Object[]} a list of bins (each bin is an Array) with x y coordinates, filled with the points
*/
getHeatmapPacking(values, vBins, hBins) {
const xBounds = d3.extent(values, d => d[0]),
yBounds = d3.extent(values, d => d[1]),
bins = [];
if (xBounds[0] === xBounds[1]) throw "Cannot pack points. There is no difference between the minimum and maximum X coordinate.";
if (yBounds[0] === yBounds[1]) throw "Cannot pack points. There is no difference between the minimum and maximum Y coordinate.";
for (let y = 0; y < vBins; y++) {
bins.push([]);
for (let x = 0; x < hBins; x++) {
const item = [];
item.y = y;
item.x = x;
bins[y].push(item);
} // x
} // y
const epsilon = 0.000000001; // This is to clamp values that are exactly the maximum;
values.forEach(v => {
const fractionOfY = (v[1] - yBounds[0]) / ((yBounds[1] + epsilon) - yBounds[0]),
fractionOfX = (v[0] - xBounds[0]) / ((xBounds[1] + epsilon) - xBounds[0]),
y = Math.floor(vBins * fractionOfY),
x = Math.floor(hBins * fractionOfX);
bins[y][x].push({x: v[0], y: v[1]});
});
return bins;
}
}
export default HeatmapChart;

296
src/core/operations/HexDensityChart.mjs Normal file
View File

@ -0,0 +1,296 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import * as d3temp from "d3";
import * as d3hexbintemp from "d3-hexbin";
import * as nodomtemp from "nodom";
import { getScatterValues, RECORD_DELIMITER_OPTIONS, COLOURS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
import Operation from "../Operation";
import Utils from "../Utils";
const d3 = d3temp.default ? d3temp.default : d3temp;
const d3hexbin = d3hexbintemp.default ? d3hexbintemp.default : d3hexbintemp;
const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
/**
* Hex Density chart operation
*/
class HexDensityChart extends Operation {
/**
* HexDensityChart constructor
*/
constructor() {
super();
this.name = "Hex Density chart";
this.module = "Charts";
this.description = "Hex density charts are used in a similar way to scatter charts, however rather than rendering tens of thousands of points, it groups the points into a few hundred hexagons to show the distribution.";
this.inputType = "string";
this.outputType = "html";
this.args = [
{
name: "Record delimiter",
type: "option",
value: RECORD_DELIMITER_OPTIONS,
},
{
name: "Field delimiter",
type: "option",
value: FIELD_DELIMITER_OPTIONS,
},
{
name: "Pack radius",
type: "number",
value: 25,
},
{
name: "Draw radius",
type: "number",
value: 15,
},
{
name: "Use column headers as labels",
type: "boolean",
value: true,
},
{
name: "X label",
type: "string",
value: "",
},
{
name: "Y label",
type: "string",
value: "",
},
{
name: "Draw hexagon edges",
type: "boolean",
value: false,
},
{
name: "Min colour value",
type: "string",
value: COLOURS.min,
},
{
name: "Max colour value",
type: "string",
value: COLOURS.max,
},
{
name: "Draw empty hexagons within data boundaries",
type: "boolean",
value: false,
}
];
}
/**
* Hex Bin chart operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {html}
*/
run(input, args) {
const recordDelimiter = Utils.charRep(args[0]),
fieldDelimiter = Utils.charRep(args[1]),
packRadius = args[2],
drawRadius = args[3],
columnHeadingsAreIncluded = args[4],
drawEdges = args[7],
minColour = args[8],
maxColour = args[9],
drawEmptyHexagons = args[10],
dimension = 500;
let xLabel = args[5],
yLabel = args[6];
const { headings, values } = getScatterValues(
input,
recordDelimiter,
fieldDelimiter,
columnHeadingsAreIncluded
);
if (headings) {
xLabel = headings.x;
yLabel = headings.y;
}
const document = new nodom.Document();
let svg = document.createElement("svg");
svg = d3.select(svg)
.attr("width", "100%")
.attr("height", "100%")
.attr("viewBox", `0 0 ${dimension} ${dimension}`);
const margin = {
top: 10,
right: 0,
bottom: 40,
left: 30,
},
width = dimension - margin.left - margin.right,
height = dimension - margin.top - margin.bottom,
marginedSpace = svg.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
const hexbin = d3hexbin.hexbin()
.radius(packRadius)
.extent([0, 0], [width, height]);
const hexPoints = hexbin(values),
maxCount = Math.max(...hexPoints.map(b => b.length));
const xExtent = d3.extent(hexPoints, d => d.x),
yExtent = d3.extent(hexPoints, d => d.y);
xExtent[0] -= 2 * packRadius;
xExtent[1] += 3 * packRadius;
yExtent[0] -= 2 * packRadius;
yExtent[1] += 2 * packRadius;
const xAxis = d3.scaleLinear()
.domain(xExtent)
.range([0, width]);
const yAxis = d3.scaleLinear()
.domain(yExtent)
.range([height, 0]);
const colour = d3.scaleSequential(d3.interpolateLab(minColour, maxColour))
.domain([0, maxCount]);
marginedSpace.append("clipPath")
.attr("id", "clip")
.append("rect")
.attr("width", width)
.attr("height", height);
if (drawEmptyHexagons) {
marginedSpace.append("g")
.attr("class", "empty-hexagon")
.selectAll("path")
.data(this.getEmptyHexagons(hexPoints, packRadius))
.enter()
.append("path")
.attr("d", d => {
return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(drawRadius)}`;
})
.attr("fill", (d) => colour(0))
.attr("stroke", drawEdges ? "black" : "none")
.attr("stroke-width", drawEdges ? "0.5" : "none")
.append("title")
.text(d => {
const count = 0,
perc = 0,
tooltip = `Count: ${count}\n
Percentage: ${perc.toFixed(2)}%\n
Center: ${d.x.toFixed(2)}, ${d.y.toFixed(2)}\n
`.replace(/\s{2,}/g, "\n");
return tooltip;
});
}
marginedSpace.append("g")
.attr("class", "hexagon")
.attr("clip-path", "url(#clip)")
.selectAll("path")
.data(hexPoints)
.enter()
.append("path")
.attr("d", d => {
return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(drawRadius)}`;
})
.attr("fill", (d) => colour(d.length))
.attr("stroke", drawEdges ? "black" : "none")
.attr("stroke-width", drawEdges ? "0.5" : "none")
.append("title")
.text(d => {
const count = d.length,
perc = 100.0 * d.length / values.length,
CX = d.x,
CY = d.y,
xMin = Math.min(...d.map(d => d[0])),
xMax = Math.max(...d.map(d => d[0])),
yMin = Math.min(...d.map(d => d[1])),
yMax = Math.max(...d.map(d => d[1])),
tooltip = `Count: ${count}\n
Percentage: ${perc.toFixed(2)}%\n
Center: ${CX.toFixed(2)}, ${CY.toFixed(2)}\n
Min X: ${xMin.toFixed(2)}\n
Max X: ${xMax.toFixed(2)}\n
Min Y: ${yMin.toFixed(2)}\n
Max Y: ${yMax.toFixed(2)}
`.replace(/\s{2,}/g, "\n");
return tooltip;
});
marginedSpace.append("g")
.attr("class", "axis axis--y")
.call(d3.axisLeft(yAxis).tickSizeOuter(-width));
svg.append("text")
.attr("transform", "rotate(-90)")
.attr("y", -margin.left)
.attr("x", -(height / 2))
.attr("dy", "1em")
.style("text-anchor", "middle")
.text(yLabel);
marginedSpace.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(xAxis).tickSizeOuter(-height));
svg.append("text")
.attr("x", width / 2)
.attr("y", dimension)
.style("text-anchor", "middle")
.text(xLabel);
return svg._groups[0][0].outerHTML;
}
/**
* Hex Bin chart operation.
*
* @param {Object[]} - centres
* @param {number} - radius
* @returns {Object[]}
*/
getEmptyHexagons(centres, radius) {
const emptyCentres = [],
boundingRect = [d3.extent(centres, d => d.x), d3.extent(centres, d => d.y)],
hexagonCenterToEdge = Math.cos(2 * Math.PI / 12) * radius,
hexagonEdgeLength = Math.sin(2 * Math.PI / 12) * radius;
let indent = false;
for (let y = boundingRect[1][0]; y <= boundingRect[1][1] + radius; y += hexagonEdgeLength + radius) {
for (let x = boundingRect[0][0]; x <= boundingRect[0][1] + radius; x += 2 * hexagonCenterToEdge) {
let cx = x;
const cy = y;
if (indent && x >= boundingRect[0][1]) break;
if (indent) cx += hexagonCenterToEdge;
emptyCentres.push({x: cx, y: cy});
}
indent = !indent;
}
return emptyCentres;
}
}
export default HexDensityChart;

2
src/core/operations/JavaScriptParser.mjs
View File

@ -21,7 +21,7 @@ class JavaScriptParser extends Operation {
this.name = "JavaScript Parser"; this.name = "JavaScript Parser";
this.module = "Code"; this.module = "Code";
this.description = "Returns an Abstract Syntax Tree for valid JavaScript code."; this.description = "Returns an Abstract Syntax Tree for valid JavaScript code.";
this.infoURL = "https://en.wikipedia.org/wiki/Abstract_syntax_tree"; this.infoURL = "https://wikipedia.org/wiki/Abstract_syntax_tree";
this.inputType = "string"; this.inputType = "string";
this.outputType = "string"; this.outputType = "string";
this.args = [ this.args = [

2
src/core/operations/PEMToHex.mjs
View File

@ -21,7 +21,7 @@ class PEMToHex extends Operation {
this.name = "PEM to Hex"; this.name = "PEM to Hex";
this.module = "PublicKey"; this.module = "PublicKey";
this.description = "Converts PEM (Privacy Enhanced Mail) format to a hexadecimal DER (Distinguished Encoding Rules) string."; this.description = "Converts PEM (Privacy Enhanced Mail) format to a hexadecimal DER (Distinguished Encoding Rules) string.";
this.infoURL = "https://en.wikipedia.org/wiki/X.690#DER_encoding"; this.infoURL = "https://wikipedia.org/wiki/X.690#DER_encoding";
this.inputType = "string"; this.inputType = "string";
this.outputType = "string"; this.outputType = "string";
this.args = []; this.args = [];

46
src/core/operations/ProtobufDecode.mjs Normal file
View File

@ -0,0 +1,46 @@
/**
* @author GCHQ Contributor [3]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Protobuf from "../lib/Protobuf";
/**
* Protobuf Decode operation
*/
class ProtobufDecode extends Operation {
/**
* ProtobufDecode constructor
*/
constructor() {
super();
this.name = "Protobuf Decode";
this.module = "Default";
this.description = "Decodes any Protobuf encoded data to a JSON representation of the data using the field number as the field key.";
this.infoURL = "https://wikipedia.org/wiki/Protocol_Buffers";
this.inputType = "byteArray";
this.outputType = "JSON";
this.args = [];
}
/**
* @param {byteArray} input
* @param {Object[]} args
* @returns {JSON}
*/
run(input, args) {
try {
return Protobuf.decode(input);
} catch (err) {
throw new OperationError(err);
}
}
}
export default ProtobufDecode;

11
src/core/operations/RegularExpression.mjs
View File

@ -230,6 +230,7 @@ function regexHighlight (input, regex, displayTotal) {
title = "", title = "",
hl = 1, hl = 1,
total = 0; total = 0;
const captureGroups = [];
output = input.replace(regex, (match, ...args) => { output = input.replace(regex, (match, ...args) => {
args.pop(); // Throw away full string args.pop(); // Throw away full string
@ -247,9 +248,15 @@ function regexHighlight (input, regex, displayTotal) {
// Switch highlight // Switch highlight
hl = hl === 1 ? 2 : 1; hl = hl === 1 ? 2 : 1;
total++; // Store highlighted match and replace with a placeholder
captureGroups.push(`<span class='hl${hl}' title='${title}'>${Utils.escapeHtml(match)}</span>`);
return `[cc_capture_group_${total++}]`;
});
return `<span class='hl${hl}' title='${title}'>${Utils.escapeHtml(match)}</span>`; // Safely escape all remaining text, then replace placeholders
output = Utils.escapeHtml(output);
output = output.replace(/\[cc_capture_group_(\d+)\]/g, (_, i) => {
return captureGroups[i];
}); });
if (displayTotal) if (displayTotal)

199
src/core/operations/ScatterChart.mjs Normal file
View File

@ -0,0 +1,199 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import * as d3temp from "d3";
import * as nodomtemp from "nodom";
import { getScatterValues, getScatterValuesWithColour, RECORD_DELIMITER_OPTIONS, COLOURS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
import Operation from "../Operation";
import Utils from "../Utils";
const d3 = d3temp.default ? d3temp.default : d3temp;
const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
/**
* Scatter chart operation
*/
class ScatterChart extends Operation {
/**
* ScatterChart constructor
*/
constructor() {
super();
this.name = "Scatter chart";
this.module = "Charts";
this.description = "Plots two-variable data as single points on a graph.";
this.infoURL = "https://wikipedia.org/wiki/Scatter_plot";
this.inputType = "string";
this.outputType = "html";
this.args = [
{
name: "Record delimiter",
type: "option",
value: RECORD_DELIMITER_OPTIONS,
},
{
name: "Field delimiter",
type: "option",
value: FIELD_DELIMITER_OPTIONS,
},
{
name: "Use column headers as labels",
type: "boolean",
value: true,
},
{
name: "X label",
type: "string",
value: "",
},
{
name: "Y label",
type: "string",
value: "",
},
{
name: "Colour",
type: "string",
value: COLOURS.max,
},
{
name: "Point radius",
type: "number",
value: 10,
},
{
name: "Use colour from third column",
type: "boolean",
value: false,
}
];
}
/**
* Scatter chart operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {html}
*/
run(input, args) {
const recordDelimiter = Utils.charRep(args[0]),
fieldDelimiter = Utils.charRep(args[1]),
columnHeadingsAreIncluded = args[2],
fillColour = args[5],
radius = args[6],
colourInInput = args[7],
dimension = 500;
let xLabel = args[3],
yLabel = args[4];
const dataFunction = colourInInput ? getScatterValuesWithColour : getScatterValues;
const { headings, values } = dataFunction(
input,
recordDelimiter,
fieldDelimiter,
columnHeadingsAreIncluded
);
if (headings) {
xLabel = headings.x;
yLabel = headings.y;
}
const document = new nodom.Document();
let svg = document.createElement("svg");
svg = d3.select(svg)
.attr("width", "100%")
.attr("height", "100%")
.attr("viewBox", `0 0 ${dimension} ${dimension}`);
const margin = {
top: 10,
right: 0,
bottom: 40,
left: 30,
},
width = dimension - margin.left - margin.right,
height = dimension - margin.top - margin.bottom,
marginedSpace = svg.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
const xExtent = d3.extent(values, d => d[0]),
xDelta = xExtent[1] - xExtent[0],
yExtent = d3.extent(values, d => d[1]),
yDelta = yExtent[1] - yExtent[0],
xAxis = d3.scaleLinear()
.domain([xExtent[0] - (0.1 * xDelta), xExtent[1] + (0.1 * xDelta)])
.range([0, width]),
yAxis = d3.scaleLinear()
.domain([yExtent[0] - (0.1 * yDelta), yExtent[1] + (0.1 * yDelta)])
.range([height, 0]);
marginedSpace.append("clipPath")
.attr("id", "clip")
.append("rect")
.attr("width", width)
.attr("height", height);
marginedSpace.append("g")
.attr("class", "points")
.attr("clip-path", "url(#clip)")
.selectAll("circle")
.data(values)
.enter()
.append("circle")
.attr("cx", (d) => xAxis(d[0]))
.attr("cy", (d) => yAxis(d[1]))
.attr("r", d => radius)
.attr("fill", d => {
return colourInInput ? d[2] : fillColour;
})
.attr("stroke", "rgba(0, 0, 0, 0.5)")
.attr("stroke-width", "0.5")
.append("title")
.text(d => {
const x = d[0],
y = d[1],
tooltip = `X: ${x}\n
Y: ${y}\n
`.replace(/\s{2,}/g, "\n");
return tooltip;
});
marginedSpace.append("g")
.attr("class", "axis axis--y")
.call(d3.axisLeft(yAxis).tickSizeOuter(-width));
svg.append("text")
.attr("transform", "rotate(-90)")
.attr("y", -margin.left)
.attr("x", -(height / 2))
.attr("dy", "1em")
.style("text-anchor", "middle")
.text(yLabel);
marginedSpace.append("g")
.attr("class", "axis axis--x")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(xAxis).tickSizeOuter(-height));
svg.append("text")
.attr("x", width / 2)
.attr("y", dimension)
.style("text-anchor", "middle")
.text(xLabel);
return svg._groups[0][0].outerHTML;
}
}
export default ScatterChart;

227
src/core/operations/SeriesChart.mjs Normal file
View File

@ -0,0 +1,227 @@
/**
* @author tlwr [toby@toby.codes]
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import * as d3temp from "d3";
import * as nodomtemp from "nodom";
import { getSeriesValues, RECORD_DELIMITER_OPTIONS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
import Operation from "../Operation";
import Utils from "../Utils";
const d3 = d3temp.default ? d3temp.default : d3temp;
const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
/**
* Series chart operation
*/
class SeriesChart extends Operation {
/**
* SeriesChart constructor
*/
constructor() {
super();
this.name = "Series chart";
this.module = "Charts";
this.description = "A time series graph is a line graph of repeated measurements taken over regular time intervals.";
this.inputType = "string";
this.outputType = "html";
this.args = [
{
name: "Record delimiter",
type: "option",
value: RECORD_DELIMITER_OPTIONS,
},
{
name: "Field delimiter",
type: "option",
value: FIELD_DELIMITER_OPTIONS,
},
{
name: "X label",
type: "string",
value: "",
},
{
name: "Point radius",
type: "number",
value: 1,
},
{
name: "Series colours",
type: "string",
value: "mediumseagreen, dodgerblue, tomato",
},
];
}
/**
* Series chart operation.
*
* @param {string} input
* @param {Object[]} args
* @returns {html}
*/
run(input, args) {
const recordDelimiter = Utils.charRep(args[0]),
fieldDelimiter = Utils.charRep(args[1]),
xLabel = args[2],
pipRadius = args[3],
seriesColours = args[4].split(","),
svgWidth = 500,
interSeriesPadding = 20,
xAxisHeight = 50,
seriesLabelWidth = 50,
seriesHeight = 100,
seriesWidth = svgWidth - seriesLabelWidth - interSeriesPadding;
const { xValues, series } = getSeriesValues(input, recordDelimiter, fieldDelimiter),
allSeriesHeight = Object.keys(series).length * (interSeriesPadding + seriesHeight),
svgHeight = allSeriesHeight + xAxisHeight + interSeriesPadding;
const document = new nodom.Document();
let svg = document.createElement("svg");
svg = d3.select(svg)
.attr("width", "100%")
.attr("height", "100%")
.attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
const xAxis = d3.scalePoint()
.domain(xValues)
.range([0, seriesWidth]);
svg.append("g")
.attr("class", "axis axis--x")
.attr("transform", `translate(${seriesLabelWidth}, ${xAxisHeight})`)
.call(
d3.axisTop(xAxis).tickValues(xValues.filter((x, i) => {
return [0, Math.round(xValues.length / 2), xValues.length -1].indexOf(i) >= 0;
}))
);
svg.append("text")
.attr("x", svgWidth / 2)
.attr("y", xAxisHeight / 2)
.style("text-anchor", "middle")
.text(xLabel);
const tooltipText = {},
tooltipAreaWidth = seriesWidth / xValues.length;
xValues.forEach(x => {
const tooltip = [];
series.forEach(serie => {
const y = serie.data[x];
if (typeof y === "undefined") return;
tooltip.push(`${serie.name}: ${y}`);
});
tooltipText[x] = tooltip.join("\n");
});
const chartArea = svg.append("g")
.attr("transform", `translate(${seriesLabelWidth}, ${xAxisHeight})`);
chartArea
.append("g")
.selectAll("rect")
.data(xValues)
.enter()
.append("rect")
.attr("x", x => {
return xAxis(x) - (tooltipAreaWidth / 2);
})
.attr("y", 0)
.attr("width", tooltipAreaWidth)
.attr("height", allSeriesHeight)
.attr("stroke", "none")
.attr("fill", "transparent")
.append("title")
.text(x => {
return `${x}\n
--\n
${tooltipText[x]}\n
`.replace(/\s{2,}/g, "\n");
});
const yAxesArea = svg.append("g")
.attr("transform", `translate(0, ${xAxisHeight})`);
series.forEach((serie, seriesIndex) => {
const yExtent = d3.extent(Object.values(serie.data)),
yAxis = d3.scaleLinear()
.domain(yExtent)
.range([seriesHeight, 0]);
const seriesGroup = chartArea
.append("g")
.attr("transform", `translate(0, ${seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`);
let path = "";
xValues.forEach((x, xIndex) => {
let nextX = xValues[xIndex + 1],
y = serie.data[x],
nextY= serie.data[nextX];
if (typeof y === "undefined" || typeof nextY === "undefined") return;
x = xAxis(x); nextX = xAxis(nextX);
y = yAxis(y); nextY = yAxis(nextY);
path += `M ${x} ${y} L ${nextX} ${nextY} z `;
});
seriesGroup
.append("path")
.attr("d", path)
.attr("fill", "none")
.attr("stroke", seriesColours[seriesIndex % seriesColours.length])
.attr("stroke-width", "1");
xValues.forEach(x => {
const y = serie.data[x];
if (typeof y === "undefined") return;
seriesGroup
.append("circle")
.attr("cx", xAxis(x))
.attr("cy", yAxis(y))
.attr("r", pipRadius)
.attr("fill", seriesColours[seriesIndex % seriesColours.length])
.append("title")
.text(d => {
return `${x}\n
--\n
${tooltipText[x]}\n
`.replace(/\s{2,}/g, "\n");
});
});
yAxesArea
.append("g")
.attr("transform", `translate(${seriesLabelWidth - interSeriesPadding}, ${seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`)
.attr("class", "axis axis--y")
.call(d3.axisLeft(yAxis).ticks(5));
yAxesArea
.append("g")
.attr("transform", `translate(0, ${seriesHeight / 2 + seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`)
.append("text")
.style("text-anchor", "middle")
.attr("transform", "rotate(-90)")
.text(serie.name);
});
return svg._groups[0][0].outerHTML;
}
}
export default SeriesChart;

2
src/core/operations/TextEncodingBruteForce.mjs
View File

@ -79,7 +79,7 @@ class TextEncodingBruteForce extends Operation {
let table = "<table class='table table-hover table-sm table-bordered table-nonfluid'><tr><th>Encoding</th><th>Value</th></tr>"; let table = "<table class='table table-hover table-sm table-bordered table-nonfluid'><tr><th>Encoding</th><th>Value</th></tr>";
for (const enc in encodings) { for (const enc in encodings) {
const value = Utils.printable(encodings[enc], true); const value = Utils.escapeHtml(Utils.printable(encodings[enc], true));
table += `<tr><td>${enc}</td><td>${value}</td></tr>`; table += `<tr><td>${enc}</td><td>${value}</td></tr>`;
} }

46
src/core/operations/VarIntDecode.mjs Normal file
View File

@ -0,0 +1,46 @@
/**
* @author GCHQ Contributor [3]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Protobuf from "../lib/Protobuf";
/**
* VarInt Decode operation
*/
class VarIntDecode extends Operation {
/**
* VarIntDecode constructor
*/
constructor() {
super();
this.name = "VarInt Decode";
this.module = "Default";
this.description = "Decodes a VarInt encoded integer. VarInt is an efficient way of encoding variable length integers and is commonly used with Protobuf.";
this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding#varints";
this.inputType = "byteArray";
this.outputType = "number";
this.args = [];
}
/**
* @param {byteArray} input
* @param {Object[]} args
* @returns {number}
*/
run(input, args) {
try {
return Protobuf.varIntDecode(input);
} catch (err) {
throw new OperationError(err);
}
}
}
export default VarIntDecode;

46
src/core/operations/VarIntEncode.mjs Normal file
View File

@ -0,0 +1,46 @@
/**
* @author GCHQ Contributor [3]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import Operation from "../Operation";
import OperationError from "../errors/OperationError";
import Protobuf from "../lib/Protobuf";
/**
* VarInt Encode operation
*/
class VarIntEncode extends Operation {
/**
* VarIntEncode constructor
*/
constructor() {
super();
this.name = "VarInt Encode";
this.module = "Default";
this.description = "Encodes a Vn integer as a VarInt. VarInt is an efficient way of encoding variable length integers and is commonly used with Protobuf.";
this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding#varints";
this.inputType = "number";
this.outputType = "byteArray";
this.args = [];
}
/**
* @param {number} input
* @param {Object[]} args
* @returns {byteArray}
*/
run(input, args) {
try {
return Protobuf.varIntEncode(input);
} catch (err) {
throw new OperationError(err);
}
}
}
export default VarIntEncode;

1
src/node/index.mjs
View File

@ -5,7 +5,6 @@
* @copyright Crown Copyright 2017 * @copyright Crown Copyright 2017
* @license Apache-2.0 * @license Apache-2.0
*/ */
import "babel-polyfill";
// Define global environment functions // Define global environment functions
global.ENVIRONMENT_IS_WORKER = function() { global.ENVIRONMENT_IS_WORKER = function() {

2
src/web/ControlsWaiter.mjs
View File

@ -338,7 +338,7 @@ class ControlsWaiter {
const saveLink = this.generateStateUrl(true, true, null, "https://gchq.github.io/CyberChef/"); const saveLink = this.generateStateUrl(true, true, null, "https://gchq.github.io/CyberChef/");
if (reportBugInfo) { if (reportBugInfo) {
reportBugInfo.innerHTML = `* Version: ${PKG_VERSION + (typeof INLINE === "undefined" ? "" : "s")} reportBugInfo.innerHTML = `* Version: ${PKG_VERSION}
* Compile time: ${COMPILE_TIME} * Compile time: ${COMPILE_TIME}
* User-Agent: * User-Agent:
${navigator.userAgent} ${navigator.userAgent}

293
src/web/SeasonalWaiter.mjs
View File

@ -4,6 +4,10 @@
* @license Apache-2.0 * @license Apache-2.0
*/ */
import clippy from "clippyjs";
import "./static/clippy_assets/agents/Clippy/agent.js";
import clippyMap from "./static/clippy_assets/agents/Clippy/map.png";
/** /**
* Waiter to handle seasonal events and easter eggs. * Waiter to handle seasonal events and easter eggs.
*/ */
@ -18,6 +22,8 @@ class SeasonalWaiter {
constructor(app, manager) { constructor(app, manager) {
this.app = app; this.app = app;
this.manager = manager; this.manager = manager;
this.clippyAgent = null;
} }
@ -28,6 +34,14 @@ class SeasonalWaiter {
// Konami code // Konami code
this.kkeys = []; this.kkeys = [];
window.addEventListener("keydown", this.konamiCodeListener.bind(this)); window.addEventListener("keydown", this.konamiCodeListener.bind(this));
// Clippy
const now = new Date();
if (now.getMonth() === 3 && now.getDate() === 1) {
this.addClippyOption();
this.manager.addDynamicListener(".option-item #clippy", "change", this.setupClippy, this);
this.setupClippy();
}
} }
@ -51,6 +65,285 @@ class SeasonalWaiter {
} }
} }
/**
* Creates an option in the Options menu for turning Clippy on or off
*/
addClippyOption() {
const optionsBody = document.getElementById("options-body"),
optionItem = document.createElement("span");
optionItem.className = "bmd-form-group is-filled";
optionItem.innerHTML = `<div class="checkbox option-item">
<label for="clippy">
<input type="checkbox" option="clippy" id="clippy" checked="">
Use the Clippy helper
</label>
</div>`;
optionsBody.appendChild(optionItem);
if (!this.app.options.hasOwnProperty("clippy")) {
this.app.options.clippy = true;
}
this.manager.options.load();
}
/**
* Sets up Clippy for April Fools Day
*/
setupClippy() {
// Destroy any previous agents
if (this.clippyAgent) {
this.clippyAgent.closeBalloonImmediately();
this.clippyAgent.hide();
}
if (!this.app.options.clippy) {
if (this.clippyTimeouts) this.clippyTimeouts.forEach(t => clearTimeout(t));
return;
}
// Set base path to # to prevent external network requests
const clippyAssets = "#";
// Shim the library to prevent external network requests
shimClippy(clippy);
const self = this;
clippy.load("Clippy", (agent) => {
shimClippyAgent(agent);
self.clippyAgent = agent;
agent.show();
agent.speak("Hello, I'm Clippy, your personal cyber assistant!");
}, undefined, clippyAssets);
// Watch for the Auto Magic button appearing
const magic = document.getElementById("magic");
const observer = new MutationObserver((mutationsList, observer) => {
// Read in message and recipe
let msg, recipe;
for (const mutation of mutationsList) {
if (mutation.attributeName === "data-original-title") {
msg = magic.getAttribute("data-original-title");
}
if (mutation.attributeName === "data-recipe") {
recipe = magic.getAttribute("data-recipe");
}
}
// Close balloon if it is currently showing a magic hint
const balloon = self.clippyAgent._balloon._balloon;
if (balloon.is(":visible") && balloon.text().indexOf("That looks like encoded data") >= 0) {
self.clippyAgent._balloon.hide(true);
this.clippyAgent._balloon._hidden = true;
}
// If a recipe was found, get Clippy to tell the user
if (recipe) {
recipe = this.manager.controls.generateStateUrl(true, true, JSON.parse(recipe));
msg = `That looks like encoded data!<br><br>${msg}<br><br>Click <a class="clippyMagicRecipe" href="${recipe}">here</a> to load this recipe.`;
// Stop current balloon activity immediately and trigger speak again
this.clippyAgent.closeBalloonImmediately();
self.clippyAgent.speak(msg, true);
// self.clippyAgent._queue.next();
}
});
observer.observe(document.getElementById("magic"), {attributes: true});
// Play animations for various things
this.manager.addListeners("#search", "click", () => {
this.clippyAgent.play("Searching");
}, this);
this.manager.addListeners("#save,#save-to-file", "click", () => {
this.clippyAgent.play("Save");
}, this);
this.manager.addListeners("#clr-recipe,#clr-io", "click", () => {
this.clippyAgent.play("EmptyTrash");
}, this);
this.manager.addListeners("#bake", "click", e => {
if (e.target.closest("button").textContent.toLowerCase().indexOf("bake") >= 0) {
this.clippyAgent.play("Thinking");
} else {
this.clippyAgent.play("EmptyTrash");
}
this.clippyAgent._queue.clear();
}, this);
this.manager.addListeners("#input-text", "keydown", () => {
this.clippyAgent.play("Writing");
this.clippyAgent._queue.clear();
}, this);
this.manager.addDynamicListener("a.clippyMagicRecipe", "click", (e) => {
this.clippyAgent.play("Congratulate");
}, this);
this.clippyTimeouts = [];
// Show challenge after timeout
this.clippyTimeouts.push(setTimeout(() => {
const hex = "1f 8b 08 00 ae a1 9b 5c 00 ff 05 40 a1 12 00 10 0c fd 26 61 5b 76 aa 9d 26 a8 02 02 37 84 f7 fb bb c5 a4 5f 22 c6 09 e5 6e c5 4c 2d 3f e9 30 a6 ea 41 a2 f2 ac 1c 00 00 00";
self.clippyAgent.speak(`How about a fun challenge?<br><br>Try decoding this (click to load):<br><a href="#recipe=[]&input=${encodeURIComponent(btoa(hex))}">${hex}</a>`, true);
self.clippyAgent.play("GetAttention");
}, 1 * 60 * 1000));
this.clippyTimeouts.push(setTimeout(() => {
self.clippyAgent.speak("<i>Did you know?</i><br><br>You can load files into CyberChef up to around 500MB using drag and drop or the load file button.", 15000);
self.clippyAgent.play("Wave");
}, 2 * 60 * 1000));
this.clippyTimeouts.push(setTimeout(() => {
self.clippyAgent.speak("<i>Did you know?</i><br><br>You can use the 'Fork' operation to split up your input and run the recipe over each branch separately.<br><br><a class='clippyMagicRecipe' href=\"#recipe=Fork('%5C%5Cn','%5C%5Cn',false)From_UNIX_Timestamp('Seconds%20(s)')&amp;input=OTc4MzQ2ODAwCjEwMTI2NTEyMDAKMTA0NjY5NjQwMAoxMDgxMDg3MjAwCjExMTUzMDUyMDAKMTE0OTYwOTYwMA\">Here's an example</a>.", 15000);
self.clippyAgent.play("Print");
}, 3 * 60 * 1000));
this.clippyTimeouts.push(setTimeout(() => {
self.clippyAgent.speak("<i>Did you know?</i><br><br>The 'Magic' operation uses a number of methods to detect encoded data and the operations which can be used to make sense of it. A technical description of these methods can be found <a href=\"https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic\">here</a>.", 15000);
self.clippyAgent.play("Alert");
}, 4 * 60 * 1000));
this.clippyTimeouts.push(setTimeout(() => {
self.clippyAgent.speak("<i>Did you know?</i><br><br>You can use parts of the input as arguments to operations.<br><br><a class='clippyMagicRecipe' href=\"#recipe=Register('key%3D(%5B%5C%5Cda-f%5D*)',true,false)Find_/_Replace(%7B'option':'Regex','string':'.*data%3D(.*)'%7D,'$1',true,false,true)RC4(%7B'option':'Hex','string':'$R0'%7D,'Hex','Latin1')&amp;input=aHR0cDovL21hbHdhcmV6LmJpei9iZWFjb24ucGhwP2tleT0wZTkzMmE1YyZkYXRhPThkYjdkNWViZTM4NjYzYTU0ZWNiYjMzNGUzZGIxMQ\">Click here for an example</a>.", 15000);
self.clippyAgent.play("CheckingSomething");
}, 5 * 60 * 1000));
}
}
/**
* Shims various ClippyJS functions to modify behaviour.
*
* @param {Clippy} clippy - The Clippy library
*/
function shimClippy(clippy) {
// Shim _loadSounds so that it doesn't actually try to load any sounds
clippy.load._loadSounds = function _loadSounds (name, path) {
let dfd = clippy.load._sounds[name];
if (dfd) return dfd;
// set dfd if not defined
dfd = clippy.load._sounds[name] = $.Deferred();
// Resolve immediately without loading
dfd.resolve({});
return dfd.promise();
};
// Shim _loadMap so that it uses the local copy
clippy.load._loadMap = function _loadMap (path) {
let dfd = clippy.load._maps[path];
if (dfd) return dfd;
// set dfd if not defined
dfd = clippy.load._maps[path] = $.Deferred();
const src = clippyMap;
const img = new Image();
img.onload = dfd.resolve;
img.onerror = dfd.reject;
// start loading the map;
img.setAttribute("src", src);
return dfd.promise();
};
// Make sure we don't request the remote map
clippy.Animator.prototype._setupElement = function _setupElement (el) {
const frameSize = this._data.framesize;
el.css("display", "none");
el.css({ width: frameSize[0], height: frameSize[1] });
el.css("background", "url('" + clippyMap + "') no-repeat");
return el;
};
}
/**
* Shims various ClippyJS Agent functions to modify behaviour.
*
* @param {Agent} agent - The Clippy Agent
*/
function shimClippyAgent(agent) {
// Turn off all sounds
agent._animator._playSound = () => {};
// Improve speak function to support HTML markup
const self = agent._balloon;
agent._balloon.speak = (complete, text, hold) => {
self._hidden = false;
self.show();
const c = self._content;
// set height to auto
c.height("auto");
c.width("auto");
// add the text
c.html(text);
// set height
c.height(c.height());
c.width(c.width());
c.text("");
self.reposition();
self._complete = complete;
self._sayWords(text, hold, complete);
if (hold) agent._queue.next();
};
// Improve the _sayWords function to allow HTML and support timeouts
agent._balloon.WORD_SPEAK_TIME = 60;
agent._balloon._sayWords = (text, hold, complete) => {
self._active = true;
self._hold = hold;
const words = text.split(/[^\S-]/);
const time = self.WORD_SPEAK_TIME;
const el = self._content;
let idx = 1;
clearTimeout(self.holdTimeout);
self._addWord = $.proxy(function () {
if (!self._active) return;
if (idx > words.length) {
delete self._addWord;
self._active = false;
if (!self._hold) {
complete();
self.hide();
} else if (typeof hold === "number") {
self.holdTimeout = setTimeout(() => {
self._hold = false;
complete();
self.hide();
}, hold);
}
} else {
el.html(words.slice(0, idx).join(" "));
idx++;
self._loop = window.setTimeout($.proxy(self._addWord, self), time);
}
}, self);
self._addWord();
};
// Add break-word to balloon CSS
agent._balloon._balloon.css("word-break", "break-word");
// Close the balloon on click (unless it was a link)
agent._balloon._balloon.click(e => {
if (e.target.nodeName !== "A") {
agent._balloon.hide(true);
agent._balloon._hidden = true;
}
});
// Add function to immediately close the balloon even if it is currently doing something
agent.closeBalloonImmediately = () => {
agent._queue.clear();
agent._balloon.hide(true);
agent._balloon._hidden = true;
agent._queue.next();
};
} }
export default SeasonalWaiter; export default SeasonalWaiter;

15
src/web/html/index.html
View File

@ -131,13 +131,6 @@
}; };
window.addEventListener("error", loadingErrorHandler); window.addEventListener("error", loadingErrorHandler);
</script> </script>
<% if (htmlWebpackPlugin.options.inline) { %>
<meta name="robots" content="noindex" />
<% } else { %>
<script type="application/ld+json">
<% print(JSON.stringify(require("../static/structuredData.json"))); %>
</script>
<% } %>
</head> </head>
<body> <body>
<!-- Preloader overlay --> <!-- Preloader overlay -->
@ -153,11 +146,7 @@
<div id="content-wrapper"> <div id="content-wrapper">
<div id="banner" class="row"> <div id="banner" class="row">
<div class="col" style="text-align: left; padding-left: 10px;"> <div class="col" style="text-align: left; padding-left: 10px;">
<% if (htmlWebpackPlugin.options.inline) { %> <a href="CyberChef_v<%= htmlWebpackPlugin.options.version %>.zip" download>Download CyberChef <i class="material-icons">file_download</i></a>
<span>Version <%= htmlWebpackPlugin.options.version %></span>
<% } else { %>
<a href="cyberchef.htm" download>Download CyberChef <i class="material-icons">file_download</i></a>
<% } %>
</div> </div>
<div class="col-md-6" id="notice-wrapper"> <div class="col-md-6" id="notice-wrapper">
<span id="notice"> <span id="notice">
@ -591,7 +580,7 @@
What sort of things can I do with CyberChef? What sort of things can I do with CyberChef?
</a> </a>
<div class="collapse" id="faq-examples"> <div class="collapse" id="faq-examples">
<p>There are around 200 operations in CyberChef allowing you to carry out simple and complex tasks easily. Here are some examples:</p> <p>There are around 300 operations in CyberChef allowing you to carry out simple and complex tasks easily. Here are some examples:</p>
<ul> <ul>
<li><a href="#recipe=From_Base64('A-Za-z0-9%2B/%3D',true)&input=VTI4Z2JHOXVaeUJoYm1RZ2RHaGhibXR6SUdadmNpQmhiR3dnZEdobElHWnBjMmd1">Decode a Base64-encoded string</a></li> <li><a href="#recipe=From_Base64('A-Za-z0-9%2B/%3D',true)&input=VTI4Z2JHOXVaeUJoYm1RZ2RHaGhibXR6SUdadmNpQmhiR3dnZEdobElHWnBjMmd1">Decode a Base64-encoded string</a></li>
<li><a href="#recipe=Translate_DateTime_Format('Standard%20date%20and%20time','DD/MM/YYYY%20HH:mm:ss','UTC','dddd%20Do%20MMMM%20YYYY%20HH:mm:ss%20Z%20z','Australia/Queensland')&input=MTUvMDYvMjAxNSAyMDo0NTowMA">Convert a date and time to a different time zone</a></li> <li><a href="#recipe=Translate_DateTime_Format('Standard%20date%20and%20time','DD/MM/YYYY%20HH:mm:ss','UTC','dddd%20Do%20MMMM%20YYYY%20HH:mm:ss%20Z%20z','Australia/Queensland')&input=MTUvMDYvMjAxNSAyMDo0NTowMA">Convert a date and time to a different time zone</a></li>

1
src/web/index.js
View File

@ -8,7 +8,6 @@
import "./stylesheets/index.js"; import "./stylesheets/index.js";
// Libs // Libs
import "babel-polyfill";
import "arrive"; import "arrive";
import "snackbarjs"; import "snackbarjs";
import "bootstrap-material-design"; import "bootstrap-material-design";

1
src/web/static/clippy_assets/agents/Clippy/agent.js Executable file
View File

File diff suppressed because one or more lines are too long

BIN
src/web/static/clippy_assets/agents/Clippy/map.png Executable file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.3 MiB

62
src/web/static/clippy_assets/clippy.css Executable file
View File

@ -0,0 +1,62 @@
.clippy, .clippy-balloon {
position: fixed;
z-index: 1000;
cursor: pointer;
}
.clippy-balloon {
background: #FFC;
color: black;
padding: 8px;
border: 1px solid black;
border-radius: 5px;
}
.clippy-content {
max-width: 200px;
min-width: 120px;
font-family: "Microsoft Sans", sans-serif;
font-size: 10pt;
}
.clippy-tip {
width: 10px;
height: 16px;
background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAgCAMAAAAlvKiEAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAAlQTFRF///MAAAA////52QwgAAAAAN0Uk5T//8A18oNQQAAAGxJREFUeNqs0kEOwCAIRFHn3//QTUU6xMyyxii+jQosrTPkyPEM6IN3FtzIRk1U4dFeKWQiH6pRRowMVKEmvronEynkwj0uZJgR22+YLopPSo9P34wJSamLSU7lSIWLJU7NkNomNlhqxUeAAQC+TQLZyEuJBwAAAABJRU5ErkJggg==) no-repeat;
position: absolute;
}
.clippy-top-left .clippy-tip {
top: 100%;
margin-top: 0px;
left: 100%;
margin-left: -50px;
}
.clippy-top-right .clippy-tip {
top: 100%;
margin-top: 0px;
left: 0;
margin-left: 50px;
background-position: -10px 0;
}
.clippy-bottom-right .clippy-tip {
top: 0;
margin-top: -16px;
left: 0;
margin-left: 50px;
background-position: -10px -16px;
}
.clippy-bottom-left .clippy-tip {
top: 0;
margin-top: -16px;
left: 100%;
margin-left: -50px;
background-position: 0px -16px;
}

BIN
src/web/static/clippy_assets/images/border.png Executable file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 229 B

BIN
src/web/static/clippy_assets/images/tip.png Executable file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 238 B

1
src/web/stylesheets/index.js
View File

@ -8,6 +8,7 @@
/* Libraries */ /* Libraries */
import "highlight.js/styles/vs.css"; import "highlight.js/styles/vs.css";
import "../static/clippy_assets/clippy.css";
/* Frameworks */ /* Frameworks */
import "./vendors/bootstrap.scss"; import "./vendors/bootstrap.scss";

5
tests/operations/index.mjs
View File

@ -10,7 +10,6 @@
* @copyright Crown Copyright 2017 * @copyright Crown Copyright 2017
* @license Apache-2.0 * @license Apache-2.0
*/ */
import "babel-polyfill";
// Define global environment functions // Define global environment functions
global.ENVIRONMENT_IS_WORKER = function() { global.ENVIRONMENT_IS_WORKER = function() {
@ -33,6 +32,7 @@ import "./tests/BitwiseOp";
import "./tests/ByteRepr"; import "./tests/ByteRepr";
import "./tests/CartesianProduct"; import "./tests/CartesianProduct";
import "./tests/CharEnc"; import "./tests/CharEnc";
import "./tests/Charts";
import "./tests/Checksum"; import "./tests/Checksum";
import "./tests/Ciphers"; import "./tests/Ciphers";
import "./tests/Code"; import "./tests/Code";
@ -87,6 +87,9 @@ import "./tests/Enigma";
import "./tests/Bombe"; import "./tests/Bombe";
import "./tests/MultipleBombe"; import "./tests/MultipleBombe";
import "./tests/Typex"; import "./tests/Typex";
import "./tests/BLAKE2b";
import "./tests/BLAKE2s";
import "./tests/Protobuf";
// Cannot test operations that use the File type yet // Cannot test operations that use the File type yet
//import "./tests/SplitColourChannels"; //import "./tests/SplitColourChannels";

56
tests/operations/tests/BLAKE2b.mjs Normal file
View File

@ -0,0 +1,56 @@
/**
* BitwiseOp tests
*
* @author h345983745
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import TestRegister from "../TestRegister";
TestRegister.addTests([
{
name: "BLAKE2b: 512 - Hello World",
input: "Hello World",
expectedOutput: "4386a08a265111c9896f56456e2cb61a64239115c4784cf438e36cc851221972da3fb0115f73cd02486254001f878ab1fd126aac69844ef1c1ca152379d0a9bd",
recipeConfig: [
{ "op": "BLAKE2b",
"args": ["512", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2b: 384 - Hello World",
input: "Hello World",
expectedOutput: "4d388e82ca8f866e606b6f6f0be910abd62ad6e98c0adfc27cf35acf948986d5c5b9c18b6f47261e1e679eb98edf8e2d",
recipeConfig: [
{ "op": "BLAKE2b",
"args": ["384", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2b: 256 - Hello World",
input: "Hello World",
expectedOutput: "1dc01772ee0171f5f614c673e3c7fa1107a8cf727bdf5a6dadb379e93c0d1d00",
recipeConfig: [
{ "op": "BLAKE2b",
"args": ["256", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2b: 160 - Hello World",
input: "Hello World",
expectedOutput: "6a8489e6fd6e51fae12ab271ec7fc8134dd5d737",
recipeConfig: [
{ "op": "BLAKE2b",
"args": ["160", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2b: Key Test",
input: "message data",
expectedOutput: "3d363ff7401e02026f4a4687d4863ced",
recipeConfig: [
{ "op": "BLAKE2b",
"args": ["128", "Hex", {string: "pseudorandom key", option: "UTF8"}] }
]
}
]);

47
tests/operations/tests/BLAKE2s.mjs Executable file
View File

@ -0,0 +1,47 @@
/**
* BitwiseOp tests
*
* @author h345983745
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import TestRegister from "../TestRegister";
TestRegister.addTests([
{
name: "BLAKE2s: 256 - Hello World",
input: "Hello World",
expectedOutput: "7706af019148849e516f95ba630307a2018bb7bf03803eca5ed7ed2c3c013513",
recipeConfig: [
{ "op": "BLAKE2s",
"args": ["256", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2s: 160 - Hello World",
input: "Hello World",
expectedOutput: "0e4fcfc2ee0097ac1d72d70b595a39e09a3c7c7e",
recipeConfig: [
{ "op": "BLAKE2s",
"args": ["160", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2s: 128 - Hello World",
input: "Hello World",
expectedOutput: "9964ee6f36126626bf864363edfa96f6",
recipeConfig: [
{ "op": "BLAKE2s",
"args": ["128", "Hex", {string: "", option: "UTF8"}] }
]
},
{
name: "BLAKE2s: Key Test",
input: "Hello World",
expectedOutput: "9964ee6f36126626bf864363edfa96f6",
recipeConfig: [
{ "op": "BLAKE2s",
"args": ["128", "Hex", {string: "", option: "UTF8"}] }
]
}
]);

55
tests/operations/tests/Charts.mjs Normal file
View File

@ -0,0 +1,55 @@
/**
* Chart tests.
*
* @author Matt C [me@mitt.dev]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import TestRegister from "../TestRegister";
TestRegister.addTests([
{
name: "Scatter chart",
input: "100 100\n200 200\n300 300\n400 400\n500 500",
expectedMatch: /^<svg width/,
recipeConfig: [
{
"op": "Scatter chart",
"args": ["Line feed", "Space", false, "time", "stress", "black", 5, false]
}
],
},
{
name: "Hex density chart",
input: "100 100\n200 200\n300 300\n400 400\n500 500",
expectedMatch: /^<svg width/,
recipeConfig: [
{
"op": "Hex Density chart",
"args": ["Line feed", "Space", 25, 15, true, "", "", true, "white", "black", true]
}
],
},
{
name: "Series chart",
input: "100 100 100\n200 200 200\n300 300 300\n400 400 400\n500 500 500",
expectedMatch: /^<svg width/,
recipeConfig: [
{
"op": "Series chart",
"args": ["Line feed", "Space", "", 1, "mediumseagreen, dodgerblue, tomato"]
}
],
},
{
name: "Heatmap chart",
input: "100 100\n200 200\n300 300\n400 400\n500 500",
expectedMatch: /^<svg width/,
recipeConfig: [
{
"op": "Heatmap chart",
"args": ["Line feed", "Space", 25, 25, true, "", "", false, "white", "black"]
}
],
},
]);

36
tests/operations/tests/Protobuf.mjs Normal file
View File

@ -0,0 +1,36 @@
/**
* Protobuf tests.
*
* @author n1474335 [n1474335@gmail.com]
* @copyright Crown Copyright 2019
* @license Apache-2.0
*/
import TestRegister from "../TestRegister";
TestRegister.addTests([
{
name: "Protobuf Decode",
input: "0d1c0000001203596f751a024d65202b2a0a0a066162633132331200",
expectedOutput: JSON.stringify({
"1": 469762048,
"2": "You",
"3": "Me",
"4": 43,
"5": {
"1": "abc123",
"2": {}
}
}, null, 4),
recipeConfig: [
{
"op": "From Hex",
"args": ["Auto"]
},
{
"op": "Protobuf Decode",
"args": []
}
]
},
]);

32
webpack.config.js
View File

@ -48,7 +48,7 @@ module.exports = {
"process.browser": "true" "process.browser": "true"
}), }),
new MiniCssExtractPlugin({ new MiniCssExtractPlugin({
filename: "[name].css" filename: "assets/[name].css"
}), }),
], ],
resolve: { resolve: {
@ -80,7 +80,12 @@ module.exports = {
{ {
test: /\.css$/, test: /\.css$/,
use: [ use: [
MiniCssExtractPlugin.loader, {
loader: MiniCssExtractPlugin.loader,
options: {
publicPath: "../"
}
},
"css-loader", "css-loader",
"postcss-loader", "postcss-loader",
] ]
@ -88,7 +93,12 @@ module.exports = {
{ {
test: /\.scss$/, test: /\.scss$/,
use: [ use: [
MiniCssExtractPlugin.loader, {
loader: MiniCssExtractPlugin.loader,
options: {
publicPath: "../"
}
},
"css-loader", "css-loader",
"sass-loader", "sass-loader",
] ]
@ -97,7 +107,9 @@ module.exports = {
test: /\.(ico|eot|ttf|woff|woff2)$/, test: /\.(ico|eot|ttf|woff|woff2)$/,
loader: "url-loader", loader: "url-loader",
options: { options: {
limit: 10000 limit: 10000,
name: "[hash].[ext]",
outputPath: "assets"
} }
}, },
{ {
@ -120,7 +132,9 @@ module.exports = {
exclude: /web\/static/, exclude: /web\/static/,
loader: "url-loader", loader: "url-loader",
options: { options: {
limit: 10000 limit: 10000,
name: "[hash].[ext]",
outputPath: "assets"
} }
}, },
] ]
@ -133,11 +147,15 @@ module.exports = {
warningsFilter: [ warningsFilter: [
/source-map/, /source-map/,
/dependency is an expression/, /dependency is an expression/,
/export 'default'/ /export 'default'/,
/Can't resolve 'sodium'/
], ],
}, },
node: { node: {
fs: "empty" fs: "empty",
"child_process": "empty",
net: "empty",
tls: "empty"
}, },
performance: { performance: {
hints: false hints: false