From 6aa9d2b492a5900014103faebbc7dac65cf69e02 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 14 Dec 2018 16:43:03 +0000 Subject: [PATCH 01/22] Added 'Extract Files' operation and 'Forensics' category. --- package-lock.json | 142 ++++++++-------- src/core/config/Categories.json | 18 ++- src/core/lib/FileExtraction.mjs | 231 +++++++++++++++++++++++++++ src/core/lib/Stream.mjs | 164 +++++++++++++++++++ src/core/operations/ExtractFiles.mjs | 91 +++++++++++ 5 files changed, 572 insertions(+), 74 deletions(-) create mode 100644 src/core/lib/FileExtraction.mjs create mode 100644 src/core/lib/Stream.mjs create mode 100644 src/core/operations/ExtractFiles.mjs diff --git a/package-lock.json b/package-lock.json index 4f1be0a3..8b236eb9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1171,7 +1171,7 @@ }, "ansi-escapes": { "version": "3.1.0", - "resolved": "http://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", "dev": true }, @@ -1284,7 +1284,7 @@ }, "array-equal": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, @@ -1369,7 +1369,7 @@ }, "util": { "version": "0.10.3", - "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1457,7 +1457,7 @@ }, "axios": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", "dev": true, "requires": { @@ -1863,7 +1863,7 @@ }, "browserify-aes": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", "dev": true, "requires": { @@ -1900,7 +1900,7 @@ }, "browserify-rsa": { "version": "4.0.1", - "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", "dev": true, "requires": { @@ -1950,7 +1950,7 @@ }, "buffer": { "version": "4.9.1", - "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "dev": true, "requires": { @@ -2015,7 +2015,7 @@ }, "cacache": { "version": "10.0.4", - "resolved": "http://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", "integrity": "sha512-Dph0MzuH+rTQzGPNT9fAnrPmMmjKfST6trxJeK7NQuHRaVw24VzPRWTmg9MpcwOVQZO0E1FBICUlFeNaKPIfHA==", "dev": true, "requires": { @@ -2092,7 +2092,7 @@ }, "camelcase-keys": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", "dev": true, "requires": { @@ -2123,7 +2123,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -2590,7 +2590,7 @@ }, "create-hash": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", "dev": true, "requires": { @@ -2603,7 +2603,7 @@ }, "create-hmac": { "version": "1.1.7", - "resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", "dev": true, "requires": { @@ -2721,7 +2721,7 @@ }, "css-select": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", "dev": true, "requires": { @@ -3055,7 +3055,7 @@ }, "diffie-hellman": { "version": "5.0.3", - "resolved": "http://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", "dev": true, "requires": { @@ -3119,7 +3119,7 @@ "dependencies": { "domelementtype": { "version": "1.1.3", - "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", "dev": true }, @@ -3307,7 +3307,7 @@ }, "entities": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=", "dev": true }, @@ -3731,7 +3731,7 @@ }, "eventemitter2": { "version": "0.4.14", - "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", + "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", "integrity": "sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas=", "dev": true }, @@ -3743,7 +3743,7 @@ }, "events": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", "dev": true }, @@ -4149,7 +4149,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -4377,7 +4377,7 @@ }, "fs-extra": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", "dev": true, "requires": { @@ -4445,12 +4445,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4470,7 +4472,8 @@ "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", @@ -4618,6 +4621,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5023,7 +5027,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, @@ -5173,7 +5177,7 @@ }, "grunt-cli": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", "integrity": "sha1-VisRnrsGndtGSs4oRVAb6Xs1tqg=", "dev": true, "requires": { @@ -5221,7 +5225,7 @@ "dependencies": { "shelljs": { "version": "0.5.3", - "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", "integrity": "sha1-xUmCuZbHbvDB5rWfvcWCX1txMRM=", "dev": true } @@ -5241,7 +5245,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -5269,7 +5273,7 @@ }, "grunt-contrib-jshint": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", "integrity": "sha1-Np2QmyWTxA6L55lAshNAhQx5Oaw=", "dev": true, "requires": { @@ -5368,7 +5372,7 @@ "dependencies": { "colors": { "version": "1.1.2", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", "dev": true } @@ -5432,7 +5436,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -5450,7 +5454,7 @@ }, "handle-thing": { "version": "1.2.5", - "resolved": "http://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", + "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", "integrity": "sha1-/Xqtcmvxpf0W38KbL3pmAdJxOcQ=", "dev": true }, @@ -5677,7 +5681,7 @@ }, "html-webpack-plugin": { "version": "3.2.0", - "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", + "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=", "dev": true, "requires": { @@ -5725,7 +5729,7 @@ }, "htmlparser2": { "version": "3.8.3", - "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", "dev": true, "requires": { @@ -5744,7 +5748,7 @@ }, "http-errors": { "version": "1.6.3", - "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", "dev": true, "requires": { @@ -5773,7 +5777,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -6225,7 +6229,7 @@ }, "is-builtin-module": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", "dev": true, "requires": { @@ -6750,7 +6754,7 @@ }, "jsonfile": { "version": "2.4.0", - "resolved": "http://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", "dev": true, "requires": { @@ -6856,7 +6860,7 @@ }, "kew": { "version": "0.7.0", - "resolved": "http://registry.npmjs.org/kew/-/kew-0.7.0.tgz", + "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz", "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=", "dev": true }, @@ -6948,7 +6952,7 @@ }, "load-json-file": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "dev": true, "requires": { @@ -6961,7 +6965,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -7196,7 +7200,7 @@ }, "media-typer": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", "dev": true }, @@ -7255,7 +7259,7 @@ }, "meow": { "version": "3.7.0", - "resolved": "http://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", "dev": true, "requires": { @@ -7432,7 +7436,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, "requires": { @@ -7554,7 +7558,7 @@ }, "ncp": { "version": "1.0.1", - "resolved": "http://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", "integrity": "sha1-0VNn5cuHQyuhF9K/gP30Wuz7QkY=", "dev": true }, @@ -7617,7 +7621,7 @@ "dependencies": { "semver": { "version": "5.3.0", - "resolved": "http://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", "dev": true } @@ -7756,7 +7760,7 @@ "dependencies": { "colors": { "version": "0.5.1", - "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" }, "underscore": { @@ -8015,13 +8019,13 @@ }, "os-homedir": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true }, "os-locale": { "version": "1.4.0", - "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", "dev": true, "requires": { @@ -8030,7 +8034,7 @@ }, "os-tmpdir": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, @@ -8173,7 +8177,7 @@ }, "parse-asn1": { "version": "5.1.1", - "resolved": "http://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==", "dev": true, "requires": { @@ -8231,7 +8235,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, @@ -8272,7 +8276,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -8839,7 +8843,7 @@ }, "progress": { "version": "1.1.8", - "resolved": "http://registry.npmjs.org/progress/-/progress-1.1.8.tgz", + "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=" }, "promise-inflight": { @@ -8864,13 +8868,13 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true }, "winston": { "version": "2.1.1", - "resolved": "http://registry.npmjs.org/winston/-/winston-2.1.1.tgz", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz", "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=", "dev": true, "requires": { @@ -8885,7 +8889,7 @@ "dependencies": { "colors": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true }, @@ -9064,7 +9068,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -9426,7 +9430,7 @@ }, "require-uncached": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=", "dev": true, "requires": { @@ -9593,7 +9597,7 @@ }, "safe-regex": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", "dev": true, "requires": { @@ -9914,7 +9918,7 @@ }, "sha.js": { "version": "2.4.11", - "resolved": "http://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", "dev": true, "requires": { @@ -9958,7 +9962,7 @@ }, "shelljs": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=", "dev": true }, @@ -10610,7 +10614,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -10627,7 +10631,7 @@ }, "strip-eof": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "dev": true }, @@ -10706,7 +10710,7 @@ }, "tar": { "version": "2.2.1", - "resolved": "http://registry.npmjs.org/tar/-/tar-2.2.1.tgz", + "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz", "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=", "dev": true, "requires": { @@ -10734,7 +10738,7 @@ }, "through": { "version": "2.3.8", - "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", "dev": true }, @@ -11381,7 +11385,7 @@ "dependencies": { "async": { "version": "0.9.2", - "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=", "dev": true }, @@ -11407,7 +11411,7 @@ }, "valid-data-url": { "version": "0.1.6", - "resolved": "http://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", + "resolved": "https://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", "integrity": "sha512-FXg2qXMzfAhZc0y2HzELNfUeiOjPr+52hU1DNBWiJJ2luXD+dD1R9NA48Ug5aj0ibbxroeGDc/RJv6ThiGgkDw==", "dev": true }, @@ -11423,7 +11427,7 @@ }, "validator": { "version": "9.4.1", - "resolved": "http://registry.npmjs.org/validator/-/validator-9.4.1.tgz", + "resolved": "https://registry.npmjs.org/validator/-/validator-9.4.1.tgz", "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA==", "dev": true }, @@ -11847,7 +11851,7 @@ }, "webpack-node-externals": { "version": "1.7.2", - "resolved": "http://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", + "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", "integrity": "sha512-ajerHZ+BJKeCLviLUUmnyd5B4RavLF76uv3cs6KNuO8W+HuQaEs0y0L7o40NQxdPy5w0pcv8Ew7yPUAQG0UdCg==", "dev": true }, @@ -11984,7 +11988,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "dev": true, "requires": { diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 4fac84c9..8eccea12 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -245,7 +245,8 @@ "XPath expression", "JPath expression", "CSS selector", - "Extract EXIF" + "Extract EXIF", + "Extract Files" ] }, { @@ -336,14 +337,23 @@ "From MessagePack" ] }, + { + "name": "Forensics", + "ops": [ + "Detect File Type", + "Scan for Embedded Files", + "Extract Files", + "Remove EXIF", + "Extract EXIF", + "Render Image" + ] + }, { "name": "Other", "ops": [ "Entropy", "Frequency distribution", "Chi Square", - "Detect File Type", - "Scan for Embedded Files", "Disassemble x86", "Pseudo-Random Number Generator", "Generate UUID", @@ -351,8 +361,6 @@ "Generate HOTP", "Haversine distance", "Render Image", - "Remove EXIF", - "Extract EXIF", "Numberwang", "XKCD Random Number" ] diff --git a/src/core/lib/FileExtraction.mjs b/src/core/lib/FileExtraction.mjs new file mode 100644 index 00000000..927aca92 --- /dev/null +++ b/src/core/lib/FileExtraction.mjs @@ -0,0 +1,231 @@ +/** + * File extraction functions + * + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + * + */ +import Stream from "./Stream"; + +/** + * Attempts to extract a file from a data stream given its mime type and offset. + * + * @param {Uint8Array} bytes + * @param {Object} fileDetail + * @param {string} fileDetail.mime + * @param {string} fileDetail.ext + * @param {number} fileDetail.offset + * @returns {File} + */ +export function extractFile(bytes, fileDetail) { + let fileData; + switch (fileDetail.mime) { + case "image/jpeg": + fileData = extractJPEG(bytes, fileDetail.offset); + break; + case "application/x-msdownload": + fileData = extractMZPE(bytes, fileDetail.offset); + break; + case "application/pdf": + fileData = extractPDF(bytes, fileDetail.offset); + break; + case "application/zip": + fileData = extractZIP(bytes, fileDetail.offset); + break; + default: + throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`); + } + + return new File([fileData], `extracted_at_0x${fileDetail.offset.toString(16)}.${fileDetail.ext}`); +} + + +/** + * JPEG extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractJPEG(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + while (stream.hasMore()) { + const marker = stream.getBytes(2); + if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); + + let segmentSize = 0; + switch (marker[1]) { + // No length + case 0xd8: // Start of Image + case 0x01: // For temporary use in arithmetic coding + break; + case 0xd9: // End found + return stream.carve(); + + // Variable size segment + case 0xc0: // Start of frame (Baseline DCT) + case 0xc1: // Start of frame (Extended sequential DCT) + case 0xc2: // Start of frame (Progressive DCT) + case 0xc3: // Start of frame (Lossless sequential) + case 0xc4: // Define Huffman Table + case 0xc5: // Start of frame (Differential sequential DCT) + case 0xc6: // Start of frame (Differential progressive DCT) + case 0xc7: // Start of frame (Differential lossless) + case 0xc8: // Reserved for JPEG extensions + case 0xc9: // Start of frame (Extended sequential DCT) + case 0xca: // Start of frame (Progressive DCT) + case 0xcb: // Start of frame (Lossless sequential) + case 0xcc: // Define arithmetic conditioning table + case 0xcd: // Start of frame (Differential sequential DCT) + case 0xce: // Start of frame (Differential progressive DCT) + case 0xcf: // Start of frame (Differential lossless) + case 0xdb: // Define Quantization Table + case 0xde: // Define hierarchical progression + case 0xe0: // Application-specific + case 0xe1: // Application-specific + case 0xe2: // Application-specific + case 0xe3: // Application-specific + case 0xe4: // Application-specific + case 0xe5: // Application-specific + case 0xe6: // Application-specific + case 0xe7: // Application-specific + case 0xe8: // Application-specific + case 0xe9: // Application-specific + case 0xea: // Application-specific + case 0xeb: // Application-specific + case 0xec: // Application-specific + case 0xed: // Application-specific + case 0xee: // Application-specific + case 0xef: // Application-specific + case 0xfe: // Comment + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + break; + + // 1 byte + case 0xdf: // Expand reference image + stream.position++; + break; + + // 2 bytes + case 0xdc: // Define number of lines + case 0xdd: // Define restart interval + stream.position += 2; + break; + + // Start scan + case 0xda: // Start of scan + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + stream.continueUntil(0xff); + break; + + // Continue through encoded data + case 0x00: // Byte stuffing + case 0xd0: // Restart + case 0xd1: // Restart + case 0xd2: // Restart + case 0xd3: // Restart + case 0xd4: // Restart + case 0xd5: // Restart + case 0xd6: // Restart + case 0xd7: // Restart + stream.continueUntil(0xff); + break; + + default: + stream.continueUntil(0xff); + break; + } + } + + throw new Error("Unable to parse JPEG successfully"); +} + + +/** + * Portable executable extractor. + * Assumes that the offset refers to an MZ header. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractMZPE(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move to PE header pointer + stream.moveTo(0x3c); + const peAddress = stream.readInt(4, "le"); + + // Move to PE header + stream.moveTo(peAddress); + + // Get number of sections + stream.moveForwardsBy(6); + const numSections = stream.readInt(2, "le"); + + // Get optional header size + stream.moveForwardsBy(12); + const optionalHeaderSize = stream.readInt(2, "le"); + + // Move past optional header to section header + stream.moveForwardsBy(2 + optionalHeaderSize); + + // Move to final section header + stream.moveForwardsBy((numSections - 1) * 0x28); + + // Get raw data info + stream.moveForwardsBy(16); + const rawDataSize = stream.readInt(4, "le"); + const rawDataAddress = stream.readInt(4, "le"); + + // Move to end of final section + stream.moveTo(rawDataAddress + rawDataSize); + + return stream.carve(); +} + + +/** + * PDF extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractPDF(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find end-of-file marker (%%EOF) + stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]); + stream.moveForwardsBy(5); + stream.consumeIf(0x0d); + stream.consumeIf(0x0a); + + return stream.carve(); +} + + +/** + * ZIP extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractZIP(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find End of central directory record + stream.continueUntil([0x50, 0x4b, 0x05, 0x06]); + + // Get comment length and consume + stream.moveForwardsBy(20); + const commentLength = stream.readInt(2, "le"); + stream.moveForwardsBy(commentLength); + + return stream.carve(); +} diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs new file mode 100644 index 00000000..5f84e13c --- /dev/null +++ b/src/core/lib/Stream.mjs @@ -0,0 +1,164 @@ +/** + * Stream class for parsing binary protocols. + * + * @author n1474335 [n1474335@gmail.com] + * @author tlwr [toby@toby.codes] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + * + */ + +/** + * A Stream can be used to traverse a binary blob, interpreting sections of it + * as various data types. + * + * @param {Uint8Array} bytes + * @param {Object} fileDetail + * @param {string} fileDetail.mime + * @param {string} fileDetail.ext + * @param {number} fileDetail.offset + * @returns {File} + */ +export default class Stream { + + /** + * Stream constructor. + * + * @param {Uint8Array} input + */ + constructor(input) { + this.bytes = input; + this.position = 0; + } + + /** + * Get a number of bytes from the current position. + * + * @param {number} numBytes + * @returns {Uint8Array} + */ + getBytes(numBytes) { + const newPosition = this.position + numBytes; + const bytes = this.bytes.slice(this.position, newPosition); + this.position = newPosition; + return bytes; + } + + /** + * Interpret the following bytes as a string, stopping at the next null byte or + * the supplied limit. + * + * @param {number} numBytes + * @returns {string} + */ + readString(numBytes) { + let result = ""; + for (let i = this.position; i < this.position + numBytes; i++) { + const currentByte = this.bytes[i]; + if (currentByte === 0) break; + result += String.fromCharCode(currentByte); + } + this.position += numBytes; + return result; + } + + /** + * Interpret the following bytes as an integer in big or little endian. + * + * @param {number} numBytes + * @param {string} [endianness="be"] + * @returns {number} + */ + readInt(numBytes, endianness="be") { + let val = 0; + if (endianness === "be") { + for (let i = this.position; i < this.position + numBytes; i++) { + val = val << 8; + val |= this.bytes[i]; + } + } else { + for (let i = this.position + numBytes - 1; i >= this.position; i--) { + val = val << 8; + val |= this.bytes[i]; + } + } + this.position += numBytes; + return val; + } + + /** + * Consume the stream until we reach the specified byte or sequence of bytes. + * + * @param {number|List} val + */ + continueUntil(val) { + if (typeof val === "number") { + while (++this.position < this.bytes.length && this.bytes[this.position] !== val) { + continue; + } + return; + } + + // val is an array + let found = false; + while (!found && this.position < this.bytes.length) { + while (++this.position < this.bytes.length && this.bytes[this.position] !== val[0]) { + continue; + } + found = true; + for (let i = 1; i < val.length; i++) { + if (this.position + i > this.bytes.length || this.bytes[this.position + i] !== val[i]) + found = false; + } + } + } + + /** + * Consume the next byte if it matches the supplied value. + * + * @param {number} val + */ + consumeIf(val) { + if (this.bytes[this.position] === val) + this.position++; + } + + /** + * Move forwards through the stream by the specified number of bytes. + * + * @param {number} numBytes + */ + moveForwardsBy(numBytes) { + this.position += numBytes; + } + + /** + * Move to a specified position in the stream. + * + * @param {number} pos + */ + moveTo(pos) { + if (pos < 0 || pos > this.bytes.length - 1) + throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); + this.position = pos; + } + + /** + * Returns true if there are more bytes left in the stream. + * + * @returns {boolean} + */ + hasMore() { + return this.position < this.bytes.length; + } + + /** + * Returns a slice of the stream up to the current position. + * + * @returns {Uint8Array} + */ + carve() { + return this.bytes.slice(0, this.position); + } + +} diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs new file mode 100644 index 00000000..c213b256 --- /dev/null +++ b/src/core/operations/ExtractFiles.mjs @@ -0,0 +1,91 @@ +/** + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +// import OperationError from "../errors/OperationError"; +import Magic from "../lib/Magic"; +import Utils from "../Utils"; +import {extractFile} from "../lib/FileExtraction"; + +/** + * Extract Files operation + */ +class ExtractFiles extends Operation { + + /** + * ExtractFiles constructor + */ + constructor() { + super(); + + this.name = "Extract Files"; + this.module = "Default"; + this.description = "TODO"; + this.infoURL = "https://forensicswiki.org/wiki/File_Carving"; + this.inputType = "ArrayBuffer"; + this.outputType = "List"; + this.presentType = "html"; + this.args = []; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {List} + */ + run(input, args) { + const bytes = new Uint8Array(input); + + // Scan for embedded files + const fileDetails = scanForEmbeddedFiles(bytes); + + // Extract each file that we support + const files = []; + fileDetails.forEach(fileDetail => { + try { + files.push(extractFile(bytes, fileDetail)); + } catch (err) {} + }); + + return files; + } + + /** + * Displays the files in HTML for web apps. + * + * @param {File[]} files + * @returns {html} + */ + async present(files) { + return await Utils.displayFilesAsHTML(files); + } + +} + +/** + * TODO refactor + * @param data + */ +function scanForEmbeddedFiles(data) { + let type; + const types = []; + + for (let i = 0; i < data.length; i++) { + type = Magic.magicFileType(data.slice(i)); + if (type) { + types.push({ + offset: i, + ext: type.ext, + mime: type.mime, + desc: type.desc + }); + } + } + + return types; +} + +export default ExtractFiles; From e6fb0be1d04bd8c96a827d244f20ea3f01a991aa Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 18 Dec 2018 17:44:42 +0000 Subject: [PATCH 02/22] Refactored file type detection engine --- src/core/lib/BCD.mjs | 0 src/core/lib/Base58.mjs | 0 src/core/lib/Base64.mjs | 0 src/core/lib/CanvasComponents.mjs | 0 src/core/lib/FileExtraction.mjs | 231 ------ src/core/lib/FileType.mjs | 808 +++++++++++++++++++ src/core/lib/Magic.mjs | 455 +---------- src/core/lib/Stream.mjs | 7 - src/core/operations/DetectFileType.mjs | 20 +- src/core/operations/ExtractFiles.mjs | 28 +- src/core/operations/RenderImage.mjs | 12 +- src/core/operations/ScanForEmbeddedFiles.mjs | 33 +- 12 files changed, 867 insertions(+), 727 deletions(-) mode change 100755 => 100644 src/core/lib/BCD.mjs mode change 100755 => 100644 src/core/lib/Base58.mjs mode change 100755 => 100644 src/core/lib/Base64.mjs mode change 100755 => 100644 src/core/lib/CanvasComponents.mjs delete mode 100644 src/core/lib/FileExtraction.mjs create mode 100644 src/core/lib/FileType.mjs diff --git a/src/core/lib/BCD.mjs b/src/core/lib/BCD.mjs old mode 100755 new mode 100644 diff --git a/src/core/lib/Base58.mjs b/src/core/lib/Base58.mjs old mode 100755 new mode 100644 diff --git a/src/core/lib/Base64.mjs b/src/core/lib/Base64.mjs old mode 100755 new mode 100644 diff --git a/src/core/lib/CanvasComponents.mjs b/src/core/lib/CanvasComponents.mjs old mode 100755 new mode 100644 diff --git a/src/core/lib/FileExtraction.mjs b/src/core/lib/FileExtraction.mjs deleted file mode 100644 index 927aca92..00000000 --- a/src/core/lib/FileExtraction.mjs +++ /dev/null @@ -1,231 +0,0 @@ -/** - * File extraction functions - * - * @author n1474335 [n1474335@gmail.com] - * @copyright Crown Copyright 2018 - * @license Apache-2.0 - * - */ -import Stream from "./Stream"; - -/** - * Attempts to extract a file from a data stream given its mime type and offset. - * - * @param {Uint8Array} bytes - * @param {Object} fileDetail - * @param {string} fileDetail.mime - * @param {string} fileDetail.ext - * @param {number} fileDetail.offset - * @returns {File} - */ -export function extractFile(bytes, fileDetail) { - let fileData; - switch (fileDetail.mime) { - case "image/jpeg": - fileData = extractJPEG(bytes, fileDetail.offset); - break; - case "application/x-msdownload": - fileData = extractMZPE(bytes, fileDetail.offset); - break; - case "application/pdf": - fileData = extractPDF(bytes, fileDetail.offset); - break; - case "application/zip": - fileData = extractZIP(bytes, fileDetail.offset); - break; - default: - throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`); - } - - return new File([fileData], `extracted_at_0x${fileDetail.offset.toString(16)}.${fileDetail.ext}`); -} - - -/** - * JPEG extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractJPEG(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - while (stream.hasMore()) { - const marker = stream.getBytes(2); - if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); - - let segmentSize = 0; - switch (marker[1]) { - // No length - case 0xd8: // Start of Image - case 0x01: // For temporary use in arithmetic coding - break; - case 0xd9: // End found - return stream.carve(); - - // Variable size segment - case 0xc0: // Start of frame (Baseline DCT) - case 0xc1: // Start of frame (Extended sequential DCT) - case 0xc2: // Start of frame (Progressive DCT) - case 0xc3: // Start of frame (Lossless sequential) - case 0xc4: // Define Huffman Table - case 0xc5: // Start of frame (Differential sequential DCT) - case 0xc6: // Start of frame (Differential progressive DCT) - case 0xc7: // Start of frame (Differential lossless) - case 0xc8: // Reserved for JPEG extensions - case 0xc9: // Start of frame (Extended sequential DCT) - case 0xca: // Start of frame (Progressive DCT) - case 0xcb: // Start of frame (Lossless sequential) - case 0xcc: // Define arithmetic conditioning table - case 0xcd: // Start of frame (Differential sequential DCT) - case 0xce: // Start of frame (Differential progressive DCT) - case 0xcf: // Start of frame (Differential lossless) - case 0xdb: // Define Quantization Table - case 0xde: // Define hierarchical progression - case 0xe0: // Application-specific - case 0xe1: // Application-specific - case 0xe2: // Application-specific - case 0xe3: // Application-specific - case 0xe4: // Application-specific - case 0xe5: // Application-specific - case 0xe6: // Application-specific - case 0xe7: // Application-specific - case 0xe8: // Application-specific - case 0xe9: // Application-specific - case 0xea: // Application-specific - case 0xeb: // Application-specific - case 0xec: // Application-specific - case 0xed: // Application-specific - case 0xee: // Application-specific - case 0xef: // Application-specific - case 0xfe: // Comment - segmentSize = stream.readInt(2, "be"); - stream.position += segmentSize - 2; - break; - - // 1 byte - case 0xdf: // Expand reference image - stream.position++; - break; - - // 2 bytes - case 0xdc: // Define number of lines - case 0xdd: // Define restart interval - stream.position += 2; - break; - - // Start scan - case 0xda: // Start of scan - segmentSize = stream.readInt(2, "be"); - stream.position += segmentSize - 2; - stream.continueUntil(0xff); - break; - - // Continue through encoded data - case 0x00: // Byte stuffing - case 0xd0: // Restart - case 0xd1: // Restart - case 0xd2: // Restart - case 0xd3: // Restart - case 0xd4: // Restart - case 0xd5: // Restart - case 0xd6: // Restart - case 0xd7: // Restart - stream.continueUntil(0xff); - break; - - default: - stream.continueUntil(0xff); - break; - } - } - - throw new Error("Unable to parse JPEG successfully"); -} - - -/** - * Portable executable extractor. - * Assumes that the offset refers to an MZ header. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractMZPE(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Move to PE header pointer - stream.moveTo(0x3c); - const peAddress = stream.readInt(4, "le"); - - // Move to PE header - stream.moveTo(peAddress); - - // Get number of sections - stream.moveForwardsBy(6); - const numSections = stream.readInt(2, "le"); - - // Get optional header size - stream.moveForwardsBy(12); - const optionalHeaderSize = stream.readInt(2, "le"); - - // Move past optional header to section header - stream.moveForwardsBy(2 + optionalHeaderSize); - - // Move to final section header - stream.moveForwardsBy((numSections - 1) * 0x28); - - // Get raw data info - stream.moveForwardsBy(16); - const rawDataSize = stream.readInt(4, "le"); - const rawDataAddress = stream.readInt(4, "le"); - - // Move to end of final section - stream.moveTo(rawDataAddress + rawDataSize); - - return stream.carve(); -} - - -/** - * PDF extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractPDF(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Find end-of-file marker (%%EOF) - stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]); - stream.moveForwardsBy(5); - stream.consumeIf(0x0d); - stream.consumeIf(0x0a); - - return stream.carve(); -} - - -/** - * ZIP extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractZIP(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Find End of central directory record - stream.continueUntil([0x50, 0x4b, 0x05, 0x06]); - - // Get comment length and consume - stream.moveForwardsBy(20); - const commentLength = stream.readInt(2, "le"); - stream.moveForwardsBy(commentLength); - - return stream.carve(); -} diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs new file mode 100644 index 00000000..443bdd36 --- /dev/null +++ b/src/core/lib/FileType.mjs @@ -0,0 +1,808 @@ +/** + * File type functions + * + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + * + */ +import Stream from "./Stream"; + +/** + * A categorised table of file types, including signatures to identifying them and functions + * to extract them where possible. + */ +const FILE_SIGNATURES = { + "Pictures": [ + { + name: "JPEG Image", + extension: "jpg", + mime: "image/jpeg", + description: "", + signature: { + 0: 0xff, + 1: 0xd8, + 2: 0xff + }, + extractor: extractJPEG + }, + { + name: "GIF Image", + extension: "gif", + mime: "image/gif", + description: "", + signature: { + 0: 0x47, + 1: 0x49, + 2: 0x46 + }, + extractor: null + }, + { + name: "PNG Image", + extension: "png", + mime: "image/png", + description: "", + signature: { + 0: 0x89, + 1: 0x50, + 2: 0x4e, + 3: 0x47 + }, + extractor: null + }, + + ], + "Documents": [ + { + name: "Portable Document Format", + extension: "pdf", + mime: "application/pdf", + description: "", + signature: { + 0: 0x25, + 1: 0x50, + 2: 0x44, + 3: 0x46 + }, + extractor: extractPDF + }, + ], + "Applications": [ + { + name: "Windows Portable Executable", + extension: "exe", + mime: "application/x-msdownload", + description: "", + signature: { + 0: 0x4d, + 1: 0x5a + }, + extractor: extractMZPE + }, + ], + "Archives": [ + { + name: "ZIP", + extension: "zip", + mime: "application/zip", + description: "", + signature: { + 0: 0x50, + 1: 0x4b, + 2: [0x3, 0x5, 0x7], + 3: [0x4, 0x6, 0x8] + }, + extractor: extractZIP + }, + + ], +}; + + +/** + * Checks whether a signature matches a buffer. + * + * @param {Object} sig - A dictionary of offsets with values assigned to them. These + * values can be numbers for static checks, arrays of potential valid matches, or + * bespoke functions to check the validity of the buffer value at that offset. + * @param {Uint8Array} buf + * @returns {boolean} + */ +function signatureMatches(sig, buf) { + for (const offset in sig) { + switch (typeof sig[offset]) { + case "number": // Static check + if (buf[offset] !== sig[offset]) + return false; + break; + case "object": // Array of options + if (sig[offset].indexOf(buf[offset]) < 0) + return false; + break; + case "function": // More complex calculation + if (!sig[offset](buf[offset])) + return false; + break; + default: + throw new Error(`Unrecognised signature type at offset ${offset}`); + } + } + return true; +} + + +/** + * Given a buffer, detects magic byte sequences at specific positions and returns the + * extension and mime type. + * + * @param {Uint8Array} buf + * @returns {Object[]} type + * @returns {string} type.ext - File extension + * @returns {string} type.mime - Mime type + * @returns {string} [type.desc] - Description + */ +export function detectFileType(buf) { + if (!(buf && buf.length > 1)) { + return []; + } + + const matchingFiles = []; + + // TODO allow user to select which categories to check + for (const cat in FILE_SIGNATURES) { + const category = FILE_SIGNATURES[cat]; + + category.forEach(filetype => { + if (signatureMatches(filetype.signature, buf)) { + matchingFiles.push(filetype); + } + }); + } + return matchingFiles; + + // Delete all below this line once implemented in FILE_SIGNATURES above. + + + /* + if (buf[0] === 0xFF && buf[1] === 0xD8 && buf[2] === 0xFF) { + return { + ext: "jpg", + mime: "image/jpeg" + }; + } + + if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4E && buf[3] === 0x47) { + return { + ext: "png", + mime: "image/png" + }; + } + + if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) { + return { + ext: "gif", + mime: "image/gif" + }; + } + + if (buf[8] === 0x57 && buf[9] === 0x45 && buf[10] === 0x42 && buf[11] === 0x50) { + return { + ext: "webp", + mime: "image/webp" + }; + } + + // needs to be before `tif` check + if (((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) && buf[8] === 0x43 && buf[9] === 0x52) { + return { + ext: "cr2", + mime: "image/x-canon-cr2" + }; + } + + if ((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) { + return { + ext: "tif", + mime: "image/tiff" + }; + } + + if (buf[0] === 0x42 && buf[1] === 0x4D) { + return { + ext: "bmp", + mime: "image/bmp" + }; + } + + if (buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0xBC) { + return { + ext: "jxr", + mime: "image/vnd.ms-photo" + }; + } + + if (buf[0] === 0x38 && buf[1] === 0x42 && buf[2] === 0x50 && buf[3] === 0x53) { + return { + ext: "psd", + mime: "image/vnd.adobe.photoshop" + }; + } + + // needs to be before `zip` check + if (buf[0] === 0x50 && buf[1] === 0x4B && buf[2] === 0x3 && buf[3] === 0x4 && buf[30] === 0x6D && buf[31] === 0x69 && buf[32] === 0x6D && buf[33] === 0x65 && buf[34] === 0x74 && buf[35] === 0x79 && buf[36] === 0x70 && buf[37] === 0x65 && buf[38] === 0x61 && buf[39] === 0x70 && buf[40] === 0x70 && buf[41] === 0x6C && buf[42] === 0x69 && buf[43] === 0x63 && buf[44] === 0x61 && buf[45] === 0x74 && buf[46] === 0x69 && buf[47] === 0x6F && buf[48] === 0x6E && buf[49] === 0x2F && buf[50] === 0x65 && buf[51] === 0x70 && buf[52] === 0x75 && buf[53] === 0x62 && buf[54] === 0x2B && buf[55] === 0x7A && buf[56] === 0x69 && buf[57] === 0x70) { + return { + ext: "epub", + mime: "application/epub+zip" + }; + } + + if (buf[0] === 0x50 && buf[1] === 0x4B && (buf[2] === 0x3 || buf[2] === 0x5 || buf[2] === 0x7) && (buf[3] === 0x4 || buf[3] === 0x6 || buf[3] === 0x8)) { + return { + ext: "zip", + mime: "application/zip" + }; + } + + if (buf[257] === 0x75 && buf[258] === 0x73 && buf[259] === 0x74 && buf[260] === 0x61 && buf[261] === 0x72) { + return { + ext: "tar", + mime: "application/x-tar" + }; + } + + if (buf[0] === 0x52 && buf[1] === 0x61 && buf[2] === 0x72 && buf[3] === 0x21 && buf[4] === 0x1A && buf[5] === 0x7 && (buf[6] === 0x0 || buf[6] === 0x1)) { + return { + ext: "rar", + mime: "application/x-rar-compressed" + }; + } + + if (buf[0] === 0x1F && buf[1] === 0x8B && buf[2] === 0x8) { + return { + ext: "gz", + mime: "application/gzip" + }; + } + + if (buf[0] === 0x42 && buf[1] === 0x5A && buf[2] === 0x68) { + return { + ext: "bz2", + mime: "application/x-bzip2" + }; + } + + if (buf[0] === 0x37 && buf[1] === 0x7A && buf[2] === 0xBC && buf[3] === 0xAF && buf[4] === 0x27 && buf[5] === 0x1C) { + return { + ext: "7z", + mime: "application/x-7z-compressed" + }; + } + + if (buf[0] === 0x78 && buf[1] === 0x01) { + return { + ext: "dmg, zlib", + mime: "application/x-apple-diskimage, application/x-deflate" + }; + } + + if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && (buf[3] === 0x18 || buf[3] === 0x20) && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) || (buf[0] === 0x33 && buf[1] === 0x67 && buf[2] === 0x70 && buf[3] === 0x35) || (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x6D && buf[9] === 0x70 && buf[10] === 0x34 && buf[11] === 0x32 && buf[16] === 0x6D && buf[17] === 0x70 && buf[18] === 0x34 && buf[19] === 0x31 && buf[20] === 0x6D && buf[21] === 0x70 && buf[22] === 0x34 && buf[23] === 0x32 && buf[24] === 0x69 && buf[25] === 0x73 && buf[26] === 0x6F && buf[27] === 0x6D)) { + return { + ext: "mp4", + mime: "video/mp4" + }; + } + + if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x56)) { + return { + ext: "m4v", + mime: "video/x-m4v" + }; + } + + if (buf[0] === 0x4D && buf[1] === 0x54 && buf[2] === 0x68 && buf[3] === 0x64) { + return { + ext: "mid", + mime: "audio/midi" + }; + } + + // needs to be before the `webm` check + if (buf[31] === 0x6D && buf[32] === 0x61 && buf[33] === 0x74 && buf[34] === 0x72 && buf[35] === 0x6f && buf[36] === 0x73 && buf[37] === 0x6B && buf[38] === 0x61) { + return { + ext: "mkv", + mime: "video/x-matroska" + }; + } + + if (buf[0] === 0x1A && buf[1] === 0x45 && buf[2] === 0xDF && buf[3] === 0xA3) { + return { + ext: "webm", + mime: "video/webm" + }; + } + + if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x14 && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) { + return { + ext: "mov", + mime: "video/quicktime" + }; + } + + if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x41 && buf[9] === 0x56 && buf[10] === 0x49) { + return { + ext: "avi", + mime: "video/x-msvideo" + }; + } + + if (buf[0] === 0x30 && buf[1] === 0x26 && buf[2] === 0xB2 && buf[3] === 0x75 && buf[4] === 0x8E && buf[5] === 0x66 && buf[6] === 0xCF && buf[7] === 0x11 && buf[8] === 0xA6 && buf[9] === 0xD9) { + return { + ext: "wmv", + mime: "video/x-ms-wmv" + }; + } + + if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x1 && buf[3].toString(16)[0] === "b") { + return { + ext: "mpg", + mime: "video/mpeg" + }; + } + + if ((buf[0] === 0x49 && buf[1] === 0x44 && buf[2] === 0x33) || (buf[0] === 0xFF && buf[1] === 0xfb)) { + return { + ext: "mp3", + mime: "audio/mpeg" + }; + } + + if ((buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x41) || (buf[0] === 0x4D && buf[1] === 0x34 && buf[2] === 0x41 && buf[3] === 0x20)) { + return { + ext: "m4a", + mime: "audio/m4a" + }; + } + + if (buf[0] === 0x4F && buf[1] === 0x67 && buf[2] === 0x67 && buf[3] === 0x53) { + return { + ext: "ogg", + mime: "audio/ogg" + }; + } + + if (buf[0] === 0x66 && buf[1] === 0x4C && buf[2] === 0x61 && buf[3] === 0x43) { + return { + ext: "flac", + mime: "audio/x-flac" + }; + } + + if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x57 && buf[9] === 0x41 && buf[10] === 0x56 && buf[11] === 0x45) { + return { + ext: "wav", + mime: "audio/x-wav" + }; + } + + if (buf[0] === 0x23 && buf[1] === 0x21 && buf[2] === 0x41 && buf[3] === 0x4D && buf[4] === 0x52 && buf[5] === 0x0A) { + return { + ext: "amr", + mime: "audio/amr" + }; + } + + if (buf[0] === 0x25 && buf[1] === 0x50 && buf[2] === 0x44 && buf[3] === 0x46) { + return { + ext: "pdf", + mime: "application/pdf" + }; + } + + if (buf[0] === 0x4D && buf[1] === 0x5A) { + return { + ext: "exe", + mime: "application/x-msdownload" + }; + } + + if ((buf[0] === 0x43 || buf[0] === 0x46) && buf[1] === 0x57 && buf[2] === 0x53) { + return { + ext: "swf", + mime: "application/x-shockwave-flash" + }; + } + + if (buf[0] === 0x7B && buf[1] === 0x5C && buf[2] === 0x72 && buf[3] === 0x74 && buf[4] === 0x66) { + return { + ext: "rtf", + mime: "application/rtf" + }; + } + + if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x46 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { + return { + ext: "woff", + mime: "application/font-woff" + }; + } + + if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x32 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { + return { + ext: "woff2", + mime: "application/font-woff" + }; + } + + if (buf[34] === 0x4C && buf[35] === 0x50 && ((buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x01) || (buf[8] === 0x01 && buf[9] === 0x00 && buf[10] === 0x00) || (buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x02))) { + return { + ext: "eot", + mime: "application/octet-stream" + }; + } + + if (buf[0] === 0x00 && buf[1] === 0x01 && buf[2] === 0x00 && buf[3] === 0x00 && buf[4] === 0x00) { + return { + ext: "ttf", + mime: "application/font-sfnt" + }; + } + + if (buf[0] === 0x4F && buf[1] === 0x54 && buf[2] === 0x54 && buf[3] === 0x4F && buf[4] === 0x00) { + return { + ext: "otf", + mime: "application/font-sfnt" + }; + } + + if (buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0x01 && buf[3] === 0x00) { + return { + ext: "ico", + mime: "image/x-icon" + }; + } + + if (buf[0] === 0x46 && buf[1] === 0x4C && buf[2] === 0x56 && buf[3] === 0x01) { + return { + ext: "flv", + mime: "video/x-flv" + }; + } + + if (buf[0] === 0x25 && buf[1] === 0x21) { + return { + ext: "ps", + mime: "application/postscript" + }; + } + + if (buf[0] === 0xFD && buf[1] === 0x37 && buf[2] === 0x7A && buf[3] === 0x58 && buf[4] === 0x5A && buf[5] === 0x00) { + return { + ext: "xz", + mime: "application/x-xz" + }; + } + + if (buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4C && buf[3] === 0x69) { + return { + ext: "sqlite", + mime: "application/x-sqlite3" + }; + } + */ + + /** + * + * Added by n1474335 [n1474335@gmail.com] from here on + * + */ + /* + if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) { + return { + ext: "z, tar.z", + mime: "application/x-gtar" + }; + } + + if (buf[0] === 0x7F && buf[1] === 0x45 && buf[2] === 0x4C && buf[3] === 0x46) { + return { + ext: "none, axf, bin, elf, o, prx, puff, so", + mime: "application/x-executable", + desc: "Executable and Linkable Format file. No standard file extension." + }; + } + + if (buf[0] === 0xCA && buf[1] === 0xFE && buf[2] === 0xBA && buf[3] === 0xBE) { + return { + ext: "class", + mime: "application/java-vm" + }; + } + + if (buf[0] === 0xEF && buf[1] === 0xBB && buf[2] === 0xBF) { + return { + ext: "txt", + mime: "text/plain", + desc: "UTF-8 encoded Unicode byte order mark detected, commonly but not exclusively seen in text files." + }; + } + + // Must be before Little-endian UTF-16 BOM + if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) { + return { + ext: "UTF32LE", + mime: "charset/utf32le", + desc: "Little-endian UTF-32 encoded Unicode byte order mark detected." + }; + } + + if (buf[0] === 0xFF && buf[1] === 0xFE) { + return { + ext: "UTF16LE", + mime: "charset/utf16le", + desc: "Little-endian UTF-16 encoded Unicode byte order mark detected." + }; + } + + if ((buf[0x8001] === 0x43 && buf[0x8002] === 0x44 && buf[0x8003] === 0x30 && buf[0x8004] === 0x30 && buf[0x8005] === 0x31) || + (buf[0x8801] === 0x43 && buf[0x8802] === 0x44 && buf[0x8803] === 0x30 && buf[0x8804] === 0x30 && buf[0x8805] === 0x31) || + (buf[0x9001] === 0x43 && buf[0x9002] === 0x44 && buf[0x9003] === 0x30 && buf[0x9004] === 0x30 && buf[0x9005] === 0x31)) { + return { + ext: "iso", + mime: "application/octet-stream", + desc: "ISO 9660 CD/DVD image file" + }; + } + + if (buf[0] === 0xD0 && buf[1] === 0xCF && buf[2] === 0x11 && buf[3] === 0xE0 && buf[4] === 0xA1 && buf[5] === 0xB1 && buf[6] === 0x1A && buf[7] === 0xE1) { + return { + ext: "doc, xls, ppt", + mime: "application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint", + desc: "Microsoft Office documents" + }; + } + + if (buf[0] === 0x64 && buf[1] === 0x65 && buf[2] === 0x78 && buf[3] === 0x0A && buf[4] === 0x30 && buf[5] === 0x33 && buf[6] === 0x35 && buf[7] === 0x00) { + return { + ext: "dex", + mime: "application/octet-stream", + desc: "Dalvik Executable (Android)" + }; + } + + if (buf[0] === 0x4B && buf[1] === 0x44 && buf[2] === 0x4D) { + return { + ext: "vmdk", + mime: "application/vmdk, application/x-virtualbox-vmdk" + }; + } + + if (buf[0] === 0x43 && buf[1] === 0x72 && buf[2] === 0x32 && buf[3] === 0x34) { + return { + ext: "crx", + mime: "application/crx", + desc: "Google Chrome extension or packaged app" + }; + } + + if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) { + return { + ext: "zlib", + mime: "application/x-deflate" + }; + } + + return null; + */ +} + + +/** + * Attempts to extract a file from a data stream given its offset and extractor function. + * + * @param {Uint8Array} bytes + * @param {Object} fileDetail + * @param {string} fileDetail.mime + * @param {string} fileDetail.extension + * @param {Function} fileDetail.extractor + * @param {number} offset + * @returns {File} + */ +export function extractFile(bytes, fileDetail, offset) { + if (fileDetail.extractor) { + const fileData = fileDetail.extractor(bytes, offset); + return new File([fileData], `extracted_at_0x${offset.toString(16)}.${fileDetail.extension}`); + } + + throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`); +} + + +/** + * JPEG extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractJPEG(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + while (stream.hasMore()) { + const marker = stream.getBytes(2); + if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); + + let segmentSize = 0; + switch (marker[1]) { + // No length + case 0xd8: // Start of Image + case 0x01: // For temporary use in arithmetic coding + break; + case 0xd9: // End found + return stream.carve(); + + // Variable size segment + case 0xc0: // Start of frame (Baseline DCT) + case 0xc1: // Start of frame (Extended sequential DCT) + case 0xc2: // Start of frame (Progressive DCT) + case 0xc3: // Start of frame (Lossless sequential) + case 0xc4: // Define Huffman Table + case 0xc5: // Start of frame (Differential sequential DCT) + case 0xc6: // Start of frame (Differential progressive DCT) + case 0xc7: // Start of frame (Differential lossless) + case 0xc8: // Reserved for JPEG extensions + case 0xc9: // Start of frame (Extended sequential DCT) + case 0xca: // Start of frame (Progressive DCT) + case 0xcb: // Start of frame (Lossless sequential) + case 0xcc: // Define arithmetic conditioning table + case 0xcd: // Start of frame (Differential sequential DCT) + case 0xce: // Start of frame (Differential progressive DCT) + case 0xcf: // Start of frame (Differential lossless) + case 0xdb: // Define Quantization Table + case 0xde: // Define hierarchical progression + case 0xe0: // Application-specific + case 0xe1: // Application-specific + case 0xe2: // Application-specific + case 0xe3: // Application-specific + case 0xe4: // Application-specific + case 0xe5: // Application-specific + case 0xe6: // Application-specific + case 0xe7: // Application-specific + case 0xe8: // Application-specific + case 0xe9: // Application-specific + case 0xea: // Application-specific + case 0xeb: // Application-specific + case 0xec: // Application-specific + case 0xed: // Application-specific + case 0xee: // Application-specific + case 0xef: // Application-specific + case 0xfe: // Comment + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + break; + + // 1 byte + case 0xdf: // Expand reference image + stream.position++; + break; + + // 2 bytes + case 0xdc: // Define number of lines + case 0xdd: // Define restart interval + stream.position += 2; + break; + + // Start scan + case 0xda: // Start of scan + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + stream.continueUntil(0xff); + break; + + // Continue through encoded data + case 0x00: // Byte stuffing + case 0xd0: // Restart + case 0xd1: // Restart + case 0xd2: // Restart + case 0xd3: // Restart + case 0xd4: // Restart + case 0xd5: // Restart + case 0xd6: // Restart + case 0xd7: // Restart + stream.continueUntil(0xff); + break; + + default: + stream.continueUntil(0xff); + break; + } + } + + throw new Error("Unable to parse JPEG successfully"); +} + + +/** + * Portable executable extractor. + * Assumes that the offset refers to an MZ header. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractMZPE(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move to PE header pointer + stream.moveTo(0x3c); + const peAddress = stream.readInt(4, "le"); + + // Move to PE header + stream.moveTo(peAddress); + + // Get number of sections + stream.moveForwardsBy(6); + const numSections = stream.readInt(2, "le"); + + // Get optional header size + stream.moveForwardsBy(12); + const optionalHeaderSize = stream.readInt(2, "le"); + + // Move past optional header to section header + stream.moveForwardsBy(2 + optionalHeaderSize); + + // Move to final section header + stream.moveForwardsBy((numSections - 1) * 0x28); + + // Get raw data info + stream.moveForwardsBy(16); + const rawDataSize = stream.readInt(4, "le"); + const rawDataAddress = stream.readInt(4, "le"); + + // Move to end of final section + stream.moveTo(rawDataAddress + rawDataSize); + + return stream.carve(); +} + + +/** + * PDF extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractPDF(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find end-of-file marker (%%EOF) + stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]); + stream.moveForwardsBy(5); + stream.consumeIf(0x0d); + stream.consumeIf(0x0a); + + return stream.carve(); +} + + +/** + * ZIP extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractZIP(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find End of central directory record + stream.continueUntil([0x50, 0x4b, 0x05, 0x06]); + + // Get comment length and consume + stream.moveForwardsBy(20); + const commentLength = stream.readInt(2, "le"); + stream.moveForwardsBy(commentLength); + + return stream.carve(); +} diff --git a/src/core/lib/Magic.mjs b/src/core/lib/Magic.mjs index b4d5a7b0..91472e21 100644 --- a/src/core/lib/Magic.mjs +++ b/src/core/lib/Magic.mjs @@ -2,6 +2,7 @@ import OperationConfig from "../config/OperationConfig.json"; import Utils from "../Utils"; import Recipe from "../Recipe"; import Dish from "../Dish"; +import {detectFileType} from "./FileType"; import chiSquared from "chi-squared"; /** @@ -92,7 +93,14 @@ class Magic { * @returns {string} [type.desc] - Description */ detectFileType() { - return Magic.magicFileType(this.inputBuffer); + const fileType = detectFileType(this.inputBuffer); + + if (!fileType.length) return null; + return { + ext: fileType[0].extension, + mime: fileType[0].mime, + desc: fileType[0].description + }; } /** @@ -784,452 +792,9 @@ class Magic { }[code]; } - - /** - * Given a buffer, detects magic byte sequences at specific positions and returns the - * extension and mime type. - * - * @param {Uint8Array} buf - * @returns {Object} type - * @returns {string} type.ext - File extension - * @returns {string} type.mime - Mime type - * @returns {string} [type.desc] - Description - */ - static magicFileType(buf) { - if (!(buf && buf.length > 1)) { - return null; - } - - if (buf[0] === 0xFF && buf[1] === 0xD8 && buf[2] === 0xFF) { - return { - ext: "jpg", - mime: "image/jpeg" - }; - } - - if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4E && buf[3] === 0x47) { - return { - ext: "png", - mime: "image/png" - }; - } - - if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) { - return { - ext: "gif", - mime: "image/gif" - }; - } - - if (buf[8] === 0x57 && buf[9] === 0x45 && buf[10] === 0x42 && buf[11] === 0x50) { - return { - ext: "webp", - mime: "image/webp" - }; - } - - // needs to be before `tif` check - if (((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) && buf[8] === 0x43 && buf[9] === 0x52) { - return { - ext: "cr2", - mime: "image/x-canon-cr2" - }; - } - - if ((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) { - return { - ext: "tif", - mime: "image/tiff" - }; - } - - if (buf[0] === 0x42 && buf[1] === 0x4D) { - return { - ext: "bmp", - mime: "image/bmp" - }; - } - - if (buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0xBC) { - return { - ext: "jxr", - mime: "image/vnd.ms-photo" - }; - } - - if (buf[0] === 0x38 && buf[1] === 0x42 && buf[2] === 0x50 && buf[3] === 0x53) { - return { - ext: "psd", - mime: "image/vnd.adobe.photoshop" - }; - } - - // needs to be before `zip` check - if (buf[0] === 0x50 && buf[1] === 0x4B && buf[2] === 0x3 && buf[3] === 0x4 && buf[30] === 0x6D && buf[31] === 0x69 && buf[32] === 0x6D && buf[33] === 0x65 && buf[34] === 0x74 && buf[35] === 0x79 && buf[36] === 0x70 && buf[37] === 0x65 && buf[38] === 0x61 && buf[39] === 0x70 && buf[40] === 0x70 && buf[41] === 0x6C && buf[42] === 0x69 && buf[43] === 0x63 && buf[44] === 0x61 && buf[45] === 0x74 && buf[46] === 0x69 && buf[47] === 0x6F && buf[48] === 0x6E && buf[49] === 0x2F && buf[50] === 0x65 && buf[51] === 0x70 && buf[52] === 0x75 && buf[53] === 0x62 && buf[54] === 0x2B && buf[55] === 0x7A && buf[56] === 0x69 && buf[57] === 0x70) { - return { - ext: "epub", - mime: "application/epub+zip" - }; - } - - if (buf[0] === 0x50 && buf[1] === 0x4B && (buf[2] === 0x3 || buf[2] === 0x5 || buf[2] === 0x7) && (buf[3] === 0x4 || buf[3] === 0x6 || buf[3] === 0x8)) { - return { - ext: "zip", - mime: "application/zip" - }; - } - - if (buf[257] === 0x75 && buf[258] === 0x73 && buf[259] === 0x74 && buf[260] === 0x61 && buf[261] === 0x72) { - return { - ext: "tar", - mime: "application/x-tar" - }; - } - - if (buf[0] === 0x52 && buf[1] === 0x61 && buf[2] === 0x72 && buf[3] === 0x21 && buf[4] === 0x1A && buf[5] === 0x7 && (buf[6] === 0x0 || buf[6] === 0x1)) { - return { - ext: "rar", - mime: "application/x-rar-compressed" - }; - } - - if (buf[0] === 0x1F && buf[1] === 0x8B && buf[2] === 0x8) { - return { - ext: "gz", - mime: "application/gzip" - }; - } - - if (buf[0] === 0x42 && buf[1] === 0x5A && buf[2] === 0x68) { - return { - ext: "bz2", - mime: "application/x-bzip2" - }; - } - - if (buf[0] === 0x37 && buf[1] === 0x7A && buf[2] === 0xBC && buf[3] === 0xAF && buf[4] === 0x27 && buf[5] === 0x1C) { - return { - ext: "7z", - mime: "application/x-7z-compressed" - }; - } - - if (buf[0] === 0x78 && buf[1] === 0x01) { - return { - ext: "dmg, zlib", - mime: "application/x-apple-diskimage, application/x-deflate" - }; - } - - if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && (buf[3] === 0x18 || buf[3] === 0x20) && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) || (buf[0] === 0x33 && buf[1] === 0x67 && buf[2] === 0x70 && buf[3] === 0x35) || (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x6D && buf[9] === 0x70 && buf[10] === 0x34 && buf[11] === 0x32 && buf[16] === 0x6D && buf[17] === 0x70 && buf[18] === 0x34 && buf[19] === 0x31 && buf[20] === 0x6D && buf[21] === 0x70 && buf[22] === 0x34 && buf[23] === 0x32 && buf[24] === 0x69 && buf[25] === 0x73 && buf[26] === 0x6F && buf[27] === 0x6D)) { - return { - ext: "mp4", - mime: "video/mp4" - }; - } - - if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x56)) { - return { - ext: "m4v", - mime: "video/x-m4v" - }; - } - - if (buf[0] === 0x4D && buf[1] === 0x54 && buf[2] === 0x68 && buf[3] === 0x64) { - return { - ext: "mid", - mime: "audio/midi" - }; - } - - // needs to be before the `webm` check - if (buf[31] === 0x6D && buf[32] === 0x61 && buf[33] === 0x74 && buf[34] === 0x72 && buf[35] === 0x6f && buf[36] === 0x73 && buf[37] === 0x6B && buf[38] === 0x61) { - return { - ext: "mkv", - mime: "video/x-matroska" - }; - } - - if (buf[0] === 0x1A && buf[1] === 0x45 && buf[2] === 0xDF && buf[3] === 0xA3) { - return { - ext: "webm", - mime: "video/webm" - }; - } - - if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x14 && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) { - return { - ext: "mov", - mime: "video/quicktime" - }; - } - - if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x41 && buf[9] === 0x56 && buf[10] === 0x49) { - return { - ext: "avi", - mime: "video/x-msvideo" - }; - } - - if (buf[0] === 0x30 && buf[1] === 0x26 && buf[2] === 0xB2 && buf[3] === 0x75 && buf[4] === 0x8E && buf[5] === 0x66 && buf[6] === 0xCF && buf[7] === 0x11 && buf[8] === 0xA6 && buf[9] === 0xD9) { - return { - ext: "wmv", - mime: "video/x-ms-wmv" - }; - } - - if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x1 && buf[3].toString(16)[0] === "b") { - return { - ext: "mpg", - mime: "video/mpeg" - }; - } - - if ((buf[0] === 0x49 && buf[1] === 0x44 && buf[2] === 0x33) || (buf[0] === 0xFF && buf[1] === 0xfb)) { - return { - ext: "mp3", - mime: "audio/mpeg" - }; - } - - if ((buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x41) || (buf[0] === 0x4D && buf[1] === 0x34 && buf[2] === 0x41 && buf[3] === 0x20)) { - return { - ext: "m4a", - mime: "audio/m4a" - }; - } - - if (buf[0] === 0x4F && buf[1] === 0x67 && buf[2] === 0x67 && buf[3] === 0x53) { - return { - ext: "ogg", - mime: "audio/ogg" - }; - } - - if (buf[0] === 0x66 && buf[1] === 0x4C && buf[2] === 0x61 && buf[3] === 0x43) { - return { - ext: "flac", - mime: "audio/x-flac" - }; - } - - if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x57 && buf[9] === 0x41 && buf[10] === 0x56 && buf[11] === 0x45) { - return { - ext: "wav", - mime: "audio/x-wav" - }; - } - - if (buf[0] === 0x23 && buf[1] === 0x21 && buf[2] === 0x41 && buf[3] === 0x4D && buf[4] === 0x52 && buf[5] === 0x0A) { - return { - ext: "amr", - mime: "audio/amr" - }; - } - - if (buf[0] === 0x25 && buf[1] === 0x50 && buf[2] === 0x44 && buf[3] === 0x46) { - return { - ext: "pdf", - mime: "application/pdf" - }; - } - - if (buf[0] === 0x4D && buf[1] === 0x5A) { - return { - ext: "exe", - mime: "application/x-msdownload" - }; - } - - if ((buf[0] === 0x43 || buf[0] === 0x46) && buf[1] === 0x57 && buf[2] === 0x53) { - return { - ext: "swf", - mime: "application/x-shockwave-flash" - }; - } - - if (buf[0] === 0x7B && buf[1] === 0x5C && buf[2] === 0x72 && buf[3] === 0x74 && buf[4] === 0x66) { - return { - ext: "rtf", - mime: "application/rtf" - }; - } - - if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x46 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { - return { - ext: "woff", - mime: "application/font-woff" - }; - } - - if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x32 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { - return { - ext: "woff2", - mime: "application/font-woff" - }; - } - - if (buf[34] === 0x4C && buf[35] === 0x50 && ((buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x01) || (buf[8] === 0x01 && buf[9] === 0x00 && buf[10] === 0x00) || (buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x02))) { - return { - ext: "eot", - mime: "application/octet-stream" - }; - } - - if (buf[0] === 0x00 && buf[1] === 0x01 && buf[2] === 0x00 && buf[3] === 0x00 && buf[4] === 0x00) { - return { - ext: "ttf", - mime: "application/font-sfnt" - }; - } - - if (buf[0] === 0x4F && buf[1] === 0x54 && buf[2] === 0x54 && buf[3] === 0x4F && buf[4] === 0x00) { - return { - ext: "otf", - mime: "application/font-sfnt" - }; - } - - if (buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0x01 && buf[3] === 0x00) { - return { - ext: "ico", - mime: "image/x-icon" - }; - } - - if (buf[0] === 0x46 && buf[1] === 0x4C && buf[2] === 0x56 && buf[3] === 0x01) { - return { - ext: "flv", - mime: "video/x-flv" - }; - } - - if (buf[0] === 0x25 && buf[1] === 0x21) { - return { - ext: "ps", - mime: "application/postscript" - }; - } - - if (buf[0] === 0xFD && buf[1] === 0x37 && buf[2] === 0x7A && buf[3] === 0x58 && buf[4] === 0x5A && buf[5] === 0x00) { - return { - ext: "xz", - mime: "application/x-xz" - }; - } - - if (buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4C && buf[3] === 0x69) { - return { - ext: "sqlite", - mime: "application/x-sqlite3" - }; - } - - /** - * - * Added by n1474335 [n1474335@gmail.com] from here on - * - */ - if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) { - return { - ext: "z, tar.z", - mime: "application/x-gtar" - }; - } - - if (buf[0] === 0x7F && buf[1] === 0x45 && buf[2] === 0x4C && buf[3] === 0x46) { - return { - ext: "none, axf, bin, elf, o, prx, puff, so", - mime: "application/x-executable", - desc: "Executable and Linkable Format file. No standard file extension." - }; - } - - if (buf[0] === 0xCA && buf[1] === 0xFE && buf[2] === 0xBA && buf[3] === 0xBE) { - return { - ext: "class", - mime: "application/java-vm" - }; - } - - if (buf[0] === 0xEF && buf[1] === 0xBB && buf[2] === 0xBF) { - return { - ext: "txt", - mime: "text/plain", - desc: "UTF-8 encoded Unicode byte order mark detected, commonly but not exclusively seen in text files." - }; - } - - // Must be before Little-endian UTF-16 BOM - if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) { - return { - ext: "UTF32LE", - mime: "charset/utf32le", - desc: "Little-endian UTF-32 encoded Unicode byte order mark detected." - }; - } - - if (buf[0] === 0xFF && buf[1] === 0xFE) { - return { - ext: "UTF16LE", - mime: "charset/utf16le", - desc: "Little-endian UTF-16 encoded Unicode byte order mark detected." - }; - } - - if ((buf[0x8001] === 0x43 && buf[0x8002] === 0x44 && buf[0x8003] === 0x30 && buf[0x8004] === 0x30 && buf[0x8005] === 0x31) || - (buf[0x8801] === 0x43 && buf[0x8802] === 0x44 && buf[0x8803] === 0x30 && buf[0x8804] === 0x30 && buf[0x8805] === 0x31) || - (buf[0x9001] === 0x43 && buf[0x9002] === 0x44 && buf[0x9003] === 0x30 && buf[0x9004] === 0x30 && buf[0x9005] === 0x31)) { - return { - ext: "iso", - mime: "application/octet-stream", - desc: "ISO 9660 CD/DVD image file" - }; - } - - if (buf[0] === 0xD0 && buf[1] === 0xCF && buf[2] === 0x11 && buf[3] === 0xE0 && buf[4] === 0xA1 && buf[5] === 0xB1 && buf[6] === 0x1A && buf[7] === 0xE1) { - return { - ext: "doc, xls, ppt", - mime: "application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint", - desc: "Microsoft Office documents" - }; - } - - if (buf[0] === 0x64 && buf[1] === 0x65 && buf[2] === 0x78 && buf[3] === 0x0A && buf[4] === 0x30 && buf[5] === 0x33 && buf[6] === 0x35 && buf[7] === 0x00) { - return { - ext: "dex", - mime: "application/octet-stream", - desc: "Dalvik Executable (Android)" - }; - } - - if (buf[0] === 0x4B && buf[1] === 0x44 && buf[2] === 0x4D) { - return { - ext: "vmdk", - mime: "application/vmdk, application/x-virtualbox-vmdk" - }; - } - - if (buf[0] === 0x43 && buf[1] === 0x72 && buf[2] === 0x32 && buf[3] === 0x34) { - return { - ext: "crx", - mime: "application/crx", - desc: "Google Chrome extension or packaged app" - }; - } - - if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) { - return { - ext: "zlib", - mime: "application/x-deflate" - }; - } - - return null; - } - } + /** * Byte frequencies of various languages generated from Wikipedia dumps taken in late 2017 and early 2018. * The Chi-Squared test cannot accept expected values of 0, so 0.0001 has been used to account for bytes diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs index 5f84e13c..19903117 100644 --- a/src/core/lib/Stream.mjs +++ b/src/core/lib/Stream.mjs @@ -11,13 +11,6 @@ /** * A Stream can be used to traverse a binary blob, interpreting sections of it * as various data types. - * - * @param {Uint8Array} bytes - * @param {Object} fileDetail - * @param {string} fileDetail.mime - * @param {string} fileDetail.ext - * @param {number} fileDetail.offset - * @returns {File} */ export default class Stream { diff --git a/src/core/operations/DetectFileType.mjs b/src/core/operations/DetectFileType.mjs index 1d6897a0..bcb4d2a5 100644 --- a/src/core/operations/DetectFileType.mjs +++ b/src/core/operations/DetectFileType.mjs @@ -5,7 +5,7 @@ */ import Operation from "../Operation"; -import Magic from "../lib/Magic"; +import {detectFileType} from "../lib/FileType"; /** * Detect File Type operation @@ -34,17 +34,21 @@ class DetectFileType extends Operation { */ run(input, args) { const data = new Uint8Array(input), - type = Magic.magicFileType(data); + types = detectFileType(data); - if (!type) { + if (!types.length) { return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?"; } else { - let output = "File extension: " + type.ext + "\n" + - "MIME type: " + type.mime; + let output; - if (type.desc && type.desc.length) { - output += "\nDescription: " + type.desc; - } + types.forEach(type => { + output += "File extension: " + type.extension + "\n" + + "MIME type: " + type.mime + "\n"; + + if (type.description && type.description.length) { + output += "\nDescription: " + type.description + "\n"; + } + }); return output; } diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index c213b256..3a87cd5e 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -6,9 +6,8 @@ import Operation from "../Operation"; // import OperationError from "../errors/OperationError"; -import Magic from "../lib/Magic"; import Utils from "../Utils"; -import {extractFile} from "../lib/FileExtraction"; +import {detectFileType, extractFile} from "../lib/FileType"; /** * Extract Files operation @@ -40,13 +39,13 @@ class ExtractFiles extends Operation { const bytes = new Uint8Array(input); // Scan for embedded files - const fileDetails = scanForEmbeddedFiles(bytes); + const detectedFiles = scanForEmbeddedFiles(bytes); // Extract each file that we support const files = []; - fileDetails.forEach(fileDetail => { + detectedFiles.forEach(detectedFile => { try { - files.push(extractFile(bytes, fileDetail)); + files.push(extractFile(bytes, detectedFile.fileDetails, detectedFile.offset)); } catch (err) {} }); @@ -70,22 +69,21 @@ class ExtractFiles extends Operation { * @param data */ function scanForEmbeddedFiles(data) { - let type; - const types = []; + const detectedFiles = []; for (let i = 0; i < data.length; i++) { - type = Magic.magicFileType(data.slice(i)); - if (type) { - types.push({ - offset: i, - ext: type.ext, - mime: type.mime, - desc: type.desc + const fileDetails = detectFileType(data.slice(i)); + if (fileDetails.length) { + fileDetails.forEach(match => { + detectedFiles.push({ + offset: i, + fileDetails: match, + }); }); } } - return types; + return detectedFiles; } export default ExtractFiles; diff --git a/src/core/operations/RenderImage.mjs b/src/core/operations/RenderImage.mjs index 7edd2072..4554c96b 100644 --- a/src/core/operations/RenderImage.mjs +++ b/src/core/operations/RenderImage.mjs @@ -9,7 +9,7 @@ import { fromHex } from "../lib/Hex"; import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import Magic from "../lib/Magic"; +import {detectFileType} from "../lib/FileType"; /** * Render Image operation @@ -72,8 +72,8 @@ class RenderImage extends Operation { } // Determine file type - const type = Magic.magicFileType(input); - if (!(type && type.mime.indexOf("image") === 0)) { + const types = detectFileType(input); + if (!(types.length && types[0].mime.indexOf("image") === 0)) { throw new OperationError("Invalid file type"); } @@ -92,9 +92,9 @@ class RenderImage extends Operation { let dataURI = "data:"; // Determine file type - const type = Magic.magicFileType(data); - if (type && type.mime.indexOf("image") === 0) { - dataURI += type.mime + ";"; + const types = detectFileType(data); + if (types.length && types[0].mime.indexOf("image") === 0) { + dataURI += types[0].mime + ";"; } else { throw new OperationError("Invalid file type"); } diff --git a/src/core/operations/ScanForEmbeddedFiles.mjs b/src/core/operations/ScanForEmbeddedFiles.mjs index a38c477f..41ea911b 100644 --- a/src/core/operations/ScanForEmbeddedFiles.mjs +++ b/src/core/operations/ScanForEmbeddedFiles.mjs @@ -6,7 +6,7 @@ import Operation from "../Operation"; import Utils from "../Utils"; -import Magic from "../lib/Magic"; +import {detectFileType} from "../lib/FileType"; /** * Scan for Embedded Files operation @@ -41,7 +41,7 @@ class ScanForEmbeddedFiles extends Operation { */ run(input, args) { let output = "Scanning data for 'magic bytes' which may indicate embedded files. The following results may be false positives and should not be treat as reliable. Any suffiently long file is likely to contain these magic bytes coincidentally.\n", - type, + types, numFound = 0, numCommonFound = 0; const ignoreCommon = args[0], @@ -49,20 +49,23 @@ class ScanForEmbeddedFiles extends Operation { data = new Uint8Array(input); for (let i = 0; i < data.length; i++) { - type = Magic.magicFileType(data.slice(i)); - if (type) { - if (ignoreCommon && commonExts.indexOf(type.ext) > -1) { - numCommonFound++; - continue; - } - numFound++; - output += "\nOffset " + i + " (0x" + Utils.hex(i) + "):\n" + - " File extension: " + type.ext + "\n" + - " MIME type: " + type.mime + "\n"; + types = detectFileType(data.slice(i)); + if (types.length) { + types.forEach(type => { + if (ignoreCommon && commonExts.indexOf(type.extension) > -1) { + numCommonFound++; + return; + } - if (type.desc && type.desc.length) { - output += " Description: " + type.desc + "\n"; - } + numFound++; + output += "\nOffset " + i + " (0x" + Utils.hex(i) + "):\n" + + " File extension: " + type.extension + "\n" + + " MIME type: " + type.mime + "\n"; + + if (type.description && type.description.length) { + output += " Description: " + type.description + "\n"; + } + }); } } From 8d3836cb16453d969d6d8a1e25f5e650e8d10394 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 21 Dec 2018 12:48:08 +0000 Subject: [PATCH 03/22] Added support for a number of further file types and file detection methods. --- src/core/lib/FileType.mjs | 256 +++++++++++++++++-------- src/core/operations/DetectFileType.mjs | 2 +- src/core/operations/PlayMedia.mjs | 14 +- 3 files changed, 189 insertions(+), 83 deletions(-) diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index 443bdd36..d38ddc80 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -13,7 +13,7 @@ import Stream from "./Stream"; * to extract them where possible. */ const FILE_SIGNATURES = { - "Pictures": [ + "Images": [ { name: "JPEG Image", extension: "jpg", @@ -51,7 +51,165 @@ const FILE_SIGNATURES = { }, extractor: null }, - + { + name: "WEBP Image", + extension: "webp", + mime: "image/webp", + description: "", + signature: { + 8: 0x57, + 9: 0x45, + 10: 0x42, + 11: 0x50 + }, + extractor: null + }, + { + name: "TIFF Image", + extension: "tif", + mime: "image/tiff", + description: "", + signature: [ + { + 0: 0x49, + 1: 0x49, + 2: 0x2a, + 3: 0x0 + }, + { + 0: 0x4d, + 1: 0x4d, + 2: 0x0, + 3: 0x2a + } + ], + extractor: null + }, /* + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + }, + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + }, + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + }, + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + }, + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + }, + { + name: " Image", + extension: "", + mime: "image/", + description: "", + signature: { + 0: 0x, + 1: 0x, + 2: 0x, + 3: 0x + }, + extractor: null + },*/ + ], + "Video": [ + { + name: "WEBM", + extension: "webm", + mime: "video/webm", + description: "", + signature: { + 0: 0x1a, + 1: 0x45, + 2: 0xdf, + 3: 0xa3 + }, + extractor: null + }, + ], + "Audio": [ + { + name: "WAV", + extension: "wav", + mime: "audio/x-wav", + description: "", + signature: { + 0: 0x52, + 1: 0x49, + 2: 0x46, + 3: 0x46, + 8: 0x57, + 9: 0x41, + 10: 0x56, + 11: 0x45 + }, + extractor: null + }, + { + name: "OGG", + extension: "ogg", + mime: "audio/ogg", + description: "", + signature: { + 0: 0x4f, + 1: 0x67, + 2: 0x67, + 3: 0x53 + }, + extractor: null + }, ], "Documents": [ { @@ -103,13 +261,31 @@ const FILE_SIGNATURES = { /** * Checks whether a signature matches a buffer. * - * @param {Object} sig - A dictionary of offsets with values assigned to them. These - * values can be numbers for static checks, arrays of potential valid matches, or - * bespoke functions to check the validity of the buffer value at that offset. + * @param {Object|Object[]} sig - A dictionary of offsets with values assigned to them. + * These values can be numbers for static checks, arrays of potential valid matches, + * or bespoke functions to check the validity of the buffer value at that offset. * @param {Uint8Array} buf * @returns {boolean} */ function signatureMatches(sig, buf) { + if (sig instanceof Array) { + return sig.reduce((acc, s) => acc || bytesMatch(s, buf), false); + } else { + return bytesMatch(sig, buf); + } +} + + +/** + * Checks whether a set of bytes match the given buffer. + * + * @param {Object} sig - A dictionary of offsets with values assigned to them. + * These values can be numbers for static checks, arrays of potential valid matches, + * or bespoke functions to check the validity of the buffer value at that offset. + * @param {Uint8Array} buf + * @returns {boolean} + */ +function bytesMatch(sig, buf) { for (const offset in sig) { switch (typeof sig[offset]) { case "number": // Static check @@ -165,34 +341,6 @@ export function detectFileType(buf) { /* - if (buf[0] === 0xFF && buf[1] === 0xD8 && buf[2] === 0xFF) { - return { - ext: "jpg", - mime: "image/jpeg" - }; - } - - if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4E && buf[3] === 0x47) { - return { - ext: "png", - mime: "image/png" - }; - } - - if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) { - return { - ext: "gif", - mime: "image/gif" - }; - } - - if (buf[8] === 0x57 && buf[9] === 0x45 && buf[10] === 0x42 && buf[11] === 0x50) { - return { - ext: "webp", - mime: "image/webp" - }; - } - // needs to be before `tif` check if (((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) && buf[8] === 0x43 && buf[9] === 0x52) { return { @@ -237,13 +385,6 @@ export function detectFileType(buf) { }; } - if (buf[0] === 0x50 && buf[1] === 0x4B && (buf[2] === 0x3 || buf[2] === 0x5 || buf[2] === 0x7) && (buf[3] === 0x4 || buf[3] === 0x6 || buf[3] === 0x8)) { - return { - ext: "zip", - mime: "application/zip" - }; - } - if (buf[257] === 0x75 && buf[258] === 0x73 && buf[259] === 0x74 && buf[260] === 0x61 && buf[261] === 0x72) { return { ext: "tar", @@ -315,13 +456,6 @@ export function detectFileType(buf) { }; } - if (buf[0] === 0x1A && buf[1] === 0x45 && buf[2] === 0xDF && buf[3] === 0xA3) { - return { - ext: "webm", - mime: "video/webm" - }; - } - if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x14 && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) { return { ext: "mov", @@ -364,13 +498,6 @@ export function detectFileType(buf) { }; } - if (buf[0] === 0x4F && buf[1] === 0x67 && buf[2] === 0x67 && buf[3] === 0x53) { - return { - ext: "ogg", - mime: "audio/ogg" - }; - } - if (buf[0] === 0x66 && buf[1] === 0x4C && buf[2] === 0x61 && buf[3] === 0x43) { return { ext: "flac", @@ -378,13 +505,6 @@ export function detectFileType(buf) { }; } - if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x57 && buf[9] === 0x41 && buf[10] === 0x56 && buf[11] === 0x45) { - return { - ext: "wav", - mime: "audio/x-wav" - }; - } - if (buf[0] === 0x23 && buf[1] === 0x21 && buf[2] === 0x41 && buf[3] === 0x4D && buf[4] === 0x52 && buf[5] === 0x0A) { return { ext: "amr", @@ -392,20 +512,6 @@ export function detectFileType(buf) { }; } - if (buf[0] === 0x25 && buf[1] === 0x50 && buf[2] === 0x44 && buf[3] === 0x46) { - return { - ext: "pdf", - mime: "application/pdf" - }; - } - - if (buf[0] === 0x4D && buf[1] === 0x5A) { - return { - ext: "exe", - mime: "application/x-msdownload" - }; - } - if ((buf[0] === 0x43 || buf[0] === 0x46) && buf[1] === 0x57 && buf[2] === 0x53) { return { ext: "swf", diff --git a/src/core/operations/DetectFileType.mjs b/src/core/operations/DetectFileType.mjs index bcb4d2a5..55db5edf 100644 --- a/src/core/operations/DetectFileType.mjs +++ b/src/core/operations/DetectFileType.mjs @@ -39,7 +39,7 @@ class DetectFileType extends Operation { if (!types.length) { return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?"; } else { - let output; + let output = ""; types.forEach(type => { output += "File extension: " + type.extension + "\n" + diff --git a/src/core/operations/PlayMedia.mjs b/src/core/operations/PlayMedia.mjs index 81328a73..d0ec78cc 100644 --- a/src/core/operations/PlayMedia.mjs +++ b/src/core/operations/PlayMedia.mjs @@ -9,7 +9,7 @@ import { fromHex } from "../lib/Hex"; import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import Magic from "../lib/Magic"; +import { detectFileType } from "../lib/FileType"; /** * PlayMedia operation @@ -66,8 +66,8 @@ class PlayMedia extends Operation { // Determine file type - const type = Magic.magicFileType(input); - if (!(type && /^audio|video/.test(type.mime))) { + const types = detectFileType(input); + if (!(types && types.length && /^audio|video/.test(types[0].mime))) { throw new OperationError("Invalid or unrecognised file type"); } @@ -84,15 +84,15 @@ class PlayMedia extends Operation { async present(data) { if (!data.length) return ""; - const type = Magic.magicFileType(data); - const matches = /^audio|video/.exec(type.mime); + const types = detectFileType(data); + const matches = /^audio|video/.exec(types[0].mime); if (!matches) { throw new OperationError("Invalid file type"); } - const dataURI = `data:${type.mime};base64,${toBase64(data)}`; + const dataURI = `data:${types[0].mime};base64,${toBase64(data)}`; const element = matches[0]; - let html = `<${element} src='${dataURI}' type='${type.mime}' controls>`; + let html = `<${element} src='${dataURI}' type='${types[0].mime}' controls>`; html += "

Unsupported media type.

"; html += ``; return html; From f4f9b5c91c95f776c2b48a4c0733416af908e63b Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 26 Dec 2018 18:40:27 +0000 Subject: [PATCH 04/22] Added 'isImage' and 'isType' functions --- src/core/lib/FileType.mjs | 33 ++++++++ src/core/operations/GenerateQRCode.mjs | 8 +- src/core/operations/ParseQRCode.mjs | 89 ++++++++++----------- src/core/operations/PlayMedia.mjs | 5 +- src/core/operations/RenderImage.mjs | 11 ++- src/core/operations/SplitColourChannels.mjs | 89 ++++++++++----------- 6 files changed, 130 insertions(+), 105 deletions(-) diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index d38ddc80..ef6cfb03 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -703,6 +703,39 @@ export function detectFileType(buf) { } +/** + * Detects whether the given buffer is a file of the type specified. + * + * @param {string|RegExp} type + * @param {Uint8Array} buf + * @returns {string|false} The mime type or false if the type does not match + */ +export function isType(type, buf) { + const types = detectFileType(buf); + + if (!(types && types.length)) return false; + + if (typeof type === "string") { + return types[0].mime.startsWith(type) ? types[0].mime : false; + } else if (type instanceof RegExp) { + return type.test(types[0].mime) ? types[0].mime : false; + } else { + throw new Error("Invalid type input."); + } +} + + +/** + * Detects whether the given buffer contains an image file. + * + * @param {Uint8Array} buf + * @returns {string|false} The mime type or false if the type does not match + */ +export function isImage(buf) { + return isType("image", buf); +} + + /** * Attempts to extract a file from a data stream given its offset and extractor function. * diff --git a/src/core/operations/GenerateQRCode.mjs b/src/core/operations/GenerateQRCode.mjs index edab6d40..ac7e5c5c 100644 --- a/src/core/operations/GenerateQRCode.mjs +++ b/src/core/operations/GenerateQRCode.mjs @@ -8,7 +8,7 @@ import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import qr from "qr-image"; import { toBase64 } from "../lib/Base64"; -import Magic from "../lib/Magic"; +import { isImage } from "../lib/FileType"; import Utils from "../Utils"; /** @@ -100,9 +100,9 @@ class GenerateQRCode extends Operation { if (format === "PNG") { let dataURI = "data:"; - const type = Magic.magicFileType(data); - if (type && type.mime.indexOf("image") === 0){ - dataURI += type.mime + ";"; + const mime = isImage(data); + if (mime){ + dataURI += mime + ";"; } else { throw new OperationError("Invalid PNG file generated by QR image"); } diff --git a/src/core/operations/ParseQRCode.mjs b/src/core/operations/ParseQRCode.mjs index 75a24d55..ef7af6d7 100644 --- a/src/core/operations/ParseQRCode.mjs +++ b/src/core/operations/ParseQRCode.mjs @@ -6,7 +6,7 @@ import Operation from "../Operation"; import OperationError from "../errors/OperationError"; -import Magic from "../lib/Magic"; +import { isImage } from "../lib/FileType"; import jsqr from "jsqr"; import jimp from "jimp"; @@ -42,64 +42,61 @@ class ParseQRCode extends Operation { * @returns {string} */ async run(input, args) { - const type = Magic.magicFileType(input); const [normalise] = args; // Make sure that the input is an image - if (type && type.mime.indexOf("image") === 0) { - let image = input; + if (!isImage(input)) throw new OperationError("Invalid file type."); - if (normalise) { - // Process the image to be easier to read by jsqr - // Disables the alpha channel - // Sets the image default background to white - // Normalises the image colours - // Makes the image greyscale - // Converts image to a JPEG - image = await new Promise((resolve, reject) => { - jimp.read(Buffer.from(input)) - .then(image => { - image - .rgba(false) - .background(0xFFFFFFFF) - .normalize() - .greyscale() - .getBuffer(jimp.MIME_JPEG, (error, result) => { - resolve(result); - }); - }) - .catch(err => { - reject(new OperationError("Error reading the image file.")); - }); - }); - } + let image = input; - if (image instanceof OperationError) { - throw image; - } - - return new Promise((resolve, reject) => { - jimp.read(Buffer.from(image)) + if (normalise) { + // Process the image to be easier to read by jsqr + // Disables the alpha channel + // Sets the image default background to white + // Normalises the image colours + // Makes the image greyscale + // Converts image to a JPEG + image = await new Promise((resolve, reject) => { + jimp.read(Buffer.from(input)) .then(image => { - if (image.bitmap != null) { - const qrData = jsqr(image.bitmap.data, image.getWidth(), image.getHeight()); - if (qrData != null) { - resolve(qrData.data); - } else { - reject(new OperationError("Couldn't read a QR code from the image.")); - } - } else { - reject(new OperationError("Error reading the image file.")); - } + image + .rgba(false) + .background(0xFFFFFFFF) + .normalize() + .greyscale() + .getBuffer(jimp.MIME_JPEG, (error, result) => { + resolve(result); + }); }) .catch(err => { reject(new OperationError("Error reading the image file.")); }); }); - } else { - throw new OperationError("Invalid file type."); } + if (image instanceof OperationError) { + throw image; + } + + return new Promise((resolve, reject) => { + jimp.read(Buffer.from(image)) + .then(image => { + if (image.bitmap != null) { + const qrData = jsqr(image.bitmap.data, image.getWidth(), image.getHeight()); + if (qrData != null) { + resolve(qrData.data); + } else { + reject(new OperationError("Couldn't read a QR code from the image.")); + } + } else { + reject(new OperationError("Error reading the image file.")); + } + }) + .catch(err => { + reject(new OperationError("Error reading the image file.")); + }); + }); + } } diff --git a/src/core/operations/PlayMedia.mjs b/src/core/operations/PlayMedia.mjs index d0ec78cc..98b7d088 100644 --- a/src/core/operations/PlayMedia.mjs +++ b/src/core/operations/PlayMedia.mjs @@ -9,7 +9,7 @@ import { fromHex } from "../lib/Hex"; import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import { detectFileType } from "../lib/FileType"; +import { isType, detectFileType } from "../lib/FileType"; /** * PlayMedia operation @@ -66,8 +66,7 @@ class PlayMedia extends Operation { // Determine file type - const types = detectFileType(input); - if (!(types && types.length && /^audio|video/.test(types[0].mime))) { + if (!isType(/^(audio|video)/, input)) { throw new OperationError("Invalid or unrecognised file type"); } diff --git a/src/core/operations/RenderImage.mjs b/src/core/operations/RenderImage.mjs index 4554c96b..07866eaf 100644 --- a/src/core/operations/RenderImage.mjs +++ b/src/core/operations/RenderImage.mjs @@ -9,7 +9,7 @@ import { fromHex } from "../lib/Hex"; import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import {detectFileType} from "../lib/FileType"; +import {isImage} from "../lib/FileType"; /** * Render Image operation @@ -72,8 +72,7 @@ class RenderImage extends Operation { } // Determine file type - const types = detectFileType(input); - if (!(types.length && types[0].mime.indexOf("image") === 0)) { + if (!isImage(input)) { throw new OperationError("Invalid file type"); } @@ -92,9 +91,9 @@ class RenderImage extends Operation { let dataURI = "data:"; // Determine file type - const types = detectFileType(data); - if (types.length && types[0].mime.indexOf("image") === 0) { - dataURI += types[0].mime + ";"; + const mime = isImage(data); + if (mime) { + dataURI += mime + ";"; } else { throw new OperationError("Invalid file type"); } diff --git a/src/core/operations/SplitColourChannels.mjs b/src/core/operations/SplitColourChannels.mjs index f11ca14e..c38af409 100644 --- a/src/core/operations/SplitColourChannels.mjs +++ b/src/core/operations/SplitColourChannels.mjs @@ -7,7 +7,7 @@ import Operation from "../Operation"; import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import Magic from "../lib/Magic"; +import {isImage} from "../lib/FileType"; import jimp from "jimp"; @@ -38,56 +38,53 @@ class SplitColourChannels extends Operation { * @returns {List} */ async run(input, args) { - const type = Magic.magicFileType(input); // Make sure that the input is an image - if (type && type.mime.indexOf("image") === 0) { - const parsedImage = await jimp.read(Buffer.from(input)); + if (!isImage(input)) throw new OperationError("Invalid file type."); - const red = new Promise(async (resolve, reject) => { - try { - const split = parsedImage - .clone() - .color([ - {apply: "blue", params: [-255]}, - {apply: "green", params: [-255]} - ]) - .getBufferAsync(jimp.MIME_PNG); - resolve(new File([new Uint8Array((await split).values())], "red.png", {type: "image/png"})); - } catch (err) { - reject(new OperationError(`Could not split red channel: ${err}`)); - } - }); + const parsedImage = await jimp.read(Buffer.from(input)); - const green = new Promise(async (resolve, reject) => { - try { - const split = parsedImage.clone() - .color([ - {apply: "red", params: [-255]}, - {apply: "blue", params: [-255]}, - ]).getBufferAsync(jimp.MIME_PNG); - resolve(new File([new Uint8Array((await split).values())], "green.png", {type: "image/png"})); - } catch (err) { - reject(new OperationError(`Could not split green channel: ${err}`)); - } - }); + const red = new Promise(async (resolve, reject) => { + try { + const split = parsedImage + .clone() + .color([ + {apply: "blue", params: [-255]}, + {apply: "green", params: [-255]} + ]) + .getBufferAsync(jimp.MIME_PNG); + resolve(new File([new Uint8Array((await split).values())], "red.png", {type: "image/png"})); + } catch (err) { + reject(new OperationError(`Could not split red channel: ${err}`)); + } + }); - const blue = new Promise(async (resolve, reject) => { - try { - const split = parsedImage - .color([ - {apply: "red", params: [-255]}, - {apply: "green", params: [-255]}, - ]).getBufferAsync(jimp.MIME_PNG); - resolve(new File([new Uint8Array((await split).values())], "blue.png", {type: "image/png"})); - } catch (err) { - reject(new OperationError(`Could not split blue channel: ${err}`)); - } - }); + const green = new Promise(async (resolve, reject) => { + try { + const split = parsedImage.clone() + .color([ + {apply: "red", params: [-255]}, + {apply: "blue", params: [-255]}, + ]).getBufferAsync(jimp.MIME_PNG); + resolve(new File([new Uint8Array((await split).values())], "green.png", {type: "image/png"})); + } catch (err) { + reject(new OperationError(`Could not split green channel: ${err}`)); + } + }); - return await Promise.all([red, green, blue]); - } else { - throw new OperationError("Invalid file type."); - } + const blue = new Promise(async (resolve, reject) => { + try { + const split = parsedImage + .color([ + {apply: "red", params: [-255]}, + {apply: "green", params: [-255]}, + ]).getBufferAsync(jimp.MIME_PNG); + resolve(new File([new Uint8Array((await split).values())], "blue.png", {type: "image/png"})); + } catch (err) { + reject(new OperationError(`Could not split blue channel: ${err}`)); + } + }); + + return await Promise.all([red, green, blue]); } /** From 729307336e58df5beb97221e3e58bc1a2eedd118 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 26 Dec 2018 23:19:46 +0000 Subject: [PATCH 05/22] Converted all previous file signatures to the new format. --- src/core/lib/FileSignatures.mjs | 1136 +++++++++++++++++++++++++++++++ src/core/lib/FileType.mjs | 811 +--------------------- 2 files changed, 1141 insertions(+), 806 deletions(-) create mode 100644 src/core/lib/FileSignatures.mjs diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs new file mode 100644 index 00000000..69699d7d --- /dev/null +++ b/src/core/lib/FileSignatures.mjs @@ -0,0 +1,1136 @@ +/** + * File signatures and extractor functions + * + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + * + */ +import Stream from "./Stream"; + +/** + * A categorised table of file types, including signatures to identify them and functions + * to extract them where possible. + */ +export const FILE_SIGNATURES = { + "Images": [ + { + name: "Joint Photographic Experts Group image", + extension: "jpg", + mime: "image/jpeg", + description: "", + signature: { + 0: 0xff, + 1: 0xd8, + 2: 0xff + }, + extractor: extractJPEG + }, + { + name: "Graphics Interchange Format image", + extension: "gif", + mime: "image/gif", + description: "", + signature: { + 0: 0x47, + 1: 0x49, + 2: 0x46 + }, + extractor: null + }, + { + name: "Portable Network Graphics image", + extension: "png", + mime: "image/png", + description: "", + signature: { + 0: 0x89, + 1: 0x50, + 2: 0x4e, + 3: 0x47 + }, + extractor: null + }, + { + name: "WEBP Image", + extension: "webp", + mime: "image/webp", + description: "", + signature: { + 8: 0x57, + 9: 0x45, + 10: 0x42, + 11: 0x50 + }, + extractor: null + }, + { // Place before tiff check + name: "Canon CR2 raw image", + extension: "cr2", + mime: "image/x-canon-cr2", + description: "", + signature: [ + { + 0: 0x49, + 1: 0x49, + 2: 0x2a, + 3: 0x0, + 8: 0x43, + 9: 0x52 + }, + { + 0: 0x4d, + 1: 0x4d, + 2: 0x0, + 3: 0x2a, + 8: 0x43, + 9: 0x52 + } + ], + extractor: null + }, + { + name: "Tagged Image File Format image", + extension: "tif", + mime: "image/tiff", + description: "", + signature: [ + { + 0: 0x49, + 1: 0x49, + 2: 0x2a, + 3: 0x0 + }, + { + 0: 0x4d, + 1: 0x4d, + 2: 0x0, + 3: 0x2a + } + ], + extractor: null + }, + { + name: "Bitmap image", + extension: "bmp", + mime: "image/bmp", + description: "", + signature: { + 0: 0x42, + 1: 0x4d + }, + extractor: null + }, + { + name: "JPEG Extended Range image", + extension: "jxr", + mime: "image/vnd.ms-photo", + description: "", + signature: { + 0: 0x49, + 1: 0x49, + 2: 0xbc + }, + extractor: null + }, + { + name: "Photoshop image", + extension: "psd", + mime: "image/vnd.adobe.photoshop", + description: "", + signature: { + 0: 0x38, + 1: 0x42, + 2: 0x50, + 3: 0x53 + }, + extractor: null + }, + { + name: "Icon image", + extension: "ico", + mime: "image/x-icon", + description: "", + signature: { + 0: 0x0, + 1: 0x0, + 2: 0x1, + 3: 0x0 + }, + extractor: null + } + ], + "Video": [ + { // Place before webm + name: "Matroska Multimedia Container", + extension: "mkv", + mime: "video/x-matroska", + description: "", + signature: { + 31: 0x6d, + 32: 0x61, + 33: 0x74, + 34: 0x72, + 35: 0x6f, + 36: 0x73, + 37: 0x6b, + 38: 0x61 + }, + extractor: null + }, + { + name: "WEBM video", + extension: "webm", + mime: "video/webm", + description: "", + signature: { + 0: 0x1a, + 1: 0x45, + 2: 0xdf, + 3: 0xa3 + }, + extractor: null + }, + { + name: "MPEG-4 video", + extension: "mp4", + mime: "video/mp4", + description: "", + signature: [ + { + 0: 0x0, + 1: 0x0, + 2: 0x0, + 3: [0x18, 0x20], + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70 + }, + { + 0: 0x33, // 3gp5 + 1: 0x67, + 2: 0x70, + 3: 0x35 + }, + { + 0: 0x0, + 1: 0x0, + 2: 0x0, + 3: 0x1c, + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70, + 8: 0x6d, + 9: 0x70, + 10: 0x34, + 11: 0x32, + 16: 0x6d, // mp41mp42isom + 17: 0x70, + 18: 0x34, + 19: 0x31, + 20: 0x6d, + 21: 0x70, + 22: 0x34, + 23: 0x32, + 24: 0x69, + 25: 0x73, + 26: 0x6f, + 27: 0x6d + } + ], + extractor: null + }, + { + name: "M4V video", + extension: "m4v", + mime: "video/x-m4v", + description: "", + signature: { + 0: 0x0, + 1: 0x0, + 2: 0x0, + 3: 0x1c, + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70, + 8: 0x4d, + 9: 0x34, + 10: 0x56 + }, + extractor: null + }, + { + name: "Quicktime video", + extension: "mov", + mime: "video/quicktime", + description: "", + signature: { + 0: 0x0, + 1: 0x0, + 2: 0x0, + 3: 0x14, + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70 + }, + extractor: null + }, + { + name: "Audio Video Interleave", + extension: "avi", + mime: "video/x-msvideo", + description: "", + signature: { + 0: 0x52, + 1: 0x49, + 2: 0x46, + 3: 0x46, + 8: 0x41, + 9: 0x56, + 10: 0x49 + }, + extractor: null + }, + { + name: "Windows Media Video", + extension: "wmv", + mime: "video/x-ms-wmv", + description: "", + signature: { + 0: 0x30, + 1: 0x26, + 2: 0xb2, + 3: 0x75, + 4: 0x8e, + 5: 0x66, + 6: 0xcf, + 7: 0x11, + 8: 0xa6, + 9: 0xd9 + }, + extractor: null + }, + { + name: "MPEG video", + extension: "mpg", + mime: "video/mpeg", + description: "", + signature: { + 0: 0x0, + 1: 0x0, + 2: 0x1, + 3: 0xba + }, + extractor: null + }, + { + name: "Flash Video", + extension: "flv", + mime: "video/x-flv", + description: "", + signature: { + 0: 0x46, + 1: 0x4c, + 2: 0x56, + 3: 0x1 + }, + extractor: null + }, + ], + "Audio": [ + { + name: "Waveform Audio", + extension: "wav", + mime: "audio/x-wav", + description: "", + signature: { + 0: 0x52, + 1: 0x49, + 2: 0x46, + 3: 0x46, + 8: 0x57, + 9: 0x41, + 10: 0x56, + 11: 0x45 + }, + extractor: null + }, + { + name: "OGG audio", + extension: "ogg", + mime: "audio/ogg", + description: "", + signature: { + 0: 0x4f, + 1: 0x67, + 2: 0x67, + 3: 0x53 + }, + extractor: null + }, + { + name: "Musical Instrument Digital Interface audio", + extension: "midi", + mime: "audio/midi", + description: "", + signature: { + 0: 0x4d, + 1: 0x54, + 2: 0x68, + 3: 0x64 + }, + extractor: null + }, + { + name: "MPEG-3 audio", + extension: "mp3", + mime: "audio/mpeg", + description: "", + signature: [ + { + 0: 0x49, + 1: 0x44, + 2: 0x33 + }, + { + 0: 0xff, + 1: 0xfb + } + ], + extractor: null + }, + { + name: "MPEG-4 Part 14 audio", + extension: "m4a", + mime: "audio/m4a", + description: "", + signature: [ + { + 4: 0x66, + 5: 0x74, + 6: 0x79, + 7: 0x70, + 8: 0x4d, + 9: 0x34, + 10: 0x41 + }, + { + 0: 0x4d, + 1: 0x34, + 2: 0x41, + 3: 0x20 + } + ], + extractor: null + }, + { + name: "Free Lossless Audio Codec", + extension: "flac", + mime: "audio/x-flac", + description: "", + signature: { + 0: 0x66, + 1: 0x4c, + 2: 0x61, + 3: 0x43 + }, + extractor: null + }, + { + name: "Adaptive Multi-Rate audio codec", + extension: "amr", + mime: "audio/amr", + description: "", + signature: { + 0: 0x23, + 1: 0x21, + 2: 0x41, + 3: 0x4d, + 4: 0x52, + 5: 0x0a + }, + extractor: null + }, + ], + "Documents": [ + { + name: "Portable Document Format", + extension: "pdf", + mime: "application/pdf", + description: "", + signature: { + 0: 0x25, + 1: 0x50, + 2: 0x44, + 3: 0x46 + }, + extractor: extractPDF + }, + { + name: "PostScript", + extension: "ps", + mime: "application/postscript", + description: "", + signature: { + 0: 0x25, + 1: 0x21 + }, + extractor: null + }, + { + name: "Rich Text Format", + extension: "rtf", + mime: "application/rtf", + description: "", + signature: { + 0: 0x7b, + 1: 0x5c, + 2: 0x72, + 3: 0x74, + 4: 0x66 + }, + extractor: null + }, + { + name: "Microsoft Office documents/OLE2", + extension: "ole2,doc,xls,dot,ppt,xla,ppa,pps,pot,msi,sdw,db,vsd,msg", + mime: "application/msword,application/vnd.ms-excel,application/vnd.ms-powerpoint", + description: "Microsoft Office documents", + signature: { + 0: 0xd0, + 1: 0xcf, + 2: 0x11, + 3: 0xe0, + 4: 0xa1, + 5: 0xb1, + 6: 0x1a, + 7: 0xe1 + }, + extractor: null + }, + { + name: "EPUB e-book", + extension: "epub", + mime: "application/epub+zip", + description: "", + signature: { + 0: 0x50, + 1: 0x4b, + 2: 0x3, + 3: 0x4, + 30: 0x6d, // mimetypeapplication/epub_zip + 31: 0x69, + 32: 0x6d, + 33: 0x65, + 34: 0x74, + 35: 0x79, + 36: 0x70, + 37: 0x65, + 38: 0x61, + 39: 0x70, + 40: 0x70, + 41: 0x6c, + 42: 0x69, + 43: 0x63, + 44: 0x61, + 45: 0x74, + 46: 0x69, + 47: 0x6f, + 48: 0x6e, + 49: 0x2f, + 50: 0x65, + 51: 0x70, + 52: 0x75, + 53: 0x62, + 54: 0x2b, + 55: 0x7a, + 56: 0x69, + 57: 0x70 + }, + extractor: null + }, + ], + "Applications": [ + { + name: "Windows Portable Executable", + extension: "exe,dll,drv,vxd,sys,ocx,vbx,com,fon,scr", + mime: "application/x-msdownload", + description: "", + signature: { + 0: 0x4d, + 1: 0x5a, + 3: [0x0, 0x1, 0x2], + 5: [0x0, 0x1, 0x2] + }, + extractor: extractMZPE + }, + { + name: "Executable and Linkable Format file", + extension: "elf,bin,axf,o,prx,so", + mime: "application/x-executable", + description: "Executable and Linkable Format file. No standard file extension.", + signature: { + 0: 0x7f, + 1: 0x45, + 2: 0x4c, + 3: 0x46 + }, + extractor: null + }, + { + name: "Adobe Flash", + extension: "swf", + mime: "application/x-shockwave-flash", + description: "", + signature: { + 0: [0x43, 0x46], + 1: 0x57, + 2: 0x53 + }, + extractor: null + }, + { + name: "Java Class", + extension: "class", + mime: "application/java-vm", + description: "", + signature: { + 0: 0xca, + 1: 0xfe, + 2: 0xba, + 3: 0xbe + }, + extractor: null + }, + { + name: "Dalvik Executable", + extension: "dex", + mime: "application/octet-stream", + description: "Dalvik Executable as used by Android", + signature: { + 0: 0x64, + 1: 0x65, + 2: 0x78, + 3: 0x0a, + 4: 0x30, + 5: 0x33, + 6: 0x35, + 7: 0x0 + }, + extractor: null + }, + { + name: "Google Chrome Extension", + extension: "crx", + mime: "application/crx", + description: "Google Chrome extension or packaged app", + signature: { + 0: 0x43, + 1: 0x72, + 2: 0x32, + 3: 0x34 + }, + extractor: null + }, + ], + "Archives": [ + { + name: "PKZIP archive", + extension: "zip", + mime: "application/zip", + description: "", + signature: { + 0: 0x50, + 1: 0x4b, + 2: [0x3, 0x5, 0x7], + 3: [0x4, 0x6, 0x8] + }, + extractor: extractZIP + }, + { + name: "TAR archive", + extension: "tar", + mime: "application/x-tar", + description: "", + signature: { + 257: 0x75, + 258: 0x73, + 259: 0x74, + 260: 0x61, + 261: 0x72 + }, + extractor: null + }, + { + name: "Roshal Archive", + extension: "rar", + mime: "application/x-rar-compressed", + description: "", + signature: { + 0: 0x52, + 1: 0x61, + 2: 0x72, + 3: 0x21, + 4: 0x1a, + 5: 0x7, + 6: [0x0, 0x1] + }, + extractor: null + }, + { + name: "Gzip", + extension: "gz", + mime: "application/gzip", + description: "", + signature: { + 0: 0x1f, + 1: 0x8b, + 2: 0x8 + }, + extractor: null + }, + { + name: "Bzip2", + extension: "bz2", + mime: "application/x-bzip2", + description: "", + signature: { + 0: 0x42, + 1: 0x5a, + 2: 0x68 + }, + extractor: null + }, + { + name: "7zip", + extension: "7z", + mime: "application/x-7z-compressed", + description: "", + signature: { + 0: 0x37, + 1: 0x7a, + 2: 0xbc, + 3: 0xaf, + 4: 0x27, + 5: 0x1c + }, + extractor: null + }, + { + name: "Zlib Deflate", + extension: "zlib", + mime: "application/x-deflate", + description: "", + signature: { + 0: 0x78, + 1: [0x1, 0x9c, 0xda, 0x5e] + }, + extractor: null + }, + { + name: "xz compression", + extension: "xz", + mime: "application/x-xz", + description: "", + signature: { + 0: 0xfd, + 1: 0x37, + 2: 0x7a, + 3: 0x58, + 4: 0x5a, + 5: 0x0 + }, + extractor: null + }, + { + name: "Tarball", + extension: "tar.z", + mime: "application/x-gtar", + description: "", + signature: { + 0: 0x1f, + 1: [0x9d, 0xa0] + }, + extractor: null + }, + { + name: "ISO disk image", + extension: "iso", + mime: "application/octet-stream", + description: "ISO 9660 CD/DVD image file", + signature: [ + { + 0x8001: 0x43, + 0x8002: 0x44, + 0x8003: 0x30, + 0x8004: 0x30, + 0x8005: 0x31 + }, + { + 0x8801: 0x43, + 0x8802: 0x44, + 0x8803: 0x30, + 0x8804: 0x30, + 0x8805: 0x31 + }, + { + 0x9001: 0x43, + 0x9002: 0x44, + 0x9003: 0x30, + 0x9004: 0x30, + 0x9005: 0x31 + } + ], + extractor: null + }, + { + name: "Virtual Machine Disk", + extension: "vmdk", + mime: "application/vmdk,application/x-virtualbox-vmdk", + description: "", + signature: { + 0: 0x4b, + 1: 0x44, + 2: 0x4d + }, + extractor: null + }, + ], + "Miscellaneous": [ + { + name: "UTF-8 text file", + extension: "txt", + mime: "text/plain", + description: "UTF-8 encoded Unicode byte order mark, commonly but not exclusively seen in text files.", + signature: { + 0: 0xef, + 1: 0xbb, + 2: 0xbf + }, + extractor: null + }, + { // Place before UTF-16 LE file + name: "UTF-32 LE file", + extension: "utf32le", + mime: "charset/utf32le", + description: "Little-endian UTF-32 encoded Unicode byte order mark.", + signature: { + 0: 0xff, + 1: 0xfe, + 2: 0x00, + 3: 0x00 + }, + extractor: null + }, + { + name: "UTF-16 LE file", + extension: "utf16le", + mime: "charset/utf16le", + description: "Little-endian UTF-16 encoded Unicode byte order mark.", + signature: { + 0: 0xff, + 1: 0xfe + }, + extractor: null + }, + { + name: "Web Open Font Format", + extension: "woff", + mime: "application/font-woff", + description: "", + signature: { + 0: 0x77, + 1: 0x4f, + 2: 0x46, + 3: 0x46, + 4: 0x0, + 5: 0x1, + 6: 0x0, + 7: 0x0 + }, + extractor: null + }, + { + name: "Web Open Font Format 2", + extension: "woff2", + mime: "application/font-woff", + description: "", + signature: { + 0: 0x77, + 1: 0x4f, + 2: 0x46, + 3: 0x32, + 4: 0x0, + 5: 0x1, + 6: 0x0, + 7: 0x0 + }, + extractor: null + }, + { + name: "Embedded OpenType font", + extension: "eot", + mime: "application/octet-stream", + description: "", + signature: [ + { + 8: 0x2, + 9: 0x0, + 10: 0x1, + 34: 0x4c, + 35: 0x50 + }, + { + 8: 0x1, + 9: 0x0, + 10: 0x0, + 34: 0x4c, + 35: 0x50 + }, + { + 8: 0x2, + 9: 0x0, + 10: 0x2, + 34: 0x4c, + 35: 0x50 + }, + ], + extractor: null + }, + { + name: "TrueType Font", + extension: "ttf", + mime: "application/font-sfnt", + description: "", + signature: { + 0: 0x0, + 1: 0x1, + 2: 0x0, + 3: 0x0, + 4: 0x0 + }, + extractor: null + }, + { + name: "OpenType Font", + extension: "otf", + mime: "application/font-sfnt", + description: "", + signature: { + 0: 0x4f, + 1: 0x54, + 2: 0x54, + 3: 0x4f, + 4: 0x0 + }, + extractor: null + }, + { + name: "SQLite", + extension: "sqlite", + mime: "application/x-sqlite3", + description: "", + signature: { + 0: 0x53, + 1: 0x51, + 2: 0x4c, + 3: 0x69 + }, + extractor: null + }, + ] +}; + + +/** + * JPEG extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractJPEG(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + while (stream.hasMore()) { + const marker = stream.getBytes(2); + if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); + + let segmentSize = 0; + switch (marker[1]) { + // No length + case 0xd8: // Start of Image + case 0x01: // For temporary use in arithmetic coding + break; + case 0xd9: // End found + return stream.carve(); + + // Variable size segment + case 0xc0: // Start of frame (Baseline DCT) + case 0xc1: // Start of frame (Extended sequential DCT) + case 0xc2: // Start of frame (Progressive DCT) + case 0xc3: // Start of frame (Lossless sequential) + case 0xc4: // Define Huffman Table + case 0xc5: // Start of frame (Differential sequential DCT) + case 0xc6: // Start of frame (Differential progressive DCT) + case 0xc7: // Start of frame (Differential lossless) + case 0xc8: // Reserved for JPEG extensions + case 0xc9: // Start of frame (Extended sequential DCT) + case 0xca: // Start of frame (Progressive DCT) + case 0xcb: // Start of frame (Lossless sequential) + case 0xcc: // Define arithmetic conditioning table + case 0xcd: // Start of frame (Differential sequential DCT) + case 0xce: // Start of frame (Differential progressive DCT) + case 0xcf: // Start of frame (Differential lossless) + case 0xdb: // Define Quantization Table + case 0xde: // Define hierarchical progression + case 0xe0: // Application-specific + case 0xe1: // Application-specific + case 0xe2: // Application-specific + case 0xe3: // Application-specific + case 0xe4: // Application-specific + case 0xe5: // Application-specific + case 0xe6: // Application-specific + case 0xe7: // Application-specific + case 0xe8: // Application-specific + case 0xe9: // Application-specific + case 0xea: // Application-specific + case 0xeb: // Application-specific + case 0xec: // Application-specific + case 0xed: // Application-specific + case 0xee: // Application-specific + case 0xef: // Application-specific + case 0xfe: // Comment + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + break; + + // 1 byte + case 0xdf: // Expand reference image + stream.position++; + break; + + // 2 bytes + case 0xdc: // Define number of lines + case 0xdd: // Define restart interval + stream.position += 2; + break; + + // Start scan + case 0xda: // Start of scan + segmentSize = stream.readInt(2, "be"); + stream.position += segmentSize - 2; + stream.continueUntil(0xff); + break; + + // Continue through encoded data + case 0x00: // Byte stuffing + case 0xd0: // Restart + case 0xd1: // Restart + case 0xd2: // Restart + case 0xd3: // Restart + case 0xd4: // Restart + case 0xd5: // Restart + case 0xd6: // Restart + case 0xd7: // Restart + stream.continueUntil(0xff); + break; + + default: + stream.continueUntil(0xff); + break; + } + } + + throw new Error("Unable to parse JPEG successfully"); +} + + +/** + * Portable executable extractor. + * Assumes that the offset refers to an MZ header. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractMZPE(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move to PE header pointer + stream.moveTo(0x3c); + const peAddress = stream.readInt(4, "le"); + + // Move to PE header + stream.moveTo(peAddress); + + // Get number of sections + stream.moveForwardsBy(6); + const numSections = stream.readInt(2, "le"); + + // Get optional header size + stream.moveForwardsBy(12); + const optionalHeaderSize = stream.readInt(2, "le"); + + // Move past optional header to section header + stream.moveForwardsBy(2 + optionalHeaderSize); + + // Move to final section header + stream.moveForwardsBy((numSections - 1) * 0x28); + + // Get raw data info + stream.moveForwardsBy(16); + const rawDataSize = stream.readInt(4, "le"); + const rawDataAddress = stream.readInt(4, "le"); + + // Move to end of final section + stream.moveTo(rawDataAddress + rawDataSize); + + return stream.carve(); +} + + +/** + * PDF extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractPDF(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find end-of-file marker (%%EOF) + stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]); + stream.moveForwardsBy(5); + stream.consumeIf(0x0d); + stream.consumeIf(0x0a); + + return stream.carve(); +} + + +/** + * ZIP extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractZIP(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Find End of central directory record + stream.continueUntil([0x50, 0x4b, 0x05, 0x06]); + + // Get comment length and consume + stream.moveForwardsBy(20); + const commentLength = stream.readInt(2, "le"); + stream.moveForwardsBy(commentLength); + + return stream.carve(); +} diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index ef6cfb03..b96ea69e 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -6,256 +6,7 @@ * @license Apache-2.0 * */ -import Stream from "./Stream"; - -/** - * A categorised table of file types, including signatures to identifying them and functions - * to extract them where possible. - */ -const FILE_SIGNATURES = { - "Images": [ - { - name: "JPEG Image", - extension: "jpg", - mime: "image/jpeg", - description: "", - signature: { - 0: 0xff, - 1: 0xd8, - 2: 0xff - }, - extractor: extractJPEG - }, - { - name: "GIF Image", - extension: "gif", - mime: "image/gif", - description: "", - signature: { - 0: 0x47, - 1: 0x49, - 2: 0x46 - }, - extractor: null - }, - { - name: "PNG Image", - extension: "png", - mime: "image/png", - description: "", - signature: { - 0: 0x89, - 1: 0x50, - 2: 0x4e, - 3: 0x47 - }, - extractor: null - }, - { - name: "WEBP Image", - extension: "webp", - mime: "image/webp", - description: "", - signature: { - 8: 0x57, - 9: 0x45, - 10: 0x42, - 11: 0x50 - }, - extractor: null - }, - { - name: "TIFF Image", - extension: "tif", - mime: "image/tiff", - description: "", - signature: [ - { - 0: 0x49, - 1: 0x49, - 2: 0x2a, - 3: 0x0 - }, - { - 0: 0x4d, - 1: 0x4d, - 2: 0x0, - 3: 0x2a - } - ], - extractor: null - }, /* - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - }, - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - }, - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - }, - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - }, - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - }, - { - name: " Image", - extension: "", - mime: "image/", - description: "", - signature: { - 0: 0x, - 1: 0x, - 2: 0x, - 3: 0x - }, - extractor: null - },*/ - ], - "Video": [ - { - name: "WEBM", - extension: "webm", - mime: "video/webm", - description: "", - signature: { - 0: 0x1a, - 1: 0x45, - 2: 0xdf, - 3: 0xa3 - }, - extractor: null - }, - ], - "Audio": [ - { - name: "WAV", - extension: "wav", - mime: "audio/x-wav", - description: "", - signature: { - 0: 0x52, - 1: 0x49, - 2: 0x46, - 3: 0x46, - 8: 0x57, - 9: 0x41, - 10: 0x56, - 11: 0x45 - }, - extractor: null - }, - { - name: "OGG", - extension: "ogg", - mime: "audio/ogg", - description: "", - signature: { - 0: 0x4f, - 1: 0x67, - 2: 0x67, - 3: 0x53 - }, - extractor: null - }, - ], - "Documents": [ - { - name: "Portable Document Format", - extension: "pdf", - mime: "application/pdf", - description: "", - signature: { - 0: 0x25, - 1: 0x50, - 2: 0x44, - 3: 0x46 - }, - extractor: extractPDF - }, - ], - "Applications": [ - { - name: "Windows Portable Executable", - extension: "exe", - mime: "application/x-msdownload", - description: "", - signature: { - 0: 0x4d, - 1: 0x5a - }, - extractor: extractMZPE - }, - ], - "Archives": [ - { - name: "ZIP", - extension: "zip", - mime: "application/zip", - description: "", - signature: { - 0: 0x50, - 1: 0x4b, - 2: [0x3, 0x5, 0x7], - 3: [0x4, 0x6, 0x8] - }, - extractor: extractZIP - }, - - ], -}; +import {FILE_SIGNATURES} from "./FileSignatures"; /** @@ -313,7 +64,8 @@ function bytesMatch(sig, buf) { * extension and mime type. * * @param {Uint8Array} buf - * @returns {Object[]} type + * @returns {Object[]} types + * @returns {string} type.name - Name of file type * @returns {string} type.ext - File extension * @returns {string} type.mime - Mime type * @returns {string} [type.desc] - Description @@ -336,370 +88,6 @@ export function detectFileType(buf) { }); } return matchingFiles; - - // Delete all below this line once implemented in FILE_SIGNATURES above. - - - /* - // needs to be before `tif` check - if (((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) && buf[8] === 0x43 && buf[9] === 0x52) { - return { - ext: "cr2", - mime: "image/x-canon-cr2" - }; - } - - if ((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) { - return { - ext: "tif", - mime: "image/tiff" - }; - } - - if (buf[0] === 0x42 && buf[1] === 0x4D) { - return { - ext: "bmp", - mime: "image/bmp" - }; - } - - if (buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0xBC) { - return { - ext: "jxr", - mime: "image/vnd.ms-photo" - }; - } - - if (buf[0] === 0x38 && buf[1] === 0x42 && buf[2] === 0x50 && buf[3] === 0x53) { - return { - ext: "psd", - mime: "image/vnd.adobe.photoshop" - }; - } - - // needs to be before `zip` check - if (buf[0] === 0x50 && buf[1] === 0x4B && buf[2] === 0x3 && buf[3] === 0x4 && buf[30] === 0x6D && buf[31] === 0x69 && buf[32] === 0x6D && buf[33] === 0x65 && buf[34] === 0x74 && buf[35] === 0x79 && buf[36] === 0x70 && buf[37] === 0x65 && buf[38] === 0x61 && buf[39] === 0x70 && buf[40] === 0x70 && buf[41] === 0x6C && buf[42] === 0x69 && buf[43] === 0x63 && buf[44] === 0x61 && buf[45] === 0x74 && buf[46] === 0x69 && buf[47] === 0x6F && buf[48] === 0x6E && buf[49] === 0x2F && buf[50] === 0x65 && buf[51] === 0x70 && buf[52] === 0x75 && buf[53] === 0x62 && buf[54] === 0x2B && buf[55] === 0x7A && buf[56] === 0x69 && buf[57] === 0x70) { - return { - ext: "epub", - mime: "application/epub+zip" - }; - } - - if (buf[257] === 0x75 && buf[258] === 0x73 && buf[259] === 0x74 && buf[260] === 0x61 && buf[261] === 0x72) { - return { - ext: "tar", - mime: "application/x-tar" - }; - } - - if (buf[0] === 0x52 && buf[1] === 0x61 && buf[2] === 0x72 && buf[3] === 0x21 && buf[4] === 0x1A && buf[5] === 0x7 && (buf[6] === 0x0 || buf[6] === 0x1)) { - return { - ext: "rar", - mime: "application/x-rar-compressed" - }; - } - - if (buf[0] === 0x1F && buf[1] === 0x8B && buf[2] === 0x8) { - return { - ext: "gz", - mime: "application/gzip" - }; - } - - if (buf[0] === 0x42 && buf[1] === 0x5A && buf[2] === 0x68) { - return { - ext: "bz2", - mime: "application/x-bzip2" - }; - } - - if (buf[0] === 0x37 && buf[1] === 0x7A && buf[2] === 0xBC && buf[3] === 0xAF && buf[4] === 0x27 && buf[5] === 0x1C) { - return { - ext: "7z", - mime: "application/x-7z-compressed" - }; - } - - if (buf[0] === 0x78 && buf[1] === 0x01) { - return { - ext: "dmg, zlib", - mime: "application/x-apple-diskimage, application/x-deflate" - }; - } - - if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && (buf[3] === 0x18 || buf[3] === 0x20) && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) || (buf[0] === 0x33 && buf[1] === 0x67 && buf[2] === 0x70 && buf[3] === 0x35) || (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x6D && buf[9] === 0x70 && buf[10] === 0x34 && buf[11] === 0x32 && buf[16] === 0x6D && buf[17] === 0x70 && buf[18] === 0x34 && buf[19] === 0x31 && buf[20] === 0x6D && buf[21] === 0x70 && buf[22] === 0x34 && buf[23] === 0x32 && buf[24] === 0x69 && buf[25] === 0x73 && buf[26] === 0x6F && buf[27] === 0x6D)) { - return { - ext: "mp4", - mime: "video/mp4" - }; - } - - if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x56)) { - return { - ext: "m4v", - mime: "video/x-m4v" - }; - } - - if (buf[0] === 0x4D && buf[1] === 0x54 && buf[2] === 0x68 && buf[3] === 0x64) { - return { - ext: "mid", - mime: "audio/midi" - }; - } - - // needs to be before the `webm` check - if (buf[31] === 0x6D && buf[32] === 0x61 && buf[33] === 0x74 && buf[34] === 0x72 && buf[35] === 0x6f && buf[36] === 0x73 && buf[37] === 0x6B && buf[38] === 0x61) { - return { - ext: "mkv", - mime: "video/x-matroska" - }; - } - - if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x14 && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) { - return { - ext: "mov", - mime: "video/quicktime" - }; - } - - if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x41 && buf[9] === 0x56 && buf[10] === 0x49) { - return { - ext: "avi", - mime: "video/x-msvideo" - }; - } - - if (buf[0] === 0x30 && buf[1] === 0x26 && buf[2] === 0xB2 && buf[3] === 0x75 && buf[4] === 0x8E && buf[5] === 0x66 && buf[6] === 0xCF && buf[7] === 0x11 && buf[8] === 0xA6 && buf[9] === 0xD9) { - return { - ext: "wmv", - mime: "video/x-ms-wmv" - }; - } - - if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x1 && buf[3].toString(16)[0] === "b") { - return { - ext: "mpg", - mime: "video/mpeg" - }; - } - - if ((buf[0] === 0x49 && buf[1] === 0x44 && buf[2] === 0x33) || (buf[0] === 0xFF && buf[1] === 0xfb)) { - return { - ext: "mp3", - mime: "audio/mpeg" - }; - } - - if ((buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x41) || (buf[0] === 0x4D && buf[1] === 0x34 && buf[2] === 0x41 && buf[3] === 0x20)) { - return { - ext: "m4a", - mime: "audio/m4a" - }; - } - - if (buf[0] === 0x66 && buf[1] === 0x4C && buf[2] === 0x61 && buf[3] === 0x43) { - return { - ext: "flac", - mime: "audio/x-flac" - }; - } - - if (buf[0] === 0x23 && buf[1] === 0x21 && buf[2] === 0x41 && buf[3] === 0x4D && buf[4] === 0x52 && buf[5] === 0x0A) { - return { - ext: "amr", - mime: "audio/amr" - }; - } - - if ((buf[0] === 0x43 || buf[0] === 0x46) && buf[1] === 0x57 && buf[2] === 0x53) { - return { - ext: "swf", - mime: "application/x-shockwave-flash" - }; - } - - if (buf[0] === 0x7B && buf[1] === 0x5C && buf[2] === 0x72 && buf[3] === 0x74 && buf[4] === 0x66) { - return { - ext: "rtf", - mime: "application/rtf" - }; - } - - if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x46 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { - return { - ext: "woff", - mime: "application/font-woff" - }; - } - - if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x32 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) { - return { - ext: "woff2", - mime: "application/font-woff" - }; - } - - if (buf[34] === 0x4C && buf[35] === 0x50 && ((buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x01) || (buf[8] === 0x01 && buf[9] === 0x00 && buf[10] === 0x00) || (buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x02))) { - return { - ext: "eot", - mime: "application/octet-stream" - }; - } - - if (buf[0] === 0x00 && buf[1] === 0x01 && buf[2] === 0x00 && buf[3] === 0x00 && buf[4] === 0x00) { - return { - ext: "ttf", - mime: "application/font-sfnt" - }; - } - - if (buf[0] === 0x4F && buf[1] === 0x54 && buf[2] === 0x54 && buf[3] === 0x4F && buf[4] === 0x00) { - return { - ext: "otf", - mime: "application/font-sfnt" - }; - } - - if (buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0x01 && buf[3] === 0x00) { - return { - ext: "ico", - mime: "image/x-icon" - }; - } - - if (buf[0] === 0x46 && buf[1] === 0x4C && buf[2] === 0x56 && buf[3] === 0x01) { - return { - ext: "flv", - mime: "video/x-flv" - }; - } - - if (buf[0] === 0x25 && buf[1] === 0x21) { - return { - ext: "ps", - mime: "application/postscript" - }; - } - - if (buf[0] === 0xFD && buf[1] === 0x37 && buf[2] === 0x7A && buf[3] === 0x58 && buf[4] === 0x5A && buf[5] === 0x00) { - return { - ext: "xz", - mime: "application/x-xz" - }; - } - - if (buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4C && buf[3] === 0x69) { - return { - ext: "sqlite", - mime: "application/x-sqlite3" - }; - } - */ - - /** - * - * Added by n1474335 [n1474335@gmail.com] from here on - * - */ - /* - if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) { - return { - ext: "z, tar.z", - mime: "application/x-gtar" - }; - } - - if (buf[0] === 0x7F && buf[1] === 0x45 && buf[2] === 0x4C && buf[3] === 0x46) { - return { - ext: "none, axf, bin, elf, o, prx, puff, so", - mime: "application/x-executable", - desc: "Executable and Linkable Format file. No standard file extension." - }; - } - - if (buf[0] === 0xCA && buf[1] === 0xFE && buf[2] === 0xBA && buf[3] === 0xBE) { - return { - ext: "class", - mime: "application/java-vm" - }; - } - - if (buf[0] === 0xEF && buf[1] === 0xBB && buf[2] === 0xBF) { - return { - ext: "txt", - mime: "text/plain", - desc: "UTF-8 encoded Unicode byte order mark detected, commonly but not exclusively seen in text files." - }; - } - - // Must be before Little-endian UTF-16 BOM - if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) { - return { - ext: "UTF32LE", - mime: "charset/utf32le", - desc: "Little-endian UTF-32 encoded Unicode byte order mark detected." - }; - } - - if (buf[0] === 0xFF && buf[1] === 0xFE) { - return { - ext: "UTF16LE", - mime: "charset/utf16le", - desc: "Little-endian UTF-16 encoded Unicode byte order mark detected." - }; - } - - if ((buf[0x8001] === 0x43 && buf[0x8002] === 0x44 && buf[0x8003] === 0x30 && buf[0x8004] === 0x30 && buf[0x8005] === 0x31) || - (buf[0x8801] === 0x43 && buf[0x8802] === 0x44 && buf[0x8803] === 0x30 && buf[0x8804] === 0x30 && buf[0x8805] === 0x31) || - (buf[0x9001] === 0x43 && buf[0x9002] === 0x44 && buf[0x9003] === 0x30 && buf[0x9004] === 0x30 && buf[0x9005] === 0x31)) { - return { - ext: "iso", - mime: "application/octet-stream", - desc: "ISO 9660 CD/DVD image file" - }; - } - - if (buf[0] === 0xD0 && buf[1] === 0xCF && buf[2] === 0x11 && buf[3] === 0xE0 && buf[4] === 0xA1 && buf[5] === 0xB1 && buf[6] === 0x1A && buf[7] === 0xE1) { - return { - ext: "doc, xls, ppt", - mime: "application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint", - desc: "Microsoft Office documents" - }; - } - - if (buf[0] === 0x64 && buf[1] === 0x65 && buf[2] === 0x78 && buf[3] === 0x0A && buf[4] === 0x30 && buf[5] === 0x33 && buf[6] === 0x35 && buf[7] === 0x00) { - return { - ext: "dex", - mime: "application/octet-stream", - desc: "Dalvik Executable (Android)" - }; - } - - if (buf[0] === 0x4B && buf[1] === 0x44 && buf[2] === 0x4D) { - return { - ext: "vmdk", - mime: "application/vmdk, application/x-virtualbox-vmdk" - }; - } - - if (buf[0] === 0x43 && buf[1] === 0x72 && buf[2] === 0x32 && buf[3] === 0x34) { - return { - ext: "crx", - mime: "application/crx", - desc: "Google Chrome extension or packaged app" - }; - } - - if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) { - return { - ext: "zlib", - mime: "application/x-deflate" - }; - } - - return null; - */ } @@ -750,198 +138,9 @@ export function isImage(buf) { export function extractFile(bytes, fileDetail, offset) { if (fileDetail.extractor) { const fileData = fileDetail.extractor(bytes, offset); - return new File([fileData], `extracted_at_0x${offset.toString(16)}.${fileDetail.extension}`); + const ext = fileDetail.extension.split(",")[0]; + return new File([fileData], `extracted_at_0x${offset.toString(16)}.${ext}`); } throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`); } - - -/** - * JPEG extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractJPEG(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - while (stream.hasMore()) { - const marker = stream.getBytes(2); - if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); - - let segmentSize = 0; - switch (marker[1]) { - // No length - case 0xd8: // Start of Image - case 0x01: // For temporary use in arithmetic coding - break; - case 0xd9: // End found - return stream.carve(); - - // Variable size segment - case 0xc0: // Start of frame (Baseline DCT) - case 0xc1: // Start of frame (Extended sequential DCT) - case 0xc2: // Start of frame (Progressive DCT) - case 0xc3: // Start of frame (Lossless sequential) - case 0xc4: // Define Huffman Table - case 0xc5: // Start of frame (Differential sequential DCT) - case 0xc6: // Start of frame (Differential progressive DCT) - case 0xc7: // Start of frame (Differential lossless) - case 0xc8: // Reserved for JPEG extensions - case 0xc9: // Start of frame (Extended sequential DCT) - case 0xca: // Start of frame (Progressive DCT) - case 0xcb: // Start of frame (Lossless sequential) - case 0xcc: // Define arithmetic conditioning table - case 0xcd: // Start of frame (Differential sequential DCT) - case 0xce: // Start of frame (Differential progressive DCT) - case 0xcf: // Start of frame (Differential lossless) - case 0xdb: // Define Quantization Table - case 0xde: // Define hierarchical progression - case 0xe0: // Application-specific - case 0xe1: // Application-specific - case 0xe2: // Application-specific - case 0xe3: // Application-specific - case 0xe4: // Application-specific - case 0xe5: // Application-specific - case 0xe6: // Application-specific - case 0xe7: // Application-specific - case 0xe8: // Application-specific - case 0xe9: // Application-specific - case 0xea: // Application-specific - case 0xeb: // Application-specific - case 0xec: // Application-specific - case 0xed: // Application-specific - case 0xee: // Application-specific - case 0xef: // Application-specific - case 0xfe: // Comment - segmentSize = stream.readInt(2, "be"); - stream.position += segmentSize - 2; - break; - - // 1 byte - case 0xdf: // Expand reference image - stream.position++; - break; - - // 2 bytes - case 0xdc: // Define number of lines - case 0xdd: // Define restart interval - stream.position += 2; - break; - - // Start scan - case 0xda: // Start of scan - segmentSize = stream.readInt(2, "be"); - stream.position += segmentSize - 2; - stream.continueUntil(0xff); - break; - - // Continue through encoded data - case 0x00: // Byte stuffing - case 0xd0: // Restart - case 0xd1: // Restart - case 0xd2: // Restart - case 0xd3: // Restart - case 0xd4: // Restart - case 0xd5: // Restart - case 0xd6: // Restart - case 0xd7: // Restart - stream.continueUntil(0xff); - break; - - default: - stream.continueUntil(0xff); - break; - } - } - - throw new Error("Unable to parse JPEG successfully"); -} - - -/** - * Portable executable extractor. - * Assumes that the offset refers to an MZ header. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractMZPE(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Move to PE header pointer - stream.moveTo(0x3c); - const peAddress = stream.readInt(4, "le"); - - // Move to PE header - stream.moveTo(peAddress); - - // Get number of sections - stream.moveForwardsBy(6); - const numSections = stream.readInt(2, "le"); - - // Get optional header size - stream.moveForwardsBy(12); - const optionalHeaderSize = stream.readInt(2, "le"); - - // Move past optional header to section header - stream.moveForwardsBy(2 + optionalHeaderSize); - - // Move to final section header - stream.moveForwardsBy((numSections - 1) * 0x28); - - // Get raw data info - stream.moveForwardsBy(16); - const rawDataSize = stream.readInt(4, "le"); - const rawDataAddress = stream.readInt(4, "le"); - - // Move to end of final section - stream.moveTo(rawDataAddress + rawDataSize); - - return stream.carve(); -} - - -/** - * PDF extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractPDF(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Find end-of-file marker (%%EOF) - stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]); - stream.moveForwardsBy(5); - stream.consumeIf(0x0d); - stream.consumeIf(0x0a); - - return stream.carve(); -} - - -/** - * ZIP extractor. - * - * @param {Uint8Array} bytes - * @param {number} offset - * @returns {Uint8Array} - */ -export function extractZIP(bytes, offset) { - const stream = new Stream(bytes.slice(offset)); - - // Find End of central directory record - stream.continueUntil([0x50, 0x4b, 0x05, 0x06]); - - // Get comment length and consume - stream.moveForwardsBy(20); - const commentLength = stream.readInt(2, "le"); - stream.moveForwardsBy(commentLength); - - return stream.carve(); -} From 0198f05112cd0c5734ee5057be94aed23b654c8c Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 27 Dec 2018 00:03:41 +0000 Subject: [PATCH 06/22] Added and improved file signatures. --- src/core/lib/FileSignatures.mjs | 117 +++++++++++++++++++++++++++++--- 1 file changed, 108 insertions(+), 9 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 69699d7d..50a5e830 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -16,13 +16,14 @@ export const FILE_SIGNATURES = { "Images": [ { name: "Joint Photographic Experts Group image", - extension: "jpg", + extension: "jpg,jpeg,jpe,thm,mpo", mime: "image/jpeg", description: "", signature: { 0: 0xff, 1: 0xd8, - 2: 0xff + 2: 0xff, + 3: [0xc0, 0xc4, 0xdb, 0xdd, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe7, 0xe8, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xfe] }, extractor: extractJPEG }, @@ -32,9 +33,12 @@ export const FILE_SIGNATURES = { mime: "image/gif", description: "", signature: { - 0: 0x47, + 0: 0x47, // GIF 1: 0x49, - 2: 0x46 + 2: 0x46, + 3: 0x38, // 8 + 4: [0x37, 0x39], // 7|9 + 5: 0x61 // a }, extractor: null }, @@ -45,9 +49,13 @@ export const FILE_SIGNATURES = { description: "", signature: { 0: 0x89, - 1: 0x50, + 1: 0x50, // PNG 2: 0x4e, - 3: 0x47 + 3: 0x47, + 4: 0x0d, + 5: 0x0a, + 6: 0x1a, + 7: 0x0a }, extractor: null }, @@ -64,6 +72,23 @@ export const FILE_SIGNATURES = { }, extractor: null }, + { + name: "Camera Image File Format", + extension: "crw", + mime: "image/x-canon-crw", + description: "", + signature: { + 6: 0x48, // HEAPCCDR + 7: 0x45, + 8: 0x41, + 9: 0x50, + 10: 0x43, + 11: 0x43, + 12: 0x44, + 13: 0x52 + }, + extractor: null + }, { // Place before tiff check name: "Canon CR2 raw image", extension: "cr2", @@ -117,7 +142,13 @@ export const FILE_SIGNATURES = { description: "", signature: { 0: 0x42, - 1: 0x4d + 1: 0x4d, + 7: 0x0, + 9: 0x0, + 14: [0x0c, 0x28, 0x38, 0x40, 0x6c, 0x7c], + 15: 0x0, + 16: 0x0, + 17: 0x0 }, extractor: null }, @@ -142,10 +173,52 @@ export const FILE_SIGNATURES = { 0: 0x38, 1: 0x42, 2: 0x50, - 3: 0x53 + 3: 0x53, + 4: 0x0, + 5: 0x1, + 6: 0x0, + 7: 0x0, + 8: 0x0, + 9: 0x0, + 10: 0x0, + 11: 0x0 }, extractor: null }, + { + name: "Paint Shop Pro image", + extension: "psp", + mime: "image/psp", + description: "", + signature: [ + { + 0: 0x50, // Paint Shop Pro Im + 1: 0x61, + 2: 0x69, + 3: 0x6e, + 4: 0x74, + 5: 0x20, + 6: 0x53, + 7: 0x68, + 8: 0x6f, + 9: 0x70, + 10: 0x20, + 11: 0x50, + 12: 0x72, + 13: 0x6f, + 14: 0x20, + 15: 0x49, + 16: 0x6d + }, + { + 0: 0x7e, + 1: 0x42, + 2: 0x4b, + 3: 0x0 + } + ], + extractor: null + }, { name: "Icon image", extension: "ico", @@ -155,7 +228,13 @@ export const FILE_SIGNATURES = { 0: 0x0, 1: 0x0, 2: 0x1, - 3: 0x0 + 3: 0x0, + 4: [0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15], + 5: 0x0, + 6: [0x10, 0x20, 0x30, 0x40, 0x80], + 7: [0x10, 0x20, 0x30, 0x40, 0x80], + 9: 0x00, + 10: [0x0, 0x1] }, extractor: null } @@ -512,6 +591,26 @@ export const FILE_SIGNATURES = { }, extractor: null }, + { + name: "Microsoft Office 2007+ documents", + extension: "docx,xlsx,pptx", + mime: "application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/vnd.openxmlformats-officedocument.presentationml.presentation", + description: "", + signature: { + 38: 0x5f, // _Types].xml + 39: 0x54, + 40: 0x79, + 41: 0x70, + 42: 0x65, + 43: 0x73, + 44: 0x5d, + 45: 0x2e, + 46: 0x78, + 47: 0x6d, + 48: 0x6c + }, + extractor: null + }, { name: "EPUB e-book", extension: "epub", From 3ae225ac59c4843371ffa97a5cbd851350ca303c Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sun, 30 Dec 2018 01:36:58 +0000 Subject: [PATCH 07/22] Untar operation now uses lib/Stream library --- src/core/operations/Untar.mjs | 35 ++--------------------------------- tests/browser/nightwatch.js | 2 +- 2 files changed, 3 insertions(+), 34 deletions(-) diff --git a/src/core/operations/Untar.mjs b/src/core/operations/Untar.mjs index af029184..8655ba68 100644 --- a/src/core/operations/Untar.mjs +++ b/src/core/operations/Untar.mjs @@ -6,6 +6,7 @@ import Operation from "../Operation"; import Utils from "../Utils"; +import Stream from "../lib/Stream"; /** * Untar operation @@ -41,38 +42,6 @@ class Untar extends Operation { * @returns {List} */ run(input, args) { - const Stream = function(input) { - this.bytes = input; - this.position = 0; - }; - - Stream.prototype.getBytes = function(bytesToGet) { - const newPosition = this.position + bytesToGet; - const bytes = this.bytes.slice(this.position, newPosition); - this.position = newPosition; - return bytes; - }; - - Stream.prototype.readString = function(numBytes) { - let result = ""; - for (let i = this.position; i < this.position + numBytes; i++) { - const currentByte = this.bytes[i]; - if (currentByte === 0) break; - result += String.fromCharCode(currentByte); - } - this.position += numBytes; - return result; - }; - - Stream.prototype.readInt = function(numBytes, base) { - const string = this.readString(numBytes); - return parseInt(string, base); - }; - - Stream.prototype.hasMore = function() { - return this.position < this.bytes.length; - }; - const stream = new Stream(input), files = []; @@ -85,7 +54,7 @@ class Untar extends Operation { ownerUID: stream.readString(8), ownerGID: stream.readString(8), size: parseInt(stream.readString(12), 8), // Octal - lastModTime: new Date(1000 * stream.readInt(12, 8)), // Octal + lastModTime: new Date(1000 * parseInt(stream.readString(12), 8)), // Octal checksum: stream.readString(8), type: stream.readString(1), linkedFileName: stream.readString(100), diff --git a/tests/browser/nightwatch.js b/tests/browser/nightwatch.js index 52587d2f..23100d8d 100644 --- a/tests/browser/nightwatch.js +++ b/tests/browser/nightwatch.js @@ -87,7 +87,7 @@ module.exports = { // Check output browser .useCss() - .waitForElementNotVisible("#stale-indicator", 500) + .waitForElementNotVisible("#stale-indicator", 1000) .expect.element("#output-text").to.have.value.that.equals("44 6f 6e 27 74 20 50 61 6e 69 63 2e"); // Clear recipe From ede75530d08506eb77e58557449982143c9567fe Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sun, 30 Dec 2018 02:21:45 +0000 Subject: [PATCH 08/22] Added PNG and BMP extractors --- src/core/lib/FileSignatures.mjs | 56 +++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 50a5e830..d4e19da9 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -57,7 +57,7 @@ export const FILE_SIGNATURES = { 6: 0x1a, 7: 0x0a }, - extractor: null + extractor: extractPNG }, { name: "WEBP Image", @@ -150,7 +150,7 @@ export const FILE_SIGNATURES = { 16: 0x0, 17: 0x0 }, - extractor: null + extractor: extractBMP }, { name: "JPEG Extended Range image", @@ -1233,3 +1233,55 @@ export function extractZIP(bytes, offset) { return stream.carve(); } + + +/** + * PNG extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractPNG(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move past signature to first chunk + stream.moveForwardsBy(8); + + let chunkSize = 0, + chunkType = ""; + + while (chunkType !== "IEND") { + chunkSize = stream.readInt(4, "be"); + chunkType = stream.readString(4); + + // Chunk data size + CRC checksum + stream.moveForwardsBy(chunkSize + 4); + } + + + return stream.carve(); +} + + +/** + * BMP extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractBMP(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move past header + stream.moveForwardsBy(2); + + // Read full file size + const bmpSize = stream.readInt(4, "le"); + + // Move to end of file (file size minus header and size field) + stream.moveForwardsBy(bmpSize - 6); + + return stream.carve(); +} From 4c285bce57b582a2246b982e83dcc3ecf86fbed0 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 1 Jan 2019 15:12:01 +0000 Subject: [PATCH 09/22] Refactored scanning for file types to be more than twice as fast. --- src/core/lib/FileType.mjs | 73 +++++++++++++++++--- src/core/operations/ExtractFiles.mjs | 26 +------ src/core/operations/ScanForEmbeddedFiles.mjs | 40 +++++------ 3 files changed, 83 insertions(+), 56 deletions(-) diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index b96ea69e..5e1dd657 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -16,13 +16,22 @@ import {FILE_SIGNATURES} from "./FileSignatures"; * These values can be numbers for static checks, arrays of potential valid matches, * or bespoke functions to check the validity of the buffer value at that offset. * @param {Uint8Array} buf + * @param {number} [offset=0] Where in the buffer to start searching from * @returns {boolean} */ -function signatureMatches(sig, buf) { - if (sig instanceof Array) { - return sig.reduce((acc, s) => acc || bytesMatch(s, buf), false); +function signatureMatches(sig, buf, offset=0) { + // Using a length check seems to be more performant than `sig instanceof Array` + if (sig.length) { + // sig is an Array - return true if any of them match + // The following `reduce` method is nice, but performance matters here, so we + // opt for a faster, if less elegant, for loop. + // return sig.reduce((acc, s) => acc || bytesMatch(s, buf, offset), false); + for (let i = 0; i < sig.length; i++) { + if (bytesMatch(sig[i], buf, offset)) return true; + } + return false; } else { - return bytesMatch(sig, buf); + return bytesMatch(sig, buf, offset); } } @@ -34,25 +43,27 @@ function signatureMatches(sig, buf) { * These values can be numbers for static checks, arrays of potential valid matches, * or bespoke functions to check the validity of the buffer value at that offset. * @param {Uint8Array} buf + * @param {number} [offset=0] Where in the buffer to start searching from * @returns {boolean} */ -function bytesMatch(sig, buf) { - for (const offset in sig) { - switch (typeof sig[offset]) { +function bytesMatch(sig, buf, offset=0) { + for (const sigoffset in sig) { + const pos = parseInt(sigoffset, 10) + offset; + switch (typeof sig[sigoffset]) { case "number": // Static check - if (buf[offset] !== sig[offset]) + if (buf[pos] !== sig[sigoffset]) return false; break; case "object": // Array of options - if (sig[offset].indexOf(buf[offset]) < 0) + if (sig[sigoffset].indexOf(buf[pos]) < 0) return false; break; case "function": // More complex calculation - if (!sig[offset](buf[offset])) + if (!sig[sigoffset](buf[pos])) return false; break; default: - throw new Error(`Unrecognised signature type at offset ${offset}`); + throw new Error(`Unrecognised signature type at offset ${sigoffset}`); } } return true; @@ -91,6 +102,46 @@ export function detectFileType(buf) { } +/** + * Given a buffer, searches for magic byte sequences at all possible positions and returns + * the extensions and mime types. + * + * @param {Uint8Array} buf + * @returns {Object[]} foundFiles + * @returns {number} foundFiles.offset - The position in the buffer at which this file was found + * @returns {Object} foundFiles.fileDetails + * @returns {string} foundFiles.fileDetails.name - Name of file type + * @returns {string} foundFiles.fileDetails.ext - File extension + * @returns {string} foundFiles.fileDetails.mime - Mime type + * @returns {string} [foundFiles.fileDetails.desc] - Description + */ +export function scanForFileTypes(buf) { + if (!(buf && buf.length > 1)) { + return []; + } + + const foundFiles = []; + + // TODO allow user to select which categories to check + for (const cat in FILE_SIGNATURES) { + const category = FILE_SIGNATURES[cat]; + + for (let i = 0; i < category.length; i++) { + const filetype = category[i]; + for (let pos = 0; pos < buf.length; pos++) { + if (signatureMatches(filetype.signature, buf, pos)) { + foundFiles.push({ + offset: pos, + fileDetails: filetype + }); + } + } + } + } + return foundFiles; +} + + /** * Detects whether the given buffer is a file of the type specified. * diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index 3a87cd5e..da9d57a9 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -7,7 +7,7 @@ import Operation from "../Operation"; // import OperationError from "../errors/OperationError"; import Utils from "../Utils"; -import {detectFileType, extractFile} from "../lib/FileType"; +import {scanForFileTypes, extractFile} from "../lib/FileType"; /** * Extract Files operation @@ -39,7 +39,7 @@ class ExtractFiles extends Operation { const bytes = new Uint8Array(input); // Scan for embedded files - const detectedFiles = scanForEmbeddedFiles(bytes); + const detectedFiles = scanForFileTypes(bytes); // Extract each file that we support const files = []; @@ -64,26 +64,4 @@ class ExtractFiles extends Operation { } -/** - * TODO refactor - * @param data - */ -function scanForEmbeddedFiles(data) { - const detectedFiles = []; - - for (let i = 0; i < data.length; i++) { - const fileDetails = detectFileType(data.slice(i)); - if (fileDetails.length) { - fileDetails.forEach(match => { - detectedFiles.push({ - offset: i, - fileDetails: match, - }); - }); - } - } - - return detectedFiles; -} - export default ExtractFiles; diff --git a/src/core/operations/ScanForEmbeddedFiles.mjs b/src/core/operations/ScanForEmbeddedFiles.mjs index 41ea911b..a0465e83 100644 --- a/src/core/operations/ScanForEmbeddedFiles.mjs +++ b/src/core/operations/ScanForEmbeddedFiles.mjs @@ -6,7 +6,7 @@ import Operation from "../Operation"; import Utils from "../Utils"; -import {detectFileType} from "../lib/FileType"; +import {scanForFileTypes} from "../lib/FileType"; /** * Scan for Embedded Files operation @@ -41,32 +41,30 @@ class ScanForEmbeddedFiles extends Operation { */ run(input, args) { let output = "Scanning data for 'magic bytes' which may indicate embedded files. The following results may be false positives and should not be treat as reliable. Any suffiently long file is likely to contain these magic bytes coincidentally.\n", - types, numFound = 0, numCommonFound = 0; const ignoreCommon = args[0], - commonExts = ["ico", "ttf", ""], - data = new Uint8Array(input); + commonExts = ["ttf", "utf16le", ""], + data = new Uint8Array(input), + types = scanForFileTypes(data); - for (let i = 0; i < data.length; i++) { - types = detectFileType(data.slice(i)); - if (types.length) { - types.forEach(type => { - if (ignoreCommon && commonExts.indexOf(type.extension) > -1) { - numCommonFound++; - return; - } - numFound++; - output += "\nOffset " + i + " (0x" + Utils.hex(i) + "):\n" + - " File extension: " + type.extension + "\n" + - " MIME type: " + type.mime + "\n"; + if (types.length) { + types.forEach(type => { + if (ignoreCommon && commonExts.indexOf(type.fileDetails.extension) > -1) { + numCommonFound++; + return; + } - if (type.description && type.description.length) { - output += " Description: " + type.description + "\n"; - } - }); - } + numFound++; + output += "\nOffset " + type.offset + " (0x" + Utils.hex(type.offset) + "):\n" + + " File extension: " + type.fileDetails.extension + "\n" + + " MIME type: " + type.fileDetails.mime + "\n"; + + if (type.fileDetails.description && type.fileDetails.description.length) { + output += " Description: " + type.fileDetails.description + "\n"; + } + }); } if (numFound === 0) { From a56f92cdee6ae9f44e20610a59dfdff02c94a185 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 2 Jan 2019 17:50:47 +0000 Subject: [PATCH 10/22] Significantly improved performance when scanning for embedded files by implementing a fastcheck algorithm. --- package-lock.json | 148 +++++++++++++++++++------------------- src/core/lib/FileType.mjs | 67 ++++++++++++++--- 2 files changed, 131 insertions(+), 84 deletions(-) diff --git a/package-lock.json b/package-lock.json index ac28bda1..fa857e05 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1497,7 +1497,7 @@ }, "ansi-escapes": { "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", + "resolved": "http://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", "dev": true }, @@ -1615,7 +1615,7 @@ }, "array-equal": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, @@ -1700,7 +1700,7 @@ }, "util": { "version": "0.10.3", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1800,7 +1800,7 @@ }, "axios": { "version": "0.18.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", "dev": true, "requires": { @@ -2232,7 +2232,7 @@ }, "browserify-aes": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", "dev": true, "requires": { @@ -2269,7 +2269,7 @@ }, "browserify-rsa": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", "dev": true, "requires": { @@ -2319,7 +2319,7 @@ }, "buffer": { "version": "4.9.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "dev": true, "requires": { @@ -2389,7 +2389,7 @@ }, "cacache": { "version": "10.0.4", - "resolved": "https://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", + "resolved": "http://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", "integrity": "sha512-Dph0MzuH+rTQzGPNT9fAnrPmMmjKfST6trxJeK7NQuHRaVw24VzPRWTmg9MpcwOVQZO0E1FBICUlFeNaKPIfHA==", "dev": true, "requires": { @@ -2466,7 +2466,7 @@ }, "camelcase-keys": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", "dev": true, "requires": { @@ -2515,7 +2515,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -3048,7 +3048,7 @@ }, "create-hash": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", "dev": true, "requires": { @@ -3061,7 +3061,7 @@ }, "create-hmac": { "version": "1.1.7", - "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", "dev": true, "requires": { @@ -3179,7 +3179,7 @@ }, "css-select": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", "dev": true, "requires": { @@ -3547,7 +3547,7 @@ }, "diffie-hellman": { "version": "5.0.3", - "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "resolved": "http://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", "dev": true, "requires": { @@ -3611,7 +3611,7 @@ "dependencies": { "domelementtype": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", "dev": true }, @@ -3816,7 +3816,7 @@ }, "entities": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/entities/-/entities-1.0.0.tgz", "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=", "dev": true }, @@ -4240,7 +4240,7 @@ }, "eventemitter2": { "version": "0.4.14", - "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", + "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", "integrity": "sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas=", "dev": true }, @@ -4252,7 +4252,7 @@ }, "events": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", "dev": true }, @@ -4675,7 +4675,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -4911,7 +4911,7 @@ }, "fs-extra": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", "dev": true, "requires": { @@ -5579,7 +5579,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, @@ -5721,7 +5721,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -5798,7 +5798,7 @@ }, "grunt-cli": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", "integrity": "sha1-VisRnrsGndtGSs4oRVAb6Xs1tqg=", "dev": true, "requires": { @@ -5846,7 +5846,7 @@ "dependencies": { "shelljs": { "version": "0.5.3", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", + "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", "integrity": "sha1-xUmCuZbHbvDB5rWfvcWCX1txMRM=", "dev": true } @@ -5866,7 +5866,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -5911,7 +5911,7 @@ }, "grunt-contrib-jshint": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", "integrity": "sha1-Np2QmyWTxA6L55lAshNAhQx5Oaw=", "dev": true, "requires": { @@ -6010,7 +6010,7 @@ "dependencies": { "colors": { "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", "dev": true } @@ -6074,7 +6074,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -6102,7 +6102,7 @@ }, "handle-thing": { "version": "1.2.5", - "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", + "resolved": "http://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", "integrity": "sha1-/Xqtcmvxpf0W38KbL3pmAdJxOcQ=", "dev": true }, @@ -6335,7 +6335,7 @@ }, "html-webpack-plugin": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", + "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz", "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=", "dev": true, "requires": { @@ -6391,7 +6391,7 @@ }, "htmlparser2": { "version": "3.8.3", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", "dev": true, "requires": { @@ -6410,7 +6410,7 @@ }, "http-errors": { "version": "1.6.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", "dev": true, "requires": { @@ -6460,7 +6460,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -6938,7 +6938,7 @@ }, "is-builtin-module": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", "dev": true, "requires": { @@ -7505,7 +7505,7 @@ }, "jsonfile": { "version": "2.4.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "resolved": "http://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", "dev": true, "requires": { @@ -7616,7 +7616,7 @@ }, "kew": { "version": "0.7.0", - "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz", + "resolved": "http://registry.npmjs.org/kew/-/kew-0.7.0.tgz", "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=", "dev": true }, @@ -7730,7 +7730,7 @@ }, "load-json-file": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "dev": true, "requires": { @@ -7743,7 +7743,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -8102,7 +8102,7 @@ }, "media-typer": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", "dev": true }, @@ -8161,7 +8161,7 @@ }, "meow": { "version": "3.7.0", - "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "resolved": "http://registry.npmjs.org/meow/-/meow-3.7.0.tgz", "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", "dev": true, "requires": { @@ -8346,7 +8346,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "requires": { "minimist": "0.0.8" @@ -8556,7 +8556,7 @@ }, "ncp": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", "integrity": "sha1-0VNn5cuHQyuhF9K/gP30Wuz7QkY=", "dev": true }, @@ -8655,7 +8655,7 @@ "dependencies": { "semver": { "version": "5.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "resolved": "http://registry.npmjs.org/semver/-/semver-5.3.0.tgz", "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", "dev": true } @@ -8827,7 +8827,7 @@ "dependencies": { "colors": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz", "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" }, "underscore": { @@ -9121,13 +9121,13 @@ }, "os-homedir": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true }, "os-locale": { "version": "1.4.0", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", "dev": true, "requires": { @@ -9136,7 +9136,7 @@ }, "os-tmpdir": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, @@ -9351,7 +9351,7 @@ }, "parse-asn1": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", + "resolved": "http://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==", "dev": true, "requires": { @@ -9437,7 +9437,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, @@ -9478,7 +9478,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -9661,7 +9661,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -10073,7 +10073,7 @@ }, "progress": { "version": "1.1.8", - "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", + "resolved": "http://registry.npmjs.org/progress/-/progress-1.1.8.tgz", "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=" }, "promise-inflight": { @@ -10098,13 +10098,13 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true }, "winston": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz", + "resolved": "http://registry.npmjs.org/winston/-/winston-2.1.1.tgz", "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=", "dev": true, "requires": { @@ -10119,7 +10119,7 @@ "dependencies": { "colors": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true }, @@ -10342,7 +10342,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -10531,7 +10531,7 @@ "dependencies": { "jsesc": { "version": "0.5.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "resolved": "http://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=", "dev": true } @@ -10582,7 +10582,7 @@ }, "htmlparser2": { "version": "3.3.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz", + "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz", "integrity": "sha1-zHDQWln2VC5D8OaFyYLhTJJKnv4=", "dev": true, "requires": { @@ -10594,7 +10594,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "dev": true, "requires": { @@ -10704,7 +10704,7 @@ }, "require-uncached": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=", "dev": true, "requires": { @@ -10871,7 +10871,7 @@ }, "safe-regex": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", "dev": true, "requires": { @@ -11191,7 +11191,7 @@ }, "sha.js": { "version": "2.4.11", - "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "resolved": "http://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", "dev": true, "requires": { @@ -11235,7 +11235,7 @@ }, "shelljs": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=", "dev": true }, @@ -11913,7 +11913,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -11930,7 +11930,7 @@ }, "strip-eof": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "dev": true }, @@ -12009,7 +12009,7 @@ }, "tar": { "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz", + "resolved": "http://registry.npmjs.org/tar/-/tar-2.2.1.tgz", "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=", "dev": true, "requires": { @@ -12064,7 +12064,7 @@ }, "through": { "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", "dev": true }, @@ -12758,7 +12758,7 @@ "dependencies": { "async": { "version": "0.9.2", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz", "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=", "dev": true }, @@ -12784,7 +12784,7 @@ }, "valid-data-url": { "version": "0.1.6", - "resolved": "https://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", + "resolved": "http://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", "integrity": "sha512-FXg2qXMzfAhZc0y2HzELNfUeiOjPr+52hU1DNBWiJJ2luXD+dD1R9NA48Ug5aj0ibbxroeGDc/RJv6ThiGgkDw==", "dev": true }, @@ -12800,7 +12800,7 @@ }, "validator": { "version": "9.4.1", - "resolved": "https://registry.npmjs.org/validator/-/validator-9.4.1.tgz", + "resolved": "http://registry.npmjs.org/validator/-/validator-9.4.1.tgz", "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA==", "dev": true }, @@ -13296,7 +13296,7 @@ }, "webpack-node-externals": { "version": "1.7.2", - "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", + "resolved": "http://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", "integrity": "sha512-ajerHZ+BJKeCLviLUUmnyd5B4RavLF76uv3cs6KNuO8W+HuQaEs0y0L7o40NQxdPy5w0pcv8Ew7yPUAQG0UdCg==", "dev": true }, @@ -13450,14 +13450,14 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true, "optional": true }, "colors": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true, "optional": true @@ -13490,7 +13490,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "dev": true, "requires": { diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index 5e1dd657..93c7634a 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -128,17 +128,58 @@ export function scanForFileTypes(buf) { for (let i = 0; i < category.length; i++) { const filetype = category[i]; - for (let pos = 0; pos < buf.length; pos++) { - if (signatureMatches(filetype.signature, buf, pos)) { - foundFiles.push({ - offset: pos, - fileDetails: filetype - }); + const sigs = filetype.signature.length ? filetype.signature : [filetype.signature]; + + sigs.forEach(sig => { + let pos = 0; + while ((pos = locatePotentialSig(buf, sig, pos)) >= 0) { + if (signatureMatches(sig, buf, pos)) { + foundFiles.push({ + offset: pos, + fileDetails: filetype + }); + } + pos++; } - } + }); } } - return foundFiles; + + // Return found files in order of increasing offset + return foundFiles.sort((a, b) => { + return a.offset - b.offset; + }); +} + + +/** + * Fastcheck function to quickly scan the buffer for the first byte in a signature. + * + * @param {Uint8Array} buf - The buffer to search + * @param {Object} sig - A single signature object (Not an array of signatures) + * @param {number} offset - Where to start search from + * @returs {number} The position of the match or -1 if one cannot be found. + */ +function locatePotentialSig(buf, sig, offset) { + // Find values for first key and value in sig + const k = parseInt(Object.keys(sig)[0], 10); + const v = Object.values(sig)[0]; + switch (typeof v) { + case "number": + return buf.indexOf(v, offset + k) - k; + case "object": + for (let i = offset + k; i < buf.length; i++) { + if (v.indexOf(buf[i]) >= 0) return i - k; + } + return -1; + case "function": + for (let i = offset + k; i < buf.length; i++) { + if (v(buf[i])) return i - k; + } + return -1; + default: + throw new Error("Unrecognised signature type"); + } } @@ -155,9 +196,15 @@ export function isType(type, buf) { if (!(types && types.length)) return false; if (typeof type === "string") { - return types[0].mime.startsWith(type) ? types[0].mime : false; + return types.reduce((acc, t) => { + const mime = t.mime.startsWith(type) ? t.mime : false; + return acc || mime; + }, false); } else if (type instanceof RegExp) { - return type.test(types[0].mime) ? types[0].mime : false; + return types.reduce((acc, t) => { + const mime = type.test(t.mime) ? t.mime : false; + return acc || mime; + }, false); } else { throw new Error("Invalid type input."); } From cd0c86e0d66ea8df0771282b0333b90607037188 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 3 Jan 2019 13:03:41 +0000 Subject: [PATCH 11/22] File scan now uses bytesMatch() instead of signatureMatches(), reducing call stack size --- src/core/lib/FileType.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index 93c7634a..202b54d9 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -133,7 +133,7 @@ export function scanForFileTypes(buf) { sigs.forEach(sig => { let pos = 0; while ((pos = locatePotentialSig(buf, sig, pos)) >= 0) { - if (signatureMatches(sig, buf, pos)) { + if (bytesMatch(sig, buf, pos)) { foundFiles.push({ offset: pos, fileDetails: filetype From 0449c46b381d5d1ff35c65aa32f7a94d654e964e Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 3 Jan 2019 18:40:22 +0000 Subject: [PATCH 12/22] Added FLV extractor. --- src/core/lib/FileSignatures.mjs | 48 ++++++++++++++++++++++++++++++++- src/core/lib/Stream.mjs | 39 ++++++++++++++++++++++----- 2 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index d4e19da9..80ea77d1 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -417,7 +417,7 @@ export const FILE_SIGNATURES = { 2: 0x56, 3: 0x1 }, - extractor: null + extractor: extractFLV }, ], "Audio": [ @@ -1285,3 +1285,49 @@ export function extractBMP(bytes, offset) { return stream.carve(); } + + +/** + * FLV extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractFLV(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Move past signature, version and flags + stream.moveForwardsBy(5); + + // Read header size + const headerSize = stream.readInt(4, "be"); + + // Skip through the rest of the header + stream.moveForwardsBy(headerSize - 9); + + let tagSize = -11; // Fake size of previous tag header + while (stream.position < stream.length) { + const prevTagSize = stream.readInt(4, "be"); + const tagType = stream.readInt(1, "be"); + + if ([8, 9, 18].indexOf(tagType) < 0) { + // This tag is not valid + stream.moveBackwardsBy(1); + break; + } + + if (prevTagSize !== tagSize + 11) { + // Previous tag was not valid + stream.moveBackwardsBy(tagSize + 11); + break; + } + + tagSize = stream.readInt(3, "be"); + + // Move past the rest of the tag header and payload + stream.moveForwardsBy(7 + tagSize); + } + + return stream.carve(); +} diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs index 19903117..0a29b5a6 100644 --- a/src/core/lib/Stream.mjs +++ b/src/core/lib/Stream.mjs @@ -22,6 +22,7 @@ export default class Stream { constructor(input) { this.bytes = input; this.position = 0; + this.length = this.bytes.length; } /** @@ -31,6 +32,8 @@ export default class Stream { * @returns {Uint8Array} */ getBytes(numBytes) { + if (this.position > this.length) return undefined; + const newPosition = this.position + numBytes; const bytes = this.bytes.slice(this.position, newPosition); this.position = newPosition; @@ -45,6 +48,8 @@ export default class Stream { * @returns {string} */ readString(numBytes) { + if (this.position > this.length) return undefined; + let result = ""; for (let i = this.position; i < this.position + numBytes; i++) { const currentByte = this.bytes[i]; @@ -63,6 +68,8 @@ export default class Stream { * @returns {number} */ readInt(numBytes, endianness="be") { + if (this.position > this.length) return undefined; + let val = 0; if (endianness === "be") { for (let i = this.position; i < this.position + numBytes; i++) { @@ -85,8 +92,10 @@ export default class Stream { * @param {number|List} val */ continueUntil(val) { + if (this.position > this.length) return; + if (typeof val === "number") { - while (++this.position < this.bytes.length && this.bytes[this.position] !== val) { + while (++this.position < this.length && this.bytes[this.position] !== val) { continue; } return; @@ -94,13 +103,13 @@ export default class Stream { // val is an array let found = false; - while (!found && this.position < this.bytes.length) { - while (++this.position < this.bytes.length && this.bytes[this.position] !== val[0]) { + while (!found && this.position < this.length) { + while (++this.position < this.length && this.bytes[this.position] !== val[0]) { continue; } found = true; for (let i = 1; i < val.length; i++) { - if (this.position + i > this.bytes.length || this.bytes[this.position + i] !== val[i]) + if (this.position + i > this.length || this.bytes[this.position + i] !== val[i]) found = false; } } @@ -122,7 +131,23 @@ export default class Stream { * @param {number} numBytes */ moveForwardsBy(numBytes) { - this.position += numBytes; + const pos = this.position + numBytes; + if (pos < 0 || pos > this.length) + throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); + this.position = pos; + } + + + /** + * Move backwards through the stream by the specified number of bytes. + * + * @param {number} numBytes + */ + moveBackwardsBy(numBytes) { + const pos = this.position - numBytes; + if (pos < 0 || pos > this.length) + throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); + this.position = pos; } /** @@ -131,7 +156,7 @@ export default class Stream { * @param {number} pos */ moveTo(pos) { - if (pos < 0 || pos > this.bytes.length - 1) + if (pos < 0 || pos > this.length) throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); this.position = pos; } @@ -142,7 +167,7 @@ export default class Stream { * @returns {boolean} */ hasMore() { - return this.position < this.bytes.length; + return this.position < this.length; } /** From 7d8d80ca2c4b89f5b62e74e3b503ed285eabe059 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 3 Jan 2019 19:01:12 +0000 Subject: [PATCH 13/22] Added extractor for MS Office 2007+ files --- src/core/lib/FileSignatures.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 80ea77d1..e879fcf7 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -609,7 +609,7 @@ export const FILE_SIGNATURES = { 47: 0x6d, 48: 0x6c }, - extractor: null + extractor: extractZIP }, { name: "EPUB e-book", From 0d2cb02f975c9a3186cf16576ee243281f780819 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 4 Jan 2019 11:49:12 +0000 Subject: [PATCH 14/22] Fixed FLV previous tag size error --- src/core/lib/FileSignatures.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index e879fcf7..e55bd566 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1317,9 +1317,10 @@ export function extractFLV(bytes, offset) { break; } - if (prevTagSize !== tagSize + 11) { - // Previous tag was not valid - stream.moveBackwardsBy(tagSize + 11); + if (prevTagSize !== (tagSize + 11)) { + // Previous tag was not valid, reverse back over this header + // and the previous tag body and header + stream.moveBackwardsBy(tagSize + 11 + 5); break; } From 19b795752372525bb811d9ea8c2de9bec3c28f9a Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 4 Jan 2019 14:57:31 +0000 Subject: [PATCH 15/22] Added RTF extractor --- src/core/lib/FileSignatures.mjs | 44 +++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index e55bd566..4409c96c 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -572,7 +572,7 @@ export const FILE_SIGNATURES = { 3: 0x74, 4: 0x66 }, - extractor: null + extractor: extractRTF }, { name: "Microsoft Office documents/OLE2", @@ -1307,7 +1307,7 @@ export function extractFLV(bytes, offset) { stream.moveForwardsBy(headerSize - 9); let tagSize = -11; // Fake size of previous tag header - while (stream.position < stream.length) { + while (stream.hasMore()) { const prevTagSize = stream.readInt(4, "be"); const tagType = stream.readInt(1, "be"); @@ -1332,3 +1332,43 @@ export function extractFLV(bytes, offset) { return stream.carve(); } + + +/** + * RTF extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractRTF(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + let openTags = 0; + + if (stream.readInt(1, "be") !== 0x7b) { // { + throw new Error("Not a valid RTF file"); + } else { + openTags++; + } + + while (openTags > 0 && stream.hasMore()) { + switch (stream.readInt(1, "be")) { + case 0x7b: // { + openTags++; + break; + case 0x7d: // } + openTags--; + break; + case 0x5c: // \ + // Consume any more escapes and then skip over the next character + stream.consumeIf(0x5c); + stream.position++; + break; + default: + break; + } + } + + return stream.carve(); +} From 2a6db47aeb1add115bbf859c12a1be468812bb03 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 4 Jan 2019 18:12:49 +0000 Subject: [PATCH 16/22] Began implementing GZIP/DEFLATE extraction. Unfinished. --- src/core/lib/FileSignatures.mjs | 109 ++++++++++++++++++++++++++++++-- src/core/lib/Stream.mjs | 65 ++++++++++++++++++- 2 files changed, 167 insertions(+), 7 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 4409c96c..b0eb5884 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -650,7 +650,7 @@ export const FILE_SIGNATURES = { 56: 0x69, 57: 0x70 }, - extractor: null + extractor: extractZIP }, ], "Applications": [ @@ -790,7 +790,7 @@ export const FILE_SIGNATURES = { 1: 0x8b, 2: 0x8 }, - extractor: null + extractor: extractGZIP }, { name: "Bzip2", @@ -1309,7 +1309,7 @@ export function extractFLV(bytes, offset) { let tagSize = -11; // Fake size of previous tag header while (stream.hasMore()) { const prevTagSize = stream.readInt(4, "be"); - const tagType = stream.readInt(1, "be"); + const tagType = stream.readInt(1); if ([8, 9, 18].indexOf(tagType) < 0) { // This tag is not valid @@ -1346,14 +1346,14 @@ export function extractRTF(bytes, offset) { let openTags = 0; - if (stream.readInt(1, "be") !== 0x7b) { // { + if (stream.readInt(1) !== 0x7b) { // { throw new Error("Not a valid RTF file"); } else { openTags++; } while (openTags > 0 && stream.hasMore()) { - switch (stream.readInt(1, "be")) { + switch (stream.readInt(1)) { case 0x7b: // { openTags++; break; @@ -1372,3 +1372,102 @@ export function extractRTF(bytes, offset) { return stream.carve(); } + + +/** + * GZIP extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractGZIP(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + /* HEADER */ + + // Skip over signature and compression method + stream.moveForwardsBy(3); + + // Read flags + const flags = stream.readInt(1); + + // Skip over last modification time + stream.moveForwardsBy(4); + + // Read compression flags + const compressionFlags = stream.readInt(1); + + // Skip over OS + stream.moveForwardsBy(1); + + + /* OPTIONAL HEADERS */ + + // Extra fields + if (flags & 0x4) { + console.log("Extra fields"); + const extraFieldsSize = stream.readInt(2, "le"); + stream.moveForwardsby(extraFieldsSize); + } + + // Original filename + if (flags & 0x8) { + console.log("Filename"); + stream.continueUntil(0x00); + stream.moveForwardsBy(1); + } + + // Comment + if (flags & 0x10) { + console.log("Comment"); + stream.continueUntil(0x00); + stream.moveForwardsBy(1); + } + + // Checksum + if (flags & 0x2) { + console.log("Checksum"); + stream.moveForwardsBy(2); + } + + + /* DEFLATE DATA */ + + let finalBlock = 0; + + while (!finalBlock) { + // Read header + const blockHeader = stream.readBits(3); + + finalBlock = blockHeader & 0x1; + const blockType = blockHeader & 0x6; + + if (blockType === 0) { + // No compression + stream.moveForwardsBy(1); + const blockLength = stream.readInt(2, "le"); + console.log("No compression. Length: " + blockLength); + stream.moveForwardsBy(2 + blockLength); + } else if (blockType === 1) { + // Fixed Huffman + + } else if (blockType === 2) { + // Dynamic Huffman + + } else { + throw new Error("Invalid block type"); + break; + } + } + + + /* FOOTER */ + + // Skip over checksum and size of original uncompressed input + stream.moveForwardsBy(8); + + console.log(stream.position); + + return stream.carve(); +} diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs index 0a29b5a6..2b5d8d09 100644 --- a/src/core/lib/Stream.mjs +++ b/src/core/lib/Stream.mjs @@ -21,8 +21,9 @@ export default class Stream { */ constructor(input) { this.bytes = input; - this.position = 0; this.length = this.bytes.length; + this.position = 0; + this.bitPos = 0; } /** @@ -37,6 +38,7 @@ export default class Stream { const newPosition = this.position + numBytes; const bytes = this.bytes.slice(this.position, newPosition); this.position = newPosition; + this.bitPos = 0; return bytes; } @@ -57,6 +59,7 @@ export default class Stream { result += String.fromCharCode(currentByte); } this.position += numBytes; + this.bitPos = 0; return result; } @@ -83,9 +86,59 @@ export default class Stream { } } this.position += numBytes; + this.bitPos = 0; return val; } + + /** + * Reads a number of bits from the buffer. + * + * @TODO Add endianness + * + * @param {number} numBits + * @returns {number} + */ + readBits(numBits) { + if (this.position > this.length) return undefined; + + let bitBuf = 0, + bitBufLen = 0; + + // Add remaining bits from current byte + bitBuf = this.bytes[this.position++] & bitMask(this.bitPos); + bitBufLen = 8 - this.bitPos; + this.bitPos = 0; + + // Not enough bits yet + while (bitBufLen < numBits) { + bitBuf |= this.bytes[this.position++] << bitBufLen; + bitBufLen += 8; + } + + // Reverse back to numBits + if (bitBufLen > numBits) { + const excess = bitBufLen - numBits; + bitBuf >>>= excess; + bitBufLen -= excess; + this.position--; + this.bitPos = 8 - excess; + } + + return bitBuf; + + /** + * Calculates the bit mask based on the current bit position. + * + * @param {number} bitPos + * @returns {number} The bit mask + */ + function bitMask(bitPos) { + return (1 << (8 - bitPos)) - 1; + } + } + + /** * Consume the stream until we reach the specified byte or sequence of bytes. * @@ -94,6 +147,8 @@ export default class Stream { continueUntil(val) { if (this.position > this.length) return; + this.bitPos = 0; + if (typeof val === "number") { while (++this.position < this.length && this.bytes[this.position] !== val) { continue; @@ -121,8 +176,10 @@ export default class Stream { * @param {number} val */ consumeIf(val) { - if (this.bytes[this.position] === val) + if (this.bytes[this.position] === val) { this.position++; + this.bitPos = 0; + } } /** @@ -135,6 +192,7 @@ export default class Stream { if (pos < 0 || pos > this.length) throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); this.position = pos; + this.bitPos = 0; } @@ -148,6 +206,7 @@ export default class Stream { if (pos < 0 || pos > this.length) throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); this.position = pos; + this.bitPos = 0; } /** @@ -159,6 +218,7 @@ export default class Stream { if (pos < 0 || pos > this.length) throw new Error("Cannot move to position " + pos + " in stream. Out of bounds."); this.position = pos; + this.bitPos = 0; } /** @@ -176,6 +236,7 @@ export default class Stream { * @returns {Uint8Array} */ carve() { + if (this.bitPos > 0) this.position++; return this.bytes.slice(0, this.position); } From c077b22410037559d0e8af9fedc6f0971a67b9ab Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 10 Jan 2019 17:30:52 +0000 Subject: [PATCH 17/22] Stream.readBits() method implemented. Unfinished. --- src/core/lib/FileSignatures.mjs | 12 +++++------- src/core/lib/Stream.mjs | 7 +++---- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index b0eb5884..d8d97a2b 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1438,10 +1438,8 @@ export function extractGZIP(bytes, offset) { while (!finalBlock) { // Read header - const blockHeader = stream.readBits(3); - - finalBlock = blockHeader & 0x1; - const blockType = blockHeader & 0x6; + finalBlock = stream.readBits(1); + const blockType = stream.readBits(2); if (blockType === 0) { // No compression @@ -1451,23 +1449,23 @@ export function extractGZIP(bytes, offset) { stream.moveForwardsBy(2 + blockLength); } else if (blockType === 1) { // Fixed Huffman + console.log("Fixed Huffman"); } else if (blockType === 2) { // Dynamic Huffman - + console.log("Dynamic Huffman"); } else { throw new Error("Invalid block type"); break; } } - /* FOOTER */ // Skip over checksum and size of original uncompressed input stream.moveForwardsBy(8); - console.log(stream.position); + console.log("Ending at " + stream.position); return stream.carve(); } diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs index 2b5d8d09..886d4589 100644 --- a/src/core/lib/Stream.mjs +++ b/src/core/lib/Stream.mjs @@ -106,7 +106,7 @@ export default class Stream { bitBufLen = 0; // Add remaining bits from current byte - bitBuf = this.bytes[this.position++] & bitMask(this.bitPos); + bitBuf = (this.bytes[this.position++] & bitMask(this.bitPos)) >>> this.bitPos; bitBufLen = 8 - this.bitPos; this.bitPos = 0; @@ -119,7 +119,7 @@ export default class Stream { // Reverse back to numBits if (bitBufLen > numBits) { const excess = bitBufLen - numBits; - bitBuf >>>= excess; + bitBuf &= (1 << numBits) - 1; bitBufLen -= excess; this.position--; this.bitPos = 8 - excess; @@ -134,11 +134,10 @@ export default class Stream { * @returns {number} The bit mask */ function bitMask(bitPos) { - return (1 << (8 - bitPos)) - 1; + return 256 - (1 << bitPos); } } - /** * Consume the stream until we reach the specified byte or sequence of bytes. * From 4e57b4be885a4ba6cef7c4132232b3675be09cba Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 11 Jan 2019 17:44:13 +0000 Subject: [PATCH 18/22] Completed GZIP extraction --- src/core/lib/FileSignatures.mjs | 215 ++++++++++++++++++++++++--- src/core/lib/Stream.mjs | 25 +++- src/core/operations/ExtractFiles.mjs | 5 +- 3 files changed, 222 insertions(+), 23 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index d8d97a2b..b2b1ff06 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1384,6 +1384,7 @@ export function extractRTF(bytes, offset) { export function extractGZIP(bytes, offset) { const stream = new Stream(bytes.slice(offset)); + /* HEADER */ // Skip over signature and compression method @@ -1396,7 +1397,7 @@ export function extractGZIP(bytes, offset) { stream.moveForwardsBy(4); // Read compression flags - const compressionFlags = stream.readInt(1); + stream.readInt(1); // Skip over OS stream.moveForwardsBy(1); @@ -1406,34 +1407,63 @@ export function extractGZIP(bytes, offset) { // Extra fields if (flags & 0x4) { - console.log("Extra fields"); const extraFieldsSize = stream.readInt(2, "le"); stream.moveForwardsby(extraFieldsSize); } // Original filename if (flags & 0x8) { - console.log("Filename"); stream.continueUntil(0x00); stream.moveForwardsBy(1); } // Comment if (flags & 0x10) { - console.log("Comment"); stream.continueUntil(0x00); stream.moveForwardsBy(1); } // Checksum if (flags & 0x2) { - console.log("Checksum"); stream.moveForwardsBy(2); } /* DEFLATE DATA */ + parseDEFLATE(stream); + + + /* FOOTER */ + + // Skip over checksum and size of original uncompressed input + stream.moveForwardsBy(8); + + return stream.carve(); +} + + +/** + * Steps through a DEFLATE stream + * + * @param {Stream} stream + */ +function parseDEFLATE(stream) { + // Construct required Huffman Tables + const fixedLiteralTableLengths = new Uint8Array(288); + for (let i = 0; i < fixedLiteralTableLengths.length; i++) { + fixedLiteralTableLengths[i] = + (i <= 143) ? 8 : + (i <= 255) ? 9 : + (i <= 279) ? 7 : + 8; + } + const fixedLiteralTable = buildHuffmanTable(fixedLiteralTableLengths); + const fixedDistanceTableLengths = new Uint8Array(30).fill(5); + const fixedDistanceTable = buildHuffmanTable(fixedDistanceTableLengths); + const huffmanOrder = new Uint8Array([16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]); + + // Parse DEFLATE data let finalBlock = 0; while (!finalBlock) { @@ -1442,30 +1472,175 @@ export function extractGZIP(bytes, offset) { const blockType = stream.readBits(2); if (blockType === 0) { - // No compression + /* No compression */ + + // Consume the rest of the current byte stream.moveForwardsBy(1); + // Read the block length value const blockLength = stream.readInt(2, "le"); - console.log("No compression. Length: " + blockLength); + // Move to the end of this block stream.moveForwardsBy(2 + blockLength); } else if (blockType === 1) { - // Fixed Huffman - console.log("Fixed Huffman"); + /* Fixed Huffman */ + parseHuffmanBlock(stream, fixedLiteralTable, fixedDistanceTable); } else if (blockType === 2) { - // Dynamic Huffman - console.log("Dynamic Huffman"); + /* Dynamic Huffman */ + + // Read the number of liternal and length codes + const hlit = stream.readBits(5) + 257; + // Read the number of distance codes + const hdist = stream.readBits(5) + 1; + // Read the number of code lengths + const hclen = stream.readBits(4) + 4; + + // Parse code lengths + const codeLengths = new Uint8Array(huffmanOrder.length); + for (let i = 0; i < hclen; i++) { + codeLengths[huffmanOrder[i]] = stream.readBits(3); + } + + // Parse length table + const codeLengthsTable = buildHuffmanTable(codeLengths); + const lengthTable = new Uint8Array(hlit + hdist); + + let code, repeat, prev; + for (let i = 0; i < hlit + hdist;) { + code = readHuffmanCode(stream, codeLengthsTable); + switch (code) { + case 16: + repeat = 3 + stream.readBits(2); + while (repeat--) lengthTable[i++] = prev; + break; + case 17: + repeat = 3 + stream.readBits(3); + while (repeat--) lengthTable[i++] = 0; + prev = 0; + break; + case 18: + repeat = 11 + stream.readBits(7); + while (repeat--) lengthTable[i++] = 0; + prev = 0; + break; + default: + lengthTable[i++] = code; + prev = code; + break; + } + } + + const dynamicLiteralTable = buildHuffmanTable(lengthTable.subarray(0, hlit)); + const dynamicDistanceTable = buildHuffmanTable(lengthTable.subarray(hlit)); + + parseHuffmanBlock(stream, dynamicLiteralTable, dynamicDistanceTable); } else { throw new Error("Invalid block type"); - break; } } - /* FOOTER */ - - // Skip over checksum and size of original uncompressed input - stream.moveForwardsBy(8); - - console.log("Ending at " + stream.position); - - return stream.carve(); + // Consume final byte if it has not been fully consumed yet + if (stream.bitPos > 0) + stream.moveForwardsBy(1); +} + + +/** + * Parses a Huffman Block given the literal and distance tables + * + * @param {Stream} stream + * @param {Uint32Array} litTab + * @param {Uint32Array} distTab + */ +function parseHuffmanBlock(stream, litTab, distTab) { + const lengthExtraTable = new Uint8Array([ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0 + ]); + const distanceExtraTable = new Uint8Array([ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 + ]); + + let code; + while ((code = readHuffmanCode(stream, litTab))) { + // console.log("Code: " + code + " (" + Utils.chr(code) + ") " + Utils.bin(code)); + + // End of block + if (code === 256) break; + + // Literal + if (code < 256) continue; + + // Length code + stream.readBits(lengthExtraTable[code - 257]); + + // Dist code + code = readHuffmanCode(stream, distTab); + stream.readBits(distanceExtraTable[code]); + } +} + + +/** + * Builds a Huffman table given the relevant code lengths + * + * @param {Uint8Array} lengths + * @returns {Array} result + * @returns {Uint32Array} result.table + * @returns {number} result.maxCodeLength + * @returns {number} result.minCodeLength + */ +function buildHuffmanTable(lengths) { + const maxCodeLength = Math.max.apply(Math, lengths); + const minCodeLength = Math.min.apply(Math, lengths); + const size = 1 << maxCodeLength; + const table = new Uint32Array(size); + + for (let bitLength = 1, code = 0, skip = 2; bitLength <= maxCodeLength;) { + for (let i = 0; i < lengths.length; i++) { + if (lengths[i] === bitLength) { + let reversed, rtemp, j; + for (reversed = 0, rtemp = code, j = 0; j < bitLength; j++) { + reversed = (reversed << 1) | (rtemp & 1); + rtemp >>= 1; + } + + const value = (bitLength << 16) | i; + for (let j = reversed; j < size; j += skip) { + table[j] = value; + } + + code++; + } + } + + bitLength++; + code <<= 1; + skip <<= 1; + } + + return [table, maxCodeLength, minCodeLength]; +} + + +/** + * Reads the next Huffman code from the stream, given the relevant code table + * + * @param {Stream} stream + * @param {Uint32Array} table + * @returns {number} + */ +function readHuffmanCode(stream, table) { + const [codeTable, maxCodeLength] = table; + + // Read max length + const bitsBuf = stream.readBits(maxCodeLength); + const codeWithLength = codeTable[bitsBuf & ((1 << maxCodeLength) - 1)]; + const codeLength = codeWithLength >>> 16; + + if (codeLength > maxCodeLength) { + throw new Error("Invalid code length: " + codeLength); + } + + stream.moveBackwardsByBits(maxCodeLength - codeLength); + + return codeWithLength & 0xffff; } diff --git a/src/core/lib/Stream.mjs b/src/core/lib/Stream.mjs index 886d4589..7e82a5eb 100644 --- a/src/core/lib/Stream.mjs +++ b/src/core/lib/Stream.mjs @@ -90,7 +90,6 @@ export default class Stream { return val; } - /** * Reads a number of bits from the buffer. * @@ -194,7 +193,6 @@ export default class Stream { this.bitPos = 0; } - /** * Move backwards through the stream by the specified number of bytes. * @@ -208,6 +206,29 @@ export default class Stream { this.bitPos = 0; } + /** + * Move backwards through the strem by the specified number of bits. + * + * @param {number} numBits + */ + moveBackwardsByBits(numBits) { + if (numBits <= this.bitPos) { + this.bitPos -= numBits; + } else { + if (this.bitPos > 0) { + numBits -= this.bitPos; + this.bitPos = 0; + } + + while (numBits > 0) { + this.moveBackwardsBy(1); + this.bitPos = 8; + this.moveBackwardsByBits(numBits); + numBits -= 8; + } + } + } + /** * Move to a specified position in the stream. * diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index da9d57a9..0f861276 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -46,7 +46,10 @@ class ExtractFiles extends Operation { detectedFiles.forEach(detectedFile => { try { files.push(extractFile(bytes, detectedFile.fileDetails, detectedFile.offset)); - } catch (err) {} + } catch (err) { + if (err.message.indexOf("No extraction algorithm available") < 0) + throw err; + } }); return files; From 2307325af8ff20523f8a4e5c46503685d9184941 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 11 Jan 2019 17:58:25 +0000 Subject: [PATCH 19/22] Added Zlib extraction --- src/core/lib/FileSignatures.mjs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index b2b1ff06..f19688d1 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -828,7 +828,7 @@ export const FILE_SIGNATURES = { 0: 0x78, 1: [0x1, 0x9c, 0xda, 0x5e] }, - extractor: null + extractor: extractZlib }, { name: "xz compression", @@ -1443,6 +1443,37 @@ export function extractGZIP(bytes, offset) { } +/** + * Zlib extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractZlib(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Skip over CMF + stream.moveForwardsBy(1); + + // Read flags + const flags = stream.readInt(1); + + // Skip over preset dictionary checksum + if (flags & 0x20) { + stream.moveForwardsBy(4); + } + + // Parse DEFLATE stream + parseDEFLATE(stream); + + // Skip over final checksum + stream.moveForwardsBy(4); + + return stream.carve(); +} + + /** * Steps through a DEFLATE stream * From cd2c8078c8f42365eb4f0723831c1e62c21d9914 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Mon, 14 Jan 2019 18:55:10 +0000 Subject: [PATCH 20/22] Added ELF extractor. You can now specific which categories to search for in file type operations. --- src/core/lib/FileSignatures.mjs | 46 +++++++++++++++++++- src/core/lib/FileType.mjs | 26 ++++++++--- src/core/operations/DetectFileType.mjs | 17 +++++++- src/core/operations/ExtractFiles.mjs | 18 ++++++-- src/core/operations/ScanForEmbeddedFiles.mjs | 42 +++++++----------- 5 files changed, 111 insertions(+), 38 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index f19688d1..93247413 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -678,7 +678,7 @@ export const FILE_SIGNATURES = { 2: 0x4c, 3: 0x46 }, - extractor: null + extractor: extractELF }, { name: "Adobe Flash", @@ -1474,6 +1474,50 @@ export function extractZlib(bytes, offset) { } +/** + * ELF extractor. + * + * @param {Uint8Array} bytes + * @param {number} offset + * @returns {Uint8Array} + */ +export function extractELF(bytes, offset) { + const stream = new Stream(bytes.slice(offset)); + + // Skip over magic number + stream.moveForwardsBy(4); + + // Read architecture (x86 == 1, x64 == 2) + const x86 = stream.readInt(1) === 1; + + // Read endianness (1 == little, 2 == big) + const endian = stream.readInt(1) === 1 ? "le" : "be"; + + // Skip over header values + stream.moveForwardsBy(x86 ? 26 : 34); + + // Read section header table offset + const shoff = x86 ? stream.readInt(4, endian) : stream.readInt(8, endian); + + // Skip over flags, header size and program header size and entries + stream.moveForwardsBy(10); + + // Read section header table entry size + const shentsize = stream.readInt(2, endian); + + // Read number of entries in the section header table + const shnum = stream.readInt(2, endian); + + // Jump to section header table + stream.moveTo(shoff); + + // Move past each section header + stream.moveForwardsBy(shentsize * shnum); + + return stream.carve(); +} + + /** * Steps through a DEFLATE stream * diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index 202b54d9..e5d990d9 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -75,22 +75,29 @@ function bytesMatch(sig, buf, offset=0) { * extension and mime type. * * @param {Uint8Array} buf + * @param {string[]} [categories=All] - Which categories of file to look for * @returns {Object[]} types * @returns {string} type.name - Name of file type * @returns {string} type.ext - File extension * @returns {string} type.mime - Mime type * @returns {string} [type.desc] - Description */ -export function detectFileType(buf) { +export function detectFileType(buf, categories=Object.keys(FILE_SIGNATURES)) { if (!(buf && buf.length > 1)) { return []; } const matchingFiles = []; + const signatures = {}; - // TODO allow user to select which categories to check for (const cat in FILE_SIGNATURES) { - const category = FILE_SIGNATURES[cat]; + if (categories.includes(cat)) { + signatures[cat] = FILE_SIGNATURES[cat]; + } + } + + for (const cat in signatures) { + const category = signatures[cat]; category.forEach(filetype => { if (signatureMatches(filetype.signature, buf)) { @@ -107,6 +114,7 @@ export function detectFileType(buf) { * the extensions and mime types. * * @param {Uint8Array} buf + * @param {string[]} [categories=All] - Which categories of file to look for * @returns {Object[]} foundFiles * @returns {number} foundFiles.offset - The position in the buffer at which this file was found * @returns {Object} foundFiles.fileDetails @@ -115,16 +123,22 @@ export function detectFileType(buf) { * @returns {string} foundFiles.fileDetails.mime - Mime type * @returns {string} [foundFiles.fileDetails.desc] - Description */ -export function scanForFileTypes(buf) { +export function scanForFileTypes(buf, categories=Object.keys(FILE_SIGNATURES)) { if (!(buf && buf.length > 1)) { return []; } const foundFiles = []; + const signatures = {}; - // TODO allow user to select which categories to check for (const cat in FILE_SIGNATURES) { - const category = FILE_SIGNATURES[cat]; + if (categories.includes(cat)) { + signatures[cat] = FILE_SIGNATURES[cat]; + } + } + + for (const cat in signatures) { + const category = signatures[cat]; for (let i = 0; i < category.length; i++) { const filetype = category[i]; diff --git a/src/core/operations/DetectFileType.mjs b/src/core/operations/DetectFileType.mjs index 55db5edf..2321cee8 100644 --- a/src/core/operations/DetectFileType.mjs +++ b/src/core/operations/DetectFileType.mjs @@ -6,6 +6,7 @@ import Operation from "../Operation"; import {detectFileType} from "../lib/FileType"; +import {FILE_SIGNATURES} from "../lib/FileSignatures"; /** * Detect File Type operation @@ -24,7 +25,13 @@ class DetectFileType extends Operation { this.infoURL = "https://wikipedia.org/wiki/List_of_file_signatures"; this.inputType = "ArrayBuffer"; this.outputType = "string"; - this.args = []; + this.args = Object.keys(FILE_SIGNATURES).map(cat => { + return { + name: cat, + type: "boolean", + value: true + }; + }); } /** @@ -34,7 +41,13 @@ class DetectFileType extends Operation { */ run(input, args) { const data = new Uint8Array(input), - types = detectFileType(data); + categories = []; + + args.forEach((cat, i) => { + if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]); + }); + + const types = detectFileType(data, categories); if (!types.length) { return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?"; diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index 0f861276..f172d926 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -8,6 +8,7 @@ import Operation from "../Operation"; // import OperationError from "../errors/OperationError"; import Utils from "../Utils"; import {scanForFileTypes, extractFile} from "../lib/FileType"; +import {FILE_SIGNATURES} from "../lib/FileSignatures"; /** * Extract Files operation @@ -27,7 +28,13 @@ class ExtractFiles extends Operation { this.inputType = "ArrayBuffer"; this.outputType = "List"; this.presentType = "html"; - this.args = []; + this.args = Object.keys(FILE_SIGNATURES).map(cat => { + return { + name: cat, + type: "boolean", + value: cat === "Miscellaneous" ? false : true + }; + }); } /** @@ -36,10 +43,15 @@ class ExtractFiles extends Operation { * @returns {List} */ run(input, args) { - const bytes = new Uint8Array(input); + const bytes = new Uint8Array(input), + categories = []; + + args.forEach((cat, i) => { + if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]); + }); // Scan for embedded files - const detectedFiles = scanForFileTypes(bytes); + const detectedFiles = scanForFileTypes(bytes, categories); // Extract each file that we support const files = []; diff --git a/src/core/operations/ScanForEmbeddedFiles.mjs b/src/core/operations/ScanForEmbeddedFiles.mjs index a0465e83..ae88134f 100644 --- a/src/core/operations/ScanForEmbeddedFiles.mjs +++ b/src/core/operations/ScanForEmbeddedFiles.mjs @@ -7,6 +7,7 @@ import Operation from "../Operation"; import Utils from "../Utils"; import {scanForFileTypes} from "../lib/FileType"; +import {FILE_SIGNATURES} from "../lib/FileSignatures"; /** * Scan for Embedded Files operation @@ -25,13 +26,13 @@ class ScanForEmbeddedFiles extends Operation { this.infoURL = "https://wikipedia.org/wiki/List_of_file_signatures"; this.inputType = "ArrayBuffer"; this.outputType = "string"; - this.args = [ - { - "name": "Ignore common byte sequences", - "type": "boolean", - "value": true - } - ]; + this.args = Object.keys(FILE_SIGNATURES).map(cat => { + return { + name: cat, + type: "boolean", + value: cat === "Miscellaneous" ? false : true + }; + }); } /** @@ -41,21 +42,18 @@ class ScanForEmbeddedFiles extends Operation { */ run(input, args) { let output = "Scanning data for 'magic bytes' which may indicate embedded files. The following results may be false positives and should not be treat as reliable. Any suffiently long file is likely to contain these magic bytes coincidentally.\n", - numFound = 0, - numCommonFound = 0; - const ignoreCommon = args[0], - commonExts = ["ttf", "utf16le", ""], - data = new Uint8Array(input), - types = scanForFileTypes(data); + numFound = 0; + const categories = [], + data = new Uint8Array(input); + args.forEach((cat, i) => { + if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]); + }); + + const types = scanForFileTypes(data, categories); if (types.length) { types.forEach(type => { - if (ignoreCommon && commonExts.indexOf(type.fileDetails.extension) > -1) { - numCommonFound++; - return; - } - numFound++; output += "\nOffset " + type.offset + " (0x" + Utils.hex(type.offset) + "):\n" + " File extension: " + type.fileDetails.extension + "\n" + @@ -71,14 +69,6 @@ class ScanForEmbeddedFiles extends Operation { output += "\nNo embedded files were found."; } - if (numCommonFound > 0) { - output += "\n\n" + numCommonFound; - output += numCommonFound === 1 ? - " file type was detected that has a common byte sequence. This is likely to be a false positive." : - " file types were detected that have common byte sequences. These are likely to be false positives."; - output += " Run this operation with the 'Ignore common byte sequences' option unchecked to see details."; - } - return output; } From 9fa7edffbf1d559e54c31501b7f8be4e7b86448b Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sat, 2 Mar 2019 16:12:21 +0000 Subject: [PATCH 21/22] Improved file extraction error handling --- src/core/lib/FileSignatures.mjs | 6 +++--- src/core/operations/ExtractFiles.mjs | 22 +++++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 93247413..36e6818e 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1057,7 +1057,7 @@ export function extractJPEG(bytes, offset) { while (stream.hasMore()) { const marker = stream.getBytes(2); - if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker); + if (marker[0] !== 0xff) throw new Error(`Invalid marker while parsing JPEG at pos ${stream.position}: ${marker}`); let segmentSize = 0; switch (marker[1]) { @@ -1609,7 +1609,7 @@ function parseDEFLATE(stream) { parseHuffmanBlock(stream, dynamicLiteralTable, dynamicDistanceTable); } else { - throw new Error("Invalid block type"); + throw new Error(`Invalid block type while parsing DEFLATE stream at pos ${stream.position}`); } } @@ -1712,7 +1712,7 @@ function readHuffmanCode(stream, table) { const codeLength = codeWithLength >>> 16; if (codeLength > maxCodeLength) { - throw new Error("Invalid code length: " + codeLength); + throw new Error(`Invalid Huffman Code length while parsing DEFLATE block at pos ${stream.position}: ${codeLength}`); } stream.moveBackwardsByBits(maxCodeLength - codeLength); diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index f172d926..d2b87990 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -5,7 +5,7 @@ */ import Operation from "../Operation"; -// import OperationError from "../errors/OperationError"; +import OperationError from "../errors/OperationError"; import Utils from "../Utils"; import {scanForFileTypes, extractFile} from "../lib/FileType"; import {FILE_SIGNATURES} from "../lib/FileSignatures"; @@ -34,7 +34,13 @@ class ExtractFiles extends Operation { type: "boolean", value: cat === "Miscellaneous" ? false : true }; - }); + }).concat([ + { + name: "Ignore failed extractions", + type: "boolean", + value: "true" + } + ]); } /** @@ -44,7 +50,8 @@ class ExtractFiles extends Operation { */ run(input, args) { const bytes = new Uint8Array(input), - categories = []; + categories = [], + ignoreFailedExtractions = args.pop(1); args.forEach((cat, i) => { if (cat) categories.push(Object.keys(FILE_SIGNATURES)[i]); @@ -59,8 +66,13 @@ class ExtractFiles extends Operation { try { files.push(extractFile(bytes, detectedFile.fileDetails, detectedFile.offset)); } catch (err) { - if (err.message.indexOf("No extraction algorithm available") < 0) - throw err; + if (!ignoreFailedExtractions && err.message.indexOf("No extraction algorithm available") < 0) { + throw new OperationError( + `Error while attempting to extract ${detectedFile.fileDetails.name} ` + + `at offset ${detectedFile.offset}:\n` + + `${err.message}` + ); + } } }); From 84d31c1d597921ade565f30e60b887f2cc20ed4c Mon Sep 17 00:00:00 2001 From: n1474335 Date: Sat, 9 Mar 2019 06:25:27 +0000 Subject: [PATCH 22/22] Added 'Move to input' button to output file list. Improved zlib extraction efficiency. --- .eslintrc.json | 1 + src/core/Utils.mjs | 31 +++++++++++++-- src/core/lib/FileSignatures.mjs | 50 +++++++++++++----------- src/core/lib/FileType.mjs | 7 +++- src/core/operations/ExtractFiles.mjs | 8 +++- src/web/Manager.mjs | 1 + src/web/OutputWaiter.mjs | 18 +++++++++ src/web/html/index.html | 2 +- src/web/stylesheets/components/_pane.css | 4 ++ 9 files changed, 94 insertions(+), 28 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index d5e4e768..7dcb705c 100755 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -102,6 +102,7 @@ "$": false, "jQuery": false, "log": false, + "app": false, "COMPILE_TIME": false, "COMPILE_MSG": false, diff --git a/src/core/Utils.mjs b/src/core/Utils.mjs index f70e2941..8e69b020 100755 --- a/src/core/Utils.mjs +++ b/src/core/Utils.mjs @@ -832,8 +832,9 @@ class Utils { const buff = await Utils.readFile(file); const blob = new Blob( [buff], - {type: "octet/stream"} + {type: file.type || "octet/stream"} ); + const blobURL = URL.createObjectURL(blob); const html = `
@@ -1163,6 +1173,21 @@ String.prototype.count = function(chr) { }; +/** + * Wrapper for self.sendStatusMessage to handle different environments. + * + * @param {string} msg + */ +export function sendStatusMessage(msg) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage(msg); + else if (ENVIRONMENT_IS_WEB()) + app.alert(msg, 10000); + else if (ENVIRONMENT_IS_NODE()) + log.debug(msg); +} + + /* * Polyfills */ diff --git a/src/core/lib/FileSignatures.mjs b/src/core/lib/FileSignatures.mjs index 36e6818e..61e37b88 100644 --- a/src/core/lib/FileSignatures.mjs +++ b/src/core/lib/FileSignatures.mjs @@ -1518,26 +1518,26 @@ export function extractELF(bytes, offset) { } +// Construct required Huffman Tables +const fixedLiteralTableLengths = new Array(288); +for (let i = 0; i < fixedLiteralTableLengths.length; i++) { + fixedLiteralTableLengths[i] = + (i <= 143) ? 8 : + (i <= 255) ? 9 : + (i <= 279) ? 7 : + 8; +} +const fixedLiteralTable = buildHuffmanTable(fixedLiteralTableLengths); +const fixedDistanceTableLengths = new Array(30).fill(5); +const fixedDistanceTable = buildHuffmanTable(fixedDistanceTableLengths); +const huffmanOrder = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]; + /** * Steps through a DEFLATE stream * * @param {Stream} stream */ function parseDEFLATE(stream) { - // Construct required Huffman Tables - const fixedLiteralTableLengths = new Uint8Array(288); - for (let i = 0; i < fixedLiteralTableLengths.length; i++) { - fixedLiteralTableLengths[i] = - (i <= 143) ? 8 : - (i <= 255) ? 9 : - (i <= 279) ? 7 : - 8; - } - const fixedLiteralTable = buildHuffmanTable(fixedLiteralTableLengths); - const fixedDistanceTableLengths = new Uint8Array(30).fill(5); - const fixedDistanceTable = buildHuffmanTable(fixedDistanceTableLengths); - const huffmanOrder = new Uint8Array([16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15]); - // Parse DEFLATE data let finalBlock = 0; @@ -1619,6 +1619,14 @@ function parseDEFLATE(stream) { } +// Static length tables +const lengthExtraTable = [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0 +]; +const distanceExtraTable = [ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 +]; + /** * Parses a Huffman Block given the literal and distance tables * @@ -1627,20 +1635,18 @@ function parseDEFLATE(stream) { * @param {Uint32Array} distTab */ function parseHuffmanBlock(stream, litTab, distTab) { - const lengthExtraTable = new Uint8Array([ - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0 - ]); - const distanceExtraTable = new Uint8Array([ - 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 - ]); - let code; + let loops = 0; while ((code = readHuffmanCode(stream, litTab))) { // console.log("Code: " + code + " (" + Utils.chr(code) + ") " + Utils.bin(code)); // End of block if (code === 256) break; + // Detect probably infinite loops + if (++loops > 10000) + throw new Error("Caught in probable infinite loop while parsing Huffman Block"); + // Literal if (code < 256) continue; @@ -1657,7 +1663,7 @@ function parseHuffmanBlock(stream, litTab, distTab) { /** * Builds a Huffman table given the relevant code lengths * - * @param {Uint8Array} lengths + * @param {Array} lengths * @returns {Array} result * @returns {Uint32Array} result.table * @returns {number} result.maxCodeLength diff --git a/src/core/lib/FileType.mjs b/src/core/lib/FileType.mjs index e5d990d9..e961a76f 100644 --- a/src/core/lib/FileType.mjs +++ b/src/core/lib/FileType.mjs @@ -7,6 +7,7 @@ * */ import {FILE_SIGNATURES} from "./FileSignatures"; +import {sendStatusMessage} from "../Utils"; /** @@ -148,6 +149,7 @@ export function scanForFileTypes(buf, categories=Object.keys(FILE_SIGNATURES)) { let pos = 0; while ((pos = locatePotentialSig(buf, sig, pos)) >= 0) { if (bytesMatch(sig, buf, pos)) { + sendStatusMessage(`Found potential signature for ${filetype.name} at pos ${pos}`); foundFiles.push({ offset: pos, fileDetails: filetype @@ -249,9 +251,12 @@ export function isImage(buf) { */ export function extractFile(bytes, fileDetail, offset) { if (fileDetail.extractor) { + sendStatusMessage(`Attempting to extract ${fileDetail.name} at pos ${offset}...`); const fileData = fileDetail.extractor(bytes, offset); const ext = fileDetail.extension.split(",")[0]; - return new File([fileData], `extracted_at_0x${offset.toString(16)}.${ext}`); + return new File([fileData], `extracted_at_0x${offset.toString(16)}.${ext}`, { + type: fileDetail.mime + }); } throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`); diff --git a/src/core/operations/ExtractFiles.mjs b/src/core/operations/ExtractFiles.mjs index d2b87990..b9b260bb 100644 --- a/src/core/operations/ExtractFiles.mjs +++ b/src/core/operations/ExtractFiles.mjs @@ -62,12 +62,13 @@ class ExtractFiles extends Operation { // Extract each file that we support const files = []; + const errors = []; detectedFiles.forEach(detectedFile => { try { files.push(extractFile(bytes, detectedFile.fileDetails, detectedFile.offset)); } catch (err) { if (!ignoreFailedExtractions && err.message.indexOf("No extraction algorithm available") < 0) { - throw new OperationError( + errors.push( `Error while attempting to extract ${detectedFile.fileDetails.name} ` + `at offset ${detectedFile.offset}:\n` + `${err.message}` @@ -76,9 +77,14 @@ class ExtractFiles extends Operation { } }); + if (errors.length) { + throw new OperationError(errors.join("\n\n")); + } + return files; } + /** * Displays the files in HTML for web apps. * diff --git a/src/web/Manager.mjs b/src/web/Manager.mjs index 30cb4943..5fa0e8c1 100755 --- a/src/web/Manager.mjs +++ b/src/web/Manager.mjs @@ -173,6 +173,7 @@ class Manager { this.addDynamicListener("#output-file-download", "click", this.output.downloadFile, this.output); this.addDynamicListener("#output-file-slice i", "click", this.output.displayFileSlice, this.output); document.getElementById("show-file-overlay").addEventListener("click", this.output.showFileOverlayClick.bind(this.output)); + this.addDynamicListener(".extract-file,.extract-file i", "click", this.output.extractFileClick, this.output); // Options document.getElementById("options").addEventListener("click", this.options.optionsClick.bind(this.options)); diff --git a/src/web/OutputWaiter.mjs b/src/web/OutputWaiter.mjs index 2d93507c..0a10b8b2 100755 --- a/src/web/OutputWaiter.mjs +++ b/src/web/OutputWaiter.mjs @@ -494,6 +494,24 @@ class OutputWaiter { magicButton.setAttribute("data-original-title", "Magic!"); } + + /** + * Handler for extract file events. + * + * @param {Event} e + */ + async extractFileClick(e) { + e.preventDefault(); + e.stopPropagation(); + + const el = e.target.nodeName === "I" ? e.target.parentNode : e.target; + const blobURL = el.getAttribute("blob-url"); + const fileName = el.getAttribute("file-name"); + + const blob = await fetch(blobURL).then(r => r.blob()); + this.manager.input.loadFile(new File([blob], fileName, {type: blob.type})); + } + } export default OutputWaiter; diff --git a/src/web/html/index.html b/src/web/html/index.html index 74eb0ed8..302355d9 100755 --- a/src/web/html/index.html +++ b/src/web/html/index.html @@ -271,7 +271,7 @@ content_copy