From cefe3fc542d93e958fec3d13ff5122aba6db9a0d Mon Sep 17 00:00:00 2001 From: n1474335 Date: Mon, 21 May 2018 18:23:05 +0000 Subject: [PATCH] ESM: Ported Bzip2, Diff and Tar operations --- src/core/operations/Bzip2Decompress.mjs | 55 ++++++++++ src/core/operations/Diff.mjs | 124 +++++++++++++++++++++ src/core/operations/Tar.mjs | 139 ++++++++++++++++++++++++ src/core/operations/Untar.mjs | 138 +++++++++++++++++++++++ src/core/vendor/bzip2.js | 2 + test/index.mjs | 3 +- 6 files changed, 459 insertions(+), 2 deletions(-) create mode 100644 src/core/operations/Bzip2Decompress.mjs create mode 100644 src/core/operations/Diff.mjs create mode 100644 src/core/operations/Tar.mjs create mode 100644 src/core/operations/Untar.mjs diff --git a/src/core/operations/Bzip2Decompress.mjs b/src/core/operations/Bzip2Decompress.mjs new file mode 100644 index 00000000..e31b3d2c --- /dev/null +++ b/src/core/operations/Bzip2Decompress.mjs @@ -0,0 +1,55 @@ +/** + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2016 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import bzip2 from "../vendor/bzip2.js"; +import OperationError from "../errors/OperationError"; + +/** + * Bzip2 Decompress operation + */ +class Bzip2Decompress extends Operation { + + /** + * Bzip2Decompress constructor + */ + constructor() { + super(); + + this.name = "Bzip2 Decompress"; + this.module = "Compression"; + this.description = "Decompresses data using the Bzip2 algorithm."; + this.inputType = "byteArray"; + this.outputType = "string"; + this.args = []; + this.patterns = [ + { + "match": "^\\x42\\x5a\\x68", + "flags": "", + "args": [] + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const compressed = new Uint8Array(input); + + try { + const bzip2Reader = bzip2.array(compressed); + return bzip2.simple(bzip2Reader); + } catch (err) { + throw new OperationError(err); + } + } + +} + +export default Bzip2Decompress; diff --git a/src/core/operations/Diff.mjs b/src/core/operations/Diff.mjs new file mode 100644 index 00000000..6627cf6e --- /dev/null +++ b/src/core/operations/Diff.mjs @@ -0,0 +1,124 @@ +/** + * @author n1474335 [n1474335@gmail.com] + * @copyright Crown Copyright 2016 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import Utils from "../Utils"; +import * as JsDiff from "diff"; +import OperationError from "../errors/OperationError"; + +/** + * Diff operation + */ +class Diff extends Operation { + + /** + * Diff constructor + */ + constructor() { + super(); + + this.name = "Diff"; + this.module = "Diff"; + this.description = "Compares two inputs (separated by the specified delimiter) and highlights the differences between them."; + this.inputType = "string"; + this.outputType = "html"; + this.args = [ + { + "name": "Sample delimiter", + "type": "binaryString", + "value": "\\n\\n" + }, + { + "name": "Diff by", + "type": "option", + "value": ["Character", "Word", "Line", "Sentence", "CSS", "JSON"] + }, + { + "name": "Show added", + "type": "boolean", + "value": true + }, + { + "name": "Show removed", + "type": "boolean", + "value": true + }, + { + "name": "Ignore whitespace (relevant for word and line)", + "type": "boolean", + "value": false + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {html} + */ + run(input, args) { + const [ + sampleDelim, + diffBy, + showAdded, + showRemoved, + ignoreWhitespace + ] = args, + samples = input.split(sampleDelim); + let output = "", + diff; + + if (!samples || samples.length !== 2) { + throw new OperationError("Incorrect number of samples, perhaps you need to modify the sample delimiter or add more samples?"); + } + + switch (diffBy) { + case "Character": + diff = JsDiff.diffChars(samples[0], samples[1]); + break; + case "Word": + if (ignoreWhitespace) { + diff = JsDiff.diffWords(samples[0], samples[1]); + } else { + diff = JsDiff.diffWordsWithSpace(samples[0], samples[1]); + } + break; + case "Line": + if (ignoreWhitespace) { + diff = JsDiff.diffTrimmedLines(samples[0], samples[1]); + } else { + diff = JsDiff.diffLines(samples[0], samples[1]); + } + break; + case "Sentence": + diff = JsDiff.diffSentences(samples[0], samples[1]); + break; + case "CSS": + diff = JsDiff.diffCss(samples[0], samples[1]); + break; + case "JSON": + diff = JsDiff.diffJson(samples[0], samples[1]); + break; + default: + throw new OperationError("Invalid 'Diff by' option."); + } + + for (let i = 0; i < diff.length; i++) { + if (diff[i].added) { + if (showAdded) output += "" + Utils.escapeHtml(diff[i].value) + ""; + } else if (diff[i].removed) { + if (showRemoved) output += "" + Utils.escapeHtml(diff[i].value) + ""; + } else { + output += Utils.escapeHtml(diff[i].value); + } + } + + return output; + } + +} + +export default Diff; diff --git a/src/core/operations/Tar.mjs b/src/core/operations/Tar.mjs new file mode 100644 index 00000000..a2e8eb70 --- /dev/null +++ b/src/core/operations/Tar.mjs @@ -0,0 +1,139 @@ +/** + * @author tlwr [toby@toby.codes] + * @copyright Crown Copyright 2016 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import Utils from "../Utils"; + +/** + * Tar operation + */ +class Tar extends Operation { + + /** + * Tar constructor + */ + constructor() { + super(); + + this.name = "Tar"; + this.module = "Compression"; + this.description = "Packs the input into a tarball.

No support for multiple files at this time."; + this.inputType = "byteArray"; + this.outputType = "File"; + this.args = [ + { + "name": "Filename", + "type": "string", + "value": "file.txt" + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {byteArray} + */ + run(input, args) { + const Tarball = function() { + this.bytes = new Array(512); + this.position = 0; + }; + + Tarball.prototype.addEmptyBlock = function() { + const filler = new Array(512); + filler.fill(0); + this.bytes = this.bytes.concat(filler); + }; + + Tarball.prototype.writeBytes = function(bytes) { + const self = this; + + if (this.position + bytes.length > this.bytes.length) { + this.addEmptyBlock(); + } + + Array.prototype.forEach.call(bytes, function(b, i) { + if (typeof b.charCodeAt !== "undefined") { + b = b.charCodeAt(); + } + + self.bytes[self.position] = b; + self.position += 1; + }); + }; + + Tarball.prototype.writeEndBlocks = function() { + const numEmptyBlocks = 2; + for (let i = 0; i < numEmptyBlocks; i++) { + this.addEmptyBlock(); + } + }; + + const fileSize = input.length.toString(8).padStart(11, "0"); + const currentUnixTimestamp = Math.floor(Date.now() / 1000); + const lastModTime = currentUnixTimestamp.toString(8).padStart(11, "0"); + + const file = { + fileName: Utils.padBytesRight(args[0], 100), + fileMode: Utils.padBytesRight("0000664", 8), + ownerUID: Utils.padBytesRight("0", 8), + ownerGID: Utils.padBytesRight("0", 8), + size: Utils.padBytesRight(fileSize, 12), + lastModTime: Utils.padBytesRight(lastModTime, 12), + checksum: " ", + type: "0", + linkedFileName: Utils.padBytesRight("", 100), + USTARFormat: Utils.padBytesRight("ustar", 6), + version: "00", + ownerUserName: Utils.padBytesRight("", 32), + ownerGroupName: Utils.padBytesRight("", 32), + deviceMajor: Utils.padBytesRight("", 8), + deviceMinor: Utils.padBytesRight("", 8), + fileNamePrefix: Utils.padBytesRight("", 155), + }; + + let checksum = 0; + for (const key in file) { + const bytes = file[key]; + Array.prototype.forEach.call(bytes, function(b) { + if (typeof b.charCodeAt !== "undefined") { + checksum += b.charCodeAt(); + } else { + checksum += b; + } + }); + } + checksum = Utils.padBytesRight(checksum.toString(8).padStart(7, "0"), 8); + file.checksum = checksum; + + const tarball = new Tarball(); + tarball.writeBytes(file.fileName); + tarball.writeBytes(file.fileMode); + tarball.writeBytes(file.ownerUID); + tarball.writeBytes(file.ownerGID); + tarball.writeBytes(file.size); + tarball.writeBytes(file.lastModTime); + tarball.writeBytes(file.checksum); + tarball.writeBytes(file.type); + tarball.writeBytes(file.linkedFileName); + tarball.writeBytes(file.USTARFormat); + tarball.writeBytes(file.version); + tarball.writeBytes(file.ownerUserName); + tarball.writeBytes(file.ownerGroupName); + tarball.writeBytes(file.deviceMajor); + tarball.writeBytes(file.deviceMinor); + tarball.writeBytes(file.fileNamePrefix); + tarball.writeBytes(Utils.padBytesRight("", 12)); + tarball.writeBytes(input); + tarball.writeEndBlocks(); + + return new File([new Uint8Array(tarball.bytes)], args[0]); + } + +} + +export default Tar; diff --git a/src/core/operations/Untar.mjs b/src/core/operations/Untar.mjs new file mode 100644 index 00000000..6bca05d0 --- /dev/null +++ b/src/core/operations/Untar.mjs @@ -0,0 +1,138 @@ +/** + * @author tlwr [toby@toby.codes] + * @copyright Crown Copyright 2016 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import Utils from "../Utils"; + +/** + * Untar operation + */ +class Untar extends Operation { + + /** + * Untar constructor + */ + constructor() { + super(); + + this.name = "Untar"; + this.module = "Compression"; + this.description = "Unpacks a tarball and displays it per file."; + this.inputType = "byteArray"; + this.outputType = "List"; + this.presentType = "html"; + this.args = []; + this.patterns = [ + { + "match": "^.{257}\\x75\\x73\\x74\\x61\\x72", + "flags": "", + "args": [] + } + ]; + } + + /** + * @param {byteArray} input + * @param {Object[]} args + * @returns {List} + */ + run(input, args) { + const Stream = function(input) { + this.bytes = input; + this.position = 0; + }; + + Stream.prototype.getBytes = function(bytesToGet) { + const newPosition = this.position + bytesToGet; + const bytes = this.bytes.slice(this.position, newPosition); + this.position = newPosition; + return bytes; + }; + + Stream.prototype.readString = function(numBytes) { + let result = ""; + for (let i = this.position; i < this.position + numBytes; i++) { + const currentByte = this.bytes[i]; + if (currentByte === 0) break; + result += String.fromCharCode(currentByte); + } + this.position += numBytes; + return result; + }; + + Stream.prototype.readInt = function(numBytes, base) { + const string = this.readString(numBytes); + return parseInt(string, base); + }; + + Stream.prototype.hasMore = function() { + return this.position < this.bytes.length; + }; + + const stream = new Stream(input), + files = []; + + while (stream.hasMore()) { + const dataPosition = stream.position + 512; + + const file = { + fileName: stream.readString(100), + fileMode: stream.readString(8), + ownerUID: stream.readString(8), + ownerGID: stream.readString(8), + size: parseInt(stream.readString(12), 8), // Octal + lastModTime: new Date(1000 * stream.readInt(12, 8)), // Octal + checksum: stream.readString(8), + type: stream.readString(1), + linkedFileName: stream.readString(100), + USTARFormat: stream.readString(6).indexOf("ustar") >= 0, + }; + + if (file.USTARFormat) { + file.version = stream.readString(2); + file.ownerUserName = stream.readString(32); + file.ownerGroupName = stream.readString(32); + file.deviceMajor = stream.readString(8); + file.deviceMinor = stream.readString(8); + file.filenamePrefix = stream.readString(155); + } + + stream.position = dataPosition; + + if (file.type === "0") { + // File + let endPosition = stream.position + file.size; + if (file.size % 512 !== 0) { + endPosition += 512 - (file.size % 512); + } + + file.bytes = stream.getBytes(file.size); + files.push(new File([new Uint8Array(file.bytes)], file.fileName)); + stream.position = endPosition; + } else if (file.type === "5") { + // Directory + files.push(new File([new Uint8Array(file.bytes)], file.fileName)); + } else { + // Symlink or empty bytes + } + } + + return files; + } + + /** + * Displays the files in HTML for web apps. + * + * @param {File[]} files + * @returns {html} + */ + async present(files) { + return await Utils.displayFilesAsHTML(files); + } + +} + +export default Untar; diff --git a/src/core/vendor/bzip2.js b/src/core/vendor/bzip2.js index 03c8c97c..12dc3852 100755 --- a/src/core/vendor/bzip2.js +++ b/src/core/vendor/bzip2.js @@ -261,3 +261,5 @@ bzip2.decompress = function(bits, size, len){ } return output; } + +module.exports = bzip2; diff --git a/test/index.mjs b/test/index.mjs index bf5e8115..a20d5fa6 100644 --- a/test/index.mjs +++ b/test/index.mjs @@ -35,7 +35,7 @@ import "./tests/operations/CharEnc"; import "./tests/operations/Ciphers"; import "./tests/operations/Checksum"; // import "./tests/operations/Code"; -// import "./tests/operations/Compress"; +import "./tests/operations/Compress"; // import "./tests/operations/Crypt"; import "./tests/operations/DateTime"; import "./tests/operations/Fork"; @@ -43,7 +43,6 @@ import "./tests/operations/Jump"; import "./tests/operations/ConditionalJump"; import "./tests/operations/Register"; import "./tests/operations/Comment"; - import "./tests/operations/Hash"; import "./tests/operations/Hexdump"; // import "./tests/operations/Image";