Added decoder for chunked HTTP encoding

This decoder will join up a HTTP response sent using chunked transfer encoding, raised in issue #168.

This is useful when attempting to extract files or gzipped responses sent using chunked transfer encoding, particularly when combined with the gunzip operation.

src/core/config/Categories.json

@@ -137,6 +137,7 @@
         "ops": [
             "HTTP request",
             "Strip HTTP headers",
+            "Dechunk HTTP response",
             "Parse User Agent",
             "Parse IP range",
             "Parse IPv6 address",

src/core/config/scripts/portOperation.mjs

@@ -2239,6 +2239,13 @@ const OP_CONFIG = {
         outputType: "string",
         args: []
     },
+    "Dechunk HTTP response": {
+        module: "HTTP",
+        description: "Parses a HTTP response transferred using transfer-encoding:chunked",
+        inputType: "string",
+        outputType: "string",
+        args: []
+    },
     "Parse User Agent": {
         module: "HTTP",
         description: "Attempts to identify and categorise information contained in a user-agent string.",

src/core/operations/DechunkHTTP.mjs

@@ -0,0 +1,50 @@
+/**
+ * @author sevzero [sevzero@protonmail.com]
+ * @copyright Crown Copyright 2016
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+
+/**
+ * Dechunk HTTP response operation
+ */
+class DechunkHTTP extends Operation {
+
+    /**
+     * DechunkHTTP constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Dechunk HTTP response";
+        this.module = "Default";
+        this.description = "Parses a HTTP response transferred using transfer-encoding:chunked";
+        this.inputType = "string";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        var chunks = [];
+        var chunkSizeEnd = input.indexOf("\n") + 1;
+        var lineEndings = input.charAt(chunkSizeEnd - 2) == "\r" ? "\r\n" : "\n";
+        var lineEndingsLength = lineEndings.length;
+        var chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16);
+        while (!isNaN(chunkSize)) {
+            chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); 
+            input = input.slice(chunkSizeEnd + chunkSize + lineEndingsLength);
+            chunkSizeEnd = input.indexOf(lineEndings) + lineEndingsLength;
+            chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16);
+        }
+        return chunks.join('') + input;
+    }
+
+}
+
+export default DechunkHTTP;

src/core/operations/legacy/HTTP.js

@@ -37,6 +37,27 @@ const HTTP = {
         return (headerEnd < 2) ? input : input.slice(headerEnd, input.length);
     },
 
+    /**
+     * Dechunk response operation
+     *
+     * @param {string} input
+     * @param {Object[]} args}
+     * @returns {string}
+    */
+    runDechunk: function(input, args) {
+        var chunks = [];
+        var chunkSizeEnd = input.indexOf("\n") + 1;
+        var lineEndings = input.charAt(chunkSizeEnd - 2) == "\r" ? "\r\n" : "\n";
+        var lineEndingsLength = lineEndings.length;
+        var chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16);
+        while (!isNaN(chunkSize)) {
+            chunks.push(input.slice(chunkSizeEnd, chunkSize + chunkSizeEnd)); 
+            input = input.slice(chunkSizeEnd + chunkSize + lineEndingsLength);
+            chunkSizeEnd = input.indexOf(lineEndings) + lineEndingsLength;
+            chunkSize = parseInt(input.slice(0, chunkSizeEnd), 16);
+        }
+        return chunks.join('') + input;
+    },
 
     /**
      * Parse User Agent operation.