From 37428fbe3bf147324ca1e122f7734a788cb4ce41 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 27 Mar 2019 09:05:10 +0000 Subject: [PATCH] Bake all inputs, not just the active tab. Limits number of workers to number of cpu threads (4 if not supported) Creates output tabs (switching doesn't work yet) Disabled some highlighting for now. --- src/core/ChefWorker.js | 23 +++-- src/web/App.mjs | 12 ++- src/web/HighlighterWaiter.mjs | 6 +- src/web/InputWaiter.mjs | 33 ++++++- src/web/Manager.mjs | 2 +- src/web/OutputWaiter.mjs | 76 +++++++++++++++ src/web/WorkerWaiter.mjs | 176 ++++++++++++++++++++++++++-------- 7 files changed, 272 insertions(+), 56 deletions(-) mode change 100755 => 100644 src/web/WorkerWaiter.mjs diff --git a/src/core/ChefWorker.js b/src/core/ChefWorker.js index a243f32c..77394564 100644 --- a/src/core/ChefWorker.js +++ b/src/core/ChefWorker.js @@ -27,6 +27,8 @@ self.chef = new Chef(); self.OpModules = OpModules; self.OperationConfig = OperationConfig; +self.inputNum = "0"; + // Tell the app that the worker has loaded and is ready to operate self.postMessage({ action: "workerLoaded", @@ -105,14 +107,16 @@ async function bake(data) { self.postMessage({ action: "bakeComplete", data: Object.assign(response, { - id: data.id + id: data.id, + inputNum: data.inputNum || 0 }) }); } catch (err) { self.postMessage({ action: "bakeError", data: Object.assign(err, { - id: data.id + id: data.id, + inputNum: data.inputNum || 0 }) }); } @@ -142,7 +146,8 @@ async function getDishAs(data) { action: "dishReturned", data: { value: value, - id: data.id + id: data.id, + inputNum: self.inputNum } }); } @@ -162,7 +167,8 @@ async function calculateHighlights(recipeConfig, direction, pos) { self.postMessage({ action: "highlightsCalculated", - data: pos + data: pos, + inputNum: self.inputNum }); } @@ -194,7 +200,8 @@ self.loadRequiredModules = function(recipeConfig) { self.sendStatusMessage = function(msg) { self.postMessage({ action: "statusMessage", - data: msg + data: msg, + inputNum: self.inputNum }); }; @@ -210,7 +217,8 @@ self.setOption = function(option, value) { action: "optionUpdate", data: { option: option, - value: value + value: value, + inputNum: self.inputNum } }); }; @@ -229,7 +237,8 @@ self.setRegisters = function(opIndex, numPrevRegisters, registers) { data: { opIndex: opIndex, numPrevRegisters: numPrevRegisters, - registers: registers + registers: registers, + inputNum: self.inputNum } }); }; diff --git a/src/web/App.mjs b/src/web/App.mjs index ac97de4c..4a939403 100755 --- a/src/web/App.mjs +++ b/src/web/App.mjs @@ -128,7 +128,7 @@ class App { this.manager.recipe.updateBreakpointIndicator(false); this.manager.worker.bake( - this.getInput(), // The user's input + this.getAllInput(), // The user's input this.getRecipeConfig(), // The configuration of the recipe this.options, // Options set by the user this.progress, // The current position in the recipe @@ -184,6 +184,14 @@ class App { return this.manager.input.get(); } + /** + * Gets the user's input data for all tabs. + * + * @returns {Array} + */ + getAllInput() { + return this.manager.input.getAll(); + } /** * Sets the user's input data. @@ -244,7 +252,7 @@ class App { /** * Sets up the adjustable splitter to allow the user to resize areas of the page. * - * @param {boolean} [minimise=false] - Set this flag if attempting to minimuse frames to 0 width + * @param {boolean} [minimise=false] - Set this flag if attempting to minimise frames to 0 width */ initialiseSplitter(minimise=false) { if (this.columnSplitter) this.columnSplitter.destroy(); diff --git a/src/web/HighlighterWaiter.mjs b/src/web/HighlighterWaiter.mjs index 99ae10b1..7bce1df1 100755 --- a/src/web/HighlighterWaiter.mjs +++ b/src/web/HighlighterWaiter.mjs @@ -126,8 +126,8 @@ class HighlighterWaiter { */ inputScroll(e) { const el = e.target; - document.getElementById("input-highlighter").scrollTop = el.scrollTop; - document.getElementById("input-highlighter").scrollLeft = el.scrollLeft; + // document.getElementById("input-highlighter").scrollTop = el.scrollTop; + // document.getElementById("input-highlighter").scrollLeft = el.scrollLeft; } @@ -326,7 +326,7 @@ class HighlighterWaiter { * Removes highlighting and selection information. */ removeHighlights() { - document.getElementById("input-highlighter").innerHTML = ""; + // document.getElementById("input-highlighter").innerHTML = ""; document.getElementById("output-highlighter").innerHTML = ""; document.getElementById("input-selection-info").innerHTML = ""; document.getElementById("output-selection-info").innerHTML = ""; diff --git a/src/web/InputWaiter.mjs b/src/web/InputWaiter.mjs index 1ca3531e..6d5c0bcd 100755 --- a/src/web/InputWaiter.mjs +++ b/src/web/InputWaiter.mjs @@ -62,6 +62,34 @@ class InputWaiter { return value; } + /** + * Gets the inputs from all tabs + * + * @returns {Array} + */ + getAll() { + const inputs = []; + const tabsContainer = document.getElementById("input-tabs"); + const tabs = tabsContainer.firstElementChild.children; + + for (let i = 0; i < tabs.length; i++) { + const inputNum = tabs.item(i).id.replace("input-tab-", ""); + if (this.fileBuffers[inputNum] && this.fileBuffers[inputNum].fileBuffer) { + inputs.push({ + inputNum: inputNum, + input: this.fileBuffers[inputNum].fileBuffer + }); + } else { + inputs.push({ + inputNum: inputNum, + input: this.inputs[inputNum] + }); + } + } + + return inputs; + } + /** * Sets the input in the input area. @@ -408,7 +436,6 @@ class InputWaiter { * @fires Manager#statechange */ clearAllIoClick() { - // TODO: Close all tabs here const tabs = document.getElementById("input-tabs").getElementsByTagName("li"); for (let i = tabs.length - 1; i >= 0; i--) { const tabItem = tabs.item(i); @@ -558,8 +585,6 @@ class InputWaiter { document.getElementById("input-file").style.display = "none"; } - window.dispatchEvent(this.manager.statechange); - } /** @@ -585,7 +610,7 @@ class InputWaiter { const activeTab = document.getElementById(`input-tab-${tabNum}`); const activeTabContent = activeTab.firstElementChild; if (input instanceof File) { - activeTabContent.innerText = input.name; + activeTabContent.innerText = `${tabNum}: ${input.name}`; } else { if (input.length > 0) { const inputText = input.slice(0, 100).split(/[\r\n]/)[0]; diff --git a/src/web/Manager.mjs b/src/web/Manager.mjs index eb7e7eea..264bb0a8 100755 --- a/src/web/Manager.mjs +++ b/src/web/Manager.mjs @@ -82,7 +82,7 @@ class Manager { * Sets up the various components and listeners. */ setup() { - this.worker.registerChefWorker(); + this.worker.setupChefWorkers(); this.recipe.initialiseOperationDragNDrop(); this.controls.initComponents(); this.controls.autoBakeChange(); diff --git a/src/web/OutputWaiter.mjs b/src/web/OutputWaiter.mjs index eaafeb8b..1f49fa24 100755 --- a/src/web/OutputWaiter.mjs +++ b/src/web/OutputWaiter.mjs @@ -25,6 +25,7 @@ class OutputWaiter { this.dishBuffer = null; this.dishStr = null; + this.outputs = []; } @@ -38,6 +39,38 @@ class OutputWaiter { } + /** + * Sets the output array for multiple outputs. + * Displays the active output in the output textarea + * + * @param {Array} outputs + */ + async multiSet(outputs) { + log.debug("Received " + outputs.length + " outputs."); + this.outputs = outputs; + const activeTab = this.manager.input.getActiveTab(); + + const tabs = document.getElementById("output-tabs").getElementsByTagName("li"); + for (let i = tabs.length - 1; i >= 0; i--) { + document.getElementById("output-tabs").firstElementChild.removeChild(tabs.item(i)); + } + + for (let i = 0; i < outputs.length; i++) { + this.addTab(outputs[i].inputNum); + if (outputs[i].inputNum === activeTab) { + await this.set(outputs[i].data.result, outputs[i].data.type, outputs[0].data.duration); + } + } + // await this.set(this.outputs[0].data.result, this.outputs[0].data.type, this.outputs[0].data.duration); + + // Create tabs + + // Select active tab + + // Display active tab data in textarea + } + + /** * Sets the output in the output textarea. * @@ -542,6 +575,49 @@ class OutputWaiter { this.manager.input.loadFile(new File([blob], fileName, {type: blob.type})); } + /** + * Function to create a new tab + * + * @param inputNum + */ + addTab(inputNum) { + const tabWrapper = document.getElementById("output-tabs"); + const tabsList = tabWrapper.firstElementChild; + + if (tabsList.children.length > 0) { + tabWrapper.style.display = "block"; + } + + document.getElementById("output-wrapper").style.height = "calc(100% - var(--tab-height) - var(--title-height))"; + document.getElementById("output-highlighter").style.height = "calc(100% - var(--tab-height) - var(--title-height))"; + document.getElementById("output-file").style.height = "calc(100% - var(--tab-height) - var(--title-height))"; + + const newTab = document.createElement("li"); + newTab.id = `output-tab-${inputNum}`; + if (inputNum === this.manager.input.getActiveTab()) { + newTab.classList.add("active-output-tab"); + } + + const newTabContent = document.createElement("div"); + newTabContent.classList.add("output-tab-content"); + newTabContent.innerText = `Tab ${inputNum}`; + + newTab.appendChild(newTabContent); + tabsList.appendChild(newTab); + } + + /** + * Function to change tabs + * + * @param {Element} tabElement + */ + changeTab(tabElement) { + const liItem = tabElement.parentElement; + const newTabNum = liItem.id.replace("input-tab-", ""); + const currentTabNum = this.getActiveTab(); + const outputText = document.getElementById("output-text"); + } + } export default OutputWaiter; diff --git a/src/web/WorkerWaiter.mjs b/src/web/WorkerWaiter.mjs old mode 100755 new mode 100644 index 7ef72263..143539c4 --- a/src/web/WorkerWaiter.mjs +++ b/src/web/WorkerWaiter.mjs @@ -1,6 +1,7 @@ /** * @author n1474335 [n1474335@gmail.com] - * @copyright Crown Copyright 2017 + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 * @license Apache-2.0 */ @@ -14,7 +15,7 @@ class WorkerWaiter { /** * WorkerWaiter constructor. * - * @param {App} app - The main view object for CyberChef. + * @param {App} app - The main view object for CyberChef * @param {Manager} manager - The CyberChef event manager. */ constructor(app, manager) { @@ -23,24 +24,34 @@ class WorkerWaiter { this.callbacks = {}; this.callbackID = 0; + this.pendingInputs = []; + this.runningWorkers = 0; + this.chefWorkers = []; + this.outputs = []; } - /** - * Sets up the ChefWorker and associated listeners. + * Sets up a pool of ChefWorkers to be used for baking */ - registerChefWorker() { - log.debug("Registering new ChefWorker"); - this.chefWorker = new ChefWorker(); - this.chefWorker.addEventListener("message", this.handleChefMessage.bind(this)); - this.setLogLevel(); + setupChefWorkers() { + const threads = navigator.hardwareConcurrency || 4; // Default to 4 - let docURL = document.location.href.split(/[#?]/)[0]; - const index = docURL.lastIndexOf("/"); - if (index > 0) { - docURL = docURL.substring(0, index); + for (let i = 0; i < threads; i++) { + const newWorker = new ChefWorker(); + newWorker.addEventListener("message", this.handleChefMessage.bind(this)); + let docURL = document.location.href.split(/[#?]/)[0]; + const index = docURL.lastIndexOf("/"); + if (index > 0) { + docURL = docURL.substring(0, index); + } + newWorker.postMessage({"action": "docURL", "data": docURL}); + newWorker.postMessage({"action": "inputNum", "data": 0}); + + this.chefWorkers.push({ + worker: newWorker, + inputNum: 0 + }); } - this.chefWorker.postMessage({"action": "docURL", "data": docURL}); } @@ -55,7 +66,22 @@ class WorkerWaiter { switch (r.action) { case "bakeComplete": - this.bakingComplete(r.data); + this.runningWorkers -= 1; + this.outputs.push({ + data: r.data, + inputNum: r.data.inputNum + }); + log.error(this.pendingInputs); + if (this.pendingInputs.length > 0) { + log.debug("Bake complete. Baking next input"); + this.bakeNextInput(r.data.inputNum); + } else if (this.runningWorkers <= 0) { + this.recipeConfig = undefined; + this.options = undefined; + this.progress = undefined; + this.step = undefined; + this.bakingComplete(); + } break; case "bakeError": this.app.handleError(r.data); @@ -104,22 +130,47 @@ class WorkerWaiter { /** - * Cancels the current bake by terminating the ChefWorker and creating a new one. + * Calcels the current bake by terminating and removing all ChefWorkers, + * and creating a new one */ cancelBake() { - this.chefWorker.terminate(); - this.registerChefWorker(); + for (let i = this.chefWorkers.length - 1; i >= 0; i--) { + this.chefWorkers[i].worker.terminate(); + this.chefWorkers.pop(); + } + this.setupChefWorkers(); this.setBakingStatus(false); this.manager.controls.showStaleIndicator(); } /** - * Handler for completed bakes. + * Handler for completed bakes + */ + bakingComplete() { + this.setBakingStatus(false); + + if (this.pendingInputs.length !== 0) return; + + for (let i = 0; i < this.outputs.length; i++) { + if (this.outputs[i].error) { + this.app.handleError(this.outputs[i].error); + } + } + + this.app.progress = this.outputs[0].data.progress; + this.app.dish = this.outputs[0].data.dish; + this.manager.recipe.updateBreakpointIndicator(this.app.progress); + this.manager.output.multiSet(this.outputs); + log.debug("--- Bake complete ---"); + } + + /** + * Handler for completed bakes * * @param {Object} response */ - bakingComplete(response) { + bakingCompleteOld(response) { this.setBakingStatus(false); if (!response) return; @@ -135,11 +186,10 @@ class WorkerWaiter { log.debug("--- Bake complete ---"); } - /** * Asks the ChefWorker to bake the current input using the current recipe. * - * @param {string} input + * @param {string | Array} input * @param {Object[]} recipeConfig * @param {Object} options * @param {number} progress @@ -148,16 +198,63 @@ class WorkerWaiter { bake(input, recipeConfig, options, progress, step) { this.setBakingStatus(true); - this.chefWorker.postMessage({ - action: "bake", - data: { + this.recipeConfig = recipeConfig; + this.options = options; + this.progress = progress; + this.step = step; + this.outputs = []; + + if (typeof input === "string") { + input = [{ input: input, - recipeConfig: recipeConfig, - options: options, - progress: progress, - step: step + inputNum: 0 + }]; + } + + const initialInputs = input.slice(0, this.chefWorkers.length); + this.pendingInputs = input.slice(this.chefWorkers.length, input.length); + + for (let i = 0; i < initialInputs.length; i++) { + this.runningWorkers += 1; + this.chefWorkers[i].inputNum = initialInputs[i].inputNum; + this.chefWorkers[i].worker.postMessage({ + action: "bake", + data: { + input: initialInputs[i].input, + recipeConfig: recipeConfig, + options: options, + progress: progress, + step: step, + inputNum: initialInputs[i].inputNum + } + }); + } + } + + + /** + * + * @param inputNum + */ + bakeNextInput(inputNum) { + this.runningWorkers += 1; + const nextInput = this.pendingInputs.pop(); + for (let i = 0; i < this.chefWorkers.length; i++) { + if (this.chefWorkers[i].inputNum === inputNum) { + this.chefWorkers[i].inputNum = nextInput.inputNum; + this.chefWorkers[i].worker.postMessage({ + action: "bake", + data: { + input: nextInput.input, + recipeConfig: this.recipeConfig, + options: this.options, + progress: this.progress, + step: this.step, + inputNum: nextInput.inputNum + } + }); } - }); + } } @@ -168,7 +265,7 @@ class WorkerWaiter { * @param {Object[]} [recipeConfig] */ silentBake(recipeConfig) { - this.chefWorker.postMessage({ + this.chefWorkers[0].worker.postMessage({ action: "silentBake", data: { recipeConfig: recipeConfig @@ -187,7 +284,7 @@ class WorkerWaiter { * @param {number} pos.end - The end offset. */ highlight(recipeConfig, direction, pos) { - this.chefWorker.postMessage({ + this.chefWorkers[0].postMessage({ action: "highlight", data: { recipeConfig: recipeConfig, @@ -208,7 +305,7 @@ class WorkerWaiter { getDishAs(dish, type, callback) { const id = this.callbackID++; this.callbacks[id] = callback; - this.chefWorker.postMessage({ + this.chefWorkers[0].worker.postMessage({ action: "getDishAs", data: { dish: dish, @@ -225,14 +322,15 @@ class WorkerWaiter { * @param {string} level */ setLogLevel(level) { - if (!this.chefWorker) return; + if (!this.chefWorkers || !this.chefWorkers.length > 0) return; - this.chefWorker.postMessage({ - action: "setLogLevel", - data: log.getLevel() - }); + for (let i = 0; i < this.chefWorkers.length; i++) { + this.chefWorkers[i].worker.postMessage({ + action: "setLogLevel", + data: log.getLevel() + }); + } } - }