Tests now display a progress bar and report long running tests

2020-03-13 14:59:48 +00:00 · 2020-03-13 14:59:48 +00:00 · 4308c717c3
parent 75da5b650c
commit 4308c717c3
10 changed files with 196 additions and 83 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -11,7 +11,7 @@ before_script:
  - export NODE_OPTIONS=--max_old_space_size=2048
 script:
  - grunt lint
-  - grunt test
+  - npm test
  - grunt testnodeconsumer
  - grunt prod --msg="$COMPILE_MSG"
  - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui
--- a/Gruntfile.js
+++ b/Gruntfile.js
@ -36,11 +36,10 @@ module.exports = function (grunt) {
            "clean:node", "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex"
        ]);

-    grunt.registerTask("test",
-        "A task which runs all the operation tests in the tests directory.",
+    grunt.registerTask("configTests",
+        "A task which configures config files in preparation for tests to be run. Use `npm tests` to run tests.",
        [
-            "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex",
-            "exec:nodeTests", "exec:opTests"
+            "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex"
        ]);

    grunt.registerTask("testui",
@ -55,7 +54,6 @@ module.exports = function (grunt) {
        "Lints the code base",
        ["eslint", "exec:repoSize"]);

-    grunt.registerTask("tests", "test");
    grunt.registerTask("lint", "eslint");

    grunt.registerTask("findModules",
@ -385,15 +383,9 @@ module.exports = function (grunt) {
                ]),
                sync: true
            },
-            opTests: {
-                command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
-            },
            browserTests: {
                command: "./node_modules/.bin/nightwatch --env prod"
            },
-            nodeTests: {
-                command: "node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs"
-            },
            setupNodeConsumers: {
                command: chainCommands([
                    "echo '\n--- Testing node consumers ---'",
--- a/package-lock.json
+++ b/package-lock.json
@ -4413,6 +4413,49 @@
        "restore-cursor": "^3.1.0"
      }
    },
+    "cli-progress": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.6.0.tgz",
+      "integrity": "sha512-elg6jkiDedYrvwqWSae2FGvtbMo37Lo04oI9jJ5cI43Ge3jrDPWzeL3axv7MgBLYHDY/kGio/CXa49m4MWMrNw==",
+      "dev": true,
+      "requires": {
+        "colors": "^1.1.2",
+        "string-width": "^2.1.1"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
+          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
+          "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
+          "dev": true
+        },
+        "string-width": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
+          "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
+          "dev": true,
+          "requires": {
+            "is-fullwidth-code-point": "^2.0.0",
+            "strip-ansi": "^4.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
+          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^3.0.0"
+          }
+        }
+      }
+    },
    "cli-spinners": {
      "version": "2.2.0",
      "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.2.0.tgz",
@ -12204,6 +12247,14 @@
        "request": "^2.81.0",
        "request-progress": "^2.0.1",
        "which": "^1.2.10"
+      },
+      "dependencies": {
+        "progress": {
+          "version": "1.1.8",
+          "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz",
+          "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=",
+          "dev": true
+        }
      }
    },
    "phin": {
@ -12589,11 +12640,6 @@
      "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
      "dev": true
    },
-    "progress": {
-      "version": "1.1.8",
-      "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz",
-      "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74="
-    },
    "promise-inflight": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
@ -15027,6 +15073,13 @@
        "more-entropy": ">=0.0.7",
        "progress": "~1.1.2",
        "uglify-js": "^3.1.9"
+      },
+      "dependencies": {
+        "progress": {
+          "version": "1.1.8",
+          "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz",
+          "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74="
+        }
      }
    },
    "true-case-path": {
--- a/package.json
+++ b/package.json
@ -44,6 +44,7 @@
    "babel-loader": "^8.0.6",
    "babel-plugin-dynamic-import-node": "^2.3.0",
    "chromedriver": "^80.0.1",
+    "cli-progress": "^3.6.0",
    "colors": "^1.4.0",
    "copy-webpack-plugin": "^5.0.5",
    "css-loader": "^3.2.1",
@ -159,7 +160,7 @@
    "start": "grunt dev",
    "build": "grunt prod",
    "repl": "node src/node/repl.js",
-    "test": "grunt test",
+    "test": "grunt configTests && node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs && node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs",
    "test-node-consumer": "grunt testnodeconsumer",
    "testui": "grunt testui",
    "testuidev": "npx nightwatch --env=dev",
--- a/src/core/Utils.mjs
+++ b/src/core/Utils.mjs
@ -1335,7 +1335,7 @@ export function sendStatusMessage(msg) {
        self.sendStatusMessage(msg);
    else if (isWebEnvironment())
        app.alert(msg, 10000);
-    else if (isNodeEnvironment())
+    else if (isNodeEnvironment() && !global.TESTING)
        // eslint-disable-next-line no-console
        console.debug(msg);
 }
--- a/src/node/api.mjs
+++ b/src/node/api.mjs
@ -282,11 +282,11 @@ export function help(input) {
        .map(result => result.hydrated);

    if (matches && matches.length) {
-        console.log(`${matches.length} result${matches.length > 1 ? "s" : ""} found.`);
+        // console.log(`${matches.length} result${matches.length > 1 ? "s" : ""} found.`);
        return matches;
    }

-    console.log("No results found.");
+    // console.log("No results found.");
    return null;
 }

--- a/tests/lib/TestRegister.mjs
+++ b/tests/lib/TestRegister.mjs
@ -10,6 +10,8 @@
 * @license Apache-2.0
 */
 import Chef from "../../src/core/Chef.mjs";
+import Utils from "../../src/core/Utils.mjs";
+import cliProgress from "cli-progress";

 /**
 * Object to store and run the list of tests.
@ -47,68 +49,99 @@ class TestRegister {
    /**
     * Runs all the tests in the register.
     */
-    runTests () {
-        console.log("Running tests...");
-        return Promise.all(
-            this.tests.map(function(test, i) {
-                const chef = new Chef();
+    async runTests () {
+        const progBar = new cliProgress.SingleBar({
+            format: formatter,
+            stopOnComplete: true
+        }, cliProgress.Presets.shades_classic);
+        const testResults = [];

-                return chef.bake(
-                    test.input,
-                    test.recipeConfig,
-                    {},
-                    0,
-                    false
-                ).then(function(result) {
-                    const ret = {
-                        test: test,
-                        status: null,
-                        output: null,
-                    };
+        console.log("Running operation tests...");
+        progBar.start(this.tests.length, 0, {
+            msg: "Setting up"
+        });

-                    if (result.error) {
-                        if (test.expectedError) {
-                            ret.status = "passing";
-                        } else {
-                            ret.status = "erroring";
-                            ret.output = result.error.displayStr;
-                        }
-                    } else {
-                        if (test.expectedError) {
-                            ret.status = "failing";
-                            ret.output = "Expected an error but did not receive one.";
-                        } else if (result.result === test.expectedOutput) {
-                            ret.status = "passing";
-                        } else if ("expectedMatch" in test && test.expectedMatch.test(result.result)) {
-                            ret.status = "passing";
-                        } else {
-                            ret.status = "failing";
-                            const expected = test.expectedOutput ? test.expectedOutput :
-                                test.expectedMatch ? test.expectedMatch.toString() : "unknown";
-                            ret.output = [
-                                "Expected",
-                                "\t" + expected.replace(/\n/g, "\n\t"),
-                                "Received",
-                                "\t" + result.result.replace(/\n/g, "\n\t"),
-                            ].join("\n");
-                        }
-                    }
+        for (const test of this.tests) {
+            progBar.update(testResults.length, {
+                msg: test.name
+            });

-                    return ret;
-                });
-            })
-        );
+            const chef = new Chef();
+            const result = await chef.bake(
+                test.input,
+                test.recipeConfig,
+                {},
+                0,
+                false
+            );
+
+            const ret = {
+                test: test,
+                status: null,
+                output: null,
+                duration: result.duration
+            };
+
+            if (result.error) {
+                if (test.expectedError) {
+                    ret.status = "passing";
+                } else {
+                    ret.status = "erroring";
+                    ret.output = result.error.displayStr;
+                }
+            } else {
+                if (test.expectedError) {
+                    ret.status = "failing";
+                    ret.output = "Expected an error but did not receive one.";
+                } else if (result.result === test.expectedOutput) {
+                    ret.status = "passing";
+                } else if ("expectedMatch" in test && test.expectedMatch.test(result.result)) {
+                    ret.status = "passing";
+                } else {
+                    ret.status = "failing";
+                    const expected = test.expectedOutput ? test.expectedOutput :
+                        test.expectedMatch ? test.expectedMatch.toString() : "unknown";
+                    ret.output = [
+                        "Expected",
+                        "\t" + expected.replace(/\n/g, "\n\t"),
+                        "Received",
+                        "\t" + result.result.replace(/\n/g, "\n\t"),
+                    ].join("\n");
+                }
+            }
+
+            testResults.push(ret);
+            progBar.increment();
+        }
+
+        return testResults;
    }

    /**
     * Run all api related tests and wrap results in report format
     */
-    runApiTests() {
-        return Promise.all(this.apiTests.map(async function(test, i) {
+    async runApiTests() {
+        const progBar = new cliProgress.SingleBar({
+            format: formatter,
+            stopOnComplete: true
+        }, cliProgress.Presets.shades_classic);
+        const testResults = [];
+
+        console.log("Running Node API tests...");
+        progBar.start(this.apiTests.length, 0, {
+            msg: "Setting up"
+        });
+
+        global.TESTING = true;
+        for (const test of this.apiTests) {
+            progBar.update(testResults.length, {
+                msg: test.name
+            });
+
            const result = {
                test: test,
                status: null,
-                output: null,
+                output: null
            };
            try {
                await test.run();
@ -117,10 +150,37 @@ class TestRegister {
                result.status = "erroring";
                result.output = e.message;
            }
-            return result;
-        }));
+
+            testResults.push(result);
+            progBar.increment();
+        }
+
+        return testResults;
    }
 }

+
+/**
+ * Formatter for the progress bar
+ *
+ * @param {Object} options
+ * @param {Object} params
+ * @param {Object} payload
+ * @returns {string}
+ */
+function formatter(options, params, payload) {
+    const bar = options.barCompleteString.substr(0, Math.round(params.progress * options.barsize)) +
+        options.barIncompleteString.substr(0, Math.round((1-params.progress) * options.barsize));
+
+    const percentage = Math.floor(params.progress * 100),
+        duration = Math.floor((Date.now() - params.startTime) / 1000);
+
+    let testName = payload.msg ? payload.msg : "";
+    if (params.value >= params.total) testName = "Tests completed";
+    testName = Utils.truncate(testName, 25).padEnd(25, " ");
+
+    return `${testName} ${bar} ${params.value}/${params.total} | ${percentage}% | Duration: ${duration}s`;
+}
+
 // Export an instance to make a singleton
 export default new TestRegister();
--- a/tests/lib/utils.mjs
+++ b/tests/lib/utils.mjs
@ -33,6 +33,10 @@ function handleTestResult(testStatus, testResult) {
    testStatus.allTestsPassing = testStatus.allTestsPassing && testResult.status === "passing";
    testStatus.counts[testResult.status] = (testStatus.counts[testResult.status] || 0) + 1;
    testStatus.counts.total += 1;
+
+    if (testResult.duration > 2000) {
+        console.log(`'${testResult.test.name}' took ${(testResult.duration / 1000).toFixed(2)}s to complete`);
+    }
 }

 /**
@ -42,8 +46,6 @@ function handleTestResult(testStatus, testResult) {
 * @param {Object[]} results - results from TestRegister
 */
 export function logTestReport(testStatus, results) {
-    console.log("Tests completed.");
-
    results.forEach(r => handleTestResult(testStatus, r));

    console.log();
@ -80,8 +82,9 @@ export function logTestReport(testStatus, results) {
 * Fail if the process takes longer than 60 seconds.
 */
 export function setLongTestFailure() {
+    const timeLimit = 60;
    setTimeout(function() {
-        console.log("Tests took longer than 60 seconds to run, returning.");
+        console.log(`Tests took longer than ${timeLimit} seconds to run, returning.`);
        process.exit(1);
-    }, 60 * 1000);
+    }, timeLimit * 1000);
 }
--- a/tests/node/index.mjs
+++ b/tests/node/index.mjs
@ -35,6 +35,7 @@ setLongTestFailure();

 const logOpsTestReport = logTestReport.bind(null, testStatus);

-TestRegister.runApiTests()
-    .then(logOpsTestReport);
-
+(async function() {
+    const results = await TestRegister.runApiTests();
+    logOpsTestReport(results);
+})();
--- a/tests/operations/index.mjs
+++ b/tests/operations/index.mjs
@ -116,5 +116,8 @@ setLongTestFailure();

 const logOpsTestReport = logTestReport.bind(null, testStatus);

-TestRegister.runTests()
-    .then(logOpsTestReport);
+(async function() {
+    const results = await TestRegister.runTests();
+    logOpsTestReport(results);
+})();
+