Add end2end tests

.github/workflows/test.yml

@@ -57,3 +57,7 @@ jobs:
               run: |
                   echo "Running in $BROWSER"
                   npm run test:${{ matrix.browser }}
+            - name: Run end2end
+              run: |
+                  echo "Running in $BROWSER"
+                  npm run test-e2e:${{ matrix.browser }}

package.json

@@ -22,10 +22,14 @@
         "pretty:check": "prettier --check ./",
         "pretty:fix": "prettier --write ./",
         "format": "npm run pretty:fix && npm run lint:fix",
-        "test:chrome": "node tests/run.mjs --browser chrome",
-        "test:firefox": "node tests/run.mjs --browser firefox",
-        "test:safari": "node tests/run.mjs --browser safari",
-        "test:edge": "node tests/run.mjs --browser edge"
+        "test:chrome": "node tests/run-unittests.mjs --browser chrome",
+        "test:firefox": "node tests/run-unittests.mjs --browser firefox",
+        "test:safari": "node tests/run-unittests.mjs --browser safari",
+        "test:edge": "node tests/run-unittests.mjs --browser edge",
+        "test-e2e:chrome": "node tests/run-end2end.mjs --browser chrome",
+        "test-e2e:firefox": "node tests/run-end2end.mjs --browser firefox",
+        "test-e2e:safari": "node tests/run-end2end.mjs --browser safari",
+        "test-e2e:edge": "node tests/run-end2end.mjs --browser edge"
     },
     "devDependencies": {
         "@babel/core": "^7.21.3",

resources/main.mjs

@@ -23,6 +23,7 @@ class MainBenchmarkClient {
     constructor() {
         window.addEventListener("DOMContentLoaded", () => this.prepareUI());
         this._showSection(window.location.hash);
+        globalThis.dispatchEvent(new Event("SpeedometerReady"));
     }
 
     start() {
@@ -150,6 +151,7 @@ class MainBenchmarkClient {
             this.showResultsDetails();
         else
             this.showResultsSummary();
+        globalThis.dispatchEvent(new Event("SpeedometerDone"));
     }
 
     handleError(error) {

resources/shared/benchmark.mjs

@@ -55,7 +55,7 @@ export class BenchmarkSuite {
             const result = await test.runAndRecord(params, this, test, this.record);
             measuredValues.tests[test.name] = result;
             measuredValues.total += result.total;
-            onProgress?.(test.name);
+            measuredValues.totalPrepare += result.onProgress?.(test.name);
         }
 
         performance.mark(suiteEndLabel);

tests/run.mjs → tests/helper.mjs

@@ -1,10 +1,8 @@
-#! /usr/bin/env node
-/* eslint-disable-next-line  no-unused-vars */
+import commandLineUsage from "command-line-usage";
+import commandLineArgs from "command-line-args";
 import serve from "./server.mjs";
+
 import { Builder, Capabilities } from "selenium-webdriver";
-import commandLineArgs from "command-line-args";
-import commandLineUsage from "command-line-usage";
-import assert from "assert";
 
 const optionDefinitions = [
     { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." },
@@ -65,63 +63,9 @@ switch (BROWSER) {
     }
 }
 
-const PORT = options.port;
+export const PORT = options.port;
 const server = serve(PORT);
-
-let driver;
-
-function printTree(node) {
-    console.log(node.title);
-
-    for (const test of node.tests) {
-        console.group();
-        if (test.state === "passed") {
-            console.log("\x1b[32m✓", `\x1b[0m${test.title}`);
-        } else {
-            console.log("\x1b[31m✖", `\x1b[0m${test.title}`);
-            console.group();
-            console.log(`\x1b[31m${test.error.name}: ${test.error.message}`);
-            console.groupEnd();
-        }
-        console.groupEnd();
-    }
-
-    for (const suite of node.suites) {
-        console.group();
-        printTree(suite);
-        console.groupEnd();
-    }
-}
-
-async function test() {
-    driver = await new Builder().withCapabilities(capabilities).build();
-    try {
-        await driver.get(`http://localhost:${PORT}/tests/index.html`);
-
-        const { testResults, stats } = await driver.executeAsyncScript(function (callback) {
-            const returnResults = () =>
-                callback({
-                    stats: globalThis.testRunner.stats,
-                    testResults: globalThis.testResults,
-                });
-            if (window.testResults)
-                returnResults();
-            else
-                window.addEventListener("test-complete", returnResults, { once: true });
-        });
-
-        printTree(testResults);
-
-        console.log("\nChecking for passed tests...");
-        assert(stats.passes > 0);
-        console.log("Checking for failed tests...");
-        assert(stats.failures === 0);
-    } finally {
-        console.log("\nTests complete!");
-        driver.quit();
-        server.close();
-    }
-}
+export let driver;
 
 process.on("unhandledRejection", (err) => {
     console.error(err);
@@ -131,5 +75,12 @@ process.once("uncaughtException", (err) => {
     console.error(err);
     process.exit(1);
 });
+process.on("exit", () => stop());
 
-setImmediate(test);
+driver = await new Builder().withCapabilities(capabilities).build();
+
+export function stop() {
+    server.close();
+    if (driver)
+        driver.close();
+}

tests/run-end2end.mjs

@@ -0,0 +1,89 @@
+#! /usr/bin/env node
+/* eslint-disable-next-line  no-unused-vars */
+import assert from "assert";
+import { driver, PORT, stop } from "./helper.mjs";
+
+import { Suites } from "../resources/tests.mjs";
+
+async function testPage(url) {
+    console.log(`Testing: ${url}`);
+    await driver.get(`http://localhost:${PORT}/${url}`);
+    await driver.executeAsyncScript((callback) => {
+        if (globalThis.benchmarkClient)
+            callback();
+        else
+            window.addEventListener("SpeedometerReady", () => callback(), { once: true });
+    });
+    console.log("    - Awaiting Benchmark");
+    const metrics = await driver.executeAsyncScript((callback) => {
+        window.addEventListener("SpeedometerDone", () => callback(globalThis.benchmarkClient.metrics));
+        globalThis.benchmarkClient.start();
+    });
+    validateMetrics(metrics);
+    return metrics;
+}
+
+function validateMetrics(metrics) {
+    for (const [name, metric] of Object.entries(metrics))
+        validateMetric(name, metric);
+    assert(metrics.Geomean.mean > 0);
+    assert(metrics.Score.mean > 0);
+}
+
+function validateMetric(name, metric) {
+    assert(metric.name === name);
+    assert(metric.mean >= 0);
+}
+
+async function testIterations() {
+    const metrics = await testPage("index.html?iterationCount=3");
+    Suites.forEach((suite) => {
+        if (!suite.disabled) {
+            const metric = metrics[suite.name];
+            assert(metric.values.length === 3);
+        }
+    });
+    assert(metrics.Geomean.values.length === 3);
+    assert(metrics.Score.values.length === 3);
+}
+
+async function testAll() {
+    const metrics = await testPage("index.html?iterationCount=1&tags=all");
+    Suites.forEach((suite) => {
+        assert(suite.name in metrics);
+        const metric = metrics[suite.name];
+        assert(metric.values.length === 1);
+    });
+    assert(metrics.Geomean.values.length === 1);
+    assert(metrics.Score.values.length === 1);
+}
+
+async function testDeveloperMode() {
+    const params = ["developerMode", "iterationCount=1", "warmupBeforeSync=2", "waitBeforeSync=2", "shuffleSeed=123", "suites=Perf-Dashboard"];
+    const metrics = await testPage(`index.html?${params.join("&")}`);
+    Suites.forEach((suite) => {
+        if (suite.name === "Perf-Dashboard") {
+            const metric = metrics[suite.name];
+            assert(metric.values.length === 1);
+        } else {
+            assert(!(suite.name in metrics));
+        }
+    });
+}
+
+async function test() {
+    try {
+        await driver.manage().setTimeouts({ script: 60000 });
+        await testIterations();
+        await testAll();
+        await testDeveloperMode();
+        console.log("\nTests complete!");
+    } catch (e) {
+        console.error("\nTests failed!");
+        throw e;
+    } finally {
+        stop();
+    }
+}
+
+setImmediate(test);

tests/run-unittests.mjs

@@ -0,0 +1,57 @@
+#! /usr/bin/env node
+/* eslint-disable-next-line  no-unused-vars */
+import assert from "assert";
+import { driver, PORT, stop } from "./helper.mjs";
+
+function printTree(node) {
+    console.log(node.title);
+
+    for (const test of node.tests) {
+        console.group();
+        if (test.state === "passed") {
+            console.log("\x1b[32m✓", `\x1b[0m${test.title}`);
+        } else {
+            console.log("\x1b[31m✖", `\x1b[0m${test.title}`);
+            console.group();
+            console.log(`\x1b[31m${test.error.name}: ${test.error.message}`);
+            console.groupEnd();
+        }
+        console.groupEnd();
+    }
+
+    for (const suite of node.suites) {
+        console.group();
+        printTree(suite);
+        console.groupEnd();
+    }
+}
+
+async function test() {
+    try {
+        await driver.get(`http://localhost:${PORT}/tests/index.html`);
+
+        const { testResults, stats } = await driver.executeAsyncScript(function (callback) {
+            const returnResults = () =>
+                callback({
+                    stats: globalThis.testRunner.stats,
+                    testResults: globalThis.testResults,
+                });
+            if (window.testResults)
+                returnResults();
+            else
+                window.addEventListener("test-complete", returnResults, { once: true });
+        });
+
+        printTree(testResults);
+
+        console.log("\nChecking for passed tests...");
+        assert(stats.passes > 0);
+        console.log("Checking for failed tests...");
+        assert(stats.failures === 0);
+    } finally {
+        console.log("\nTests complete!");
+        stop();
+    }
+}
+
+setImmediate(test);

tests/server.mjs

@@ -16,15 +16,16 @@ const MIME_TYPES = {
 };
 
 const STATIC_PATH = path.join(process.cwd(), "./");
+console.log(STATIC_PATH);
 const toBool = [() => true, () => false];
 
 export default function serve(port) {
     if (!port)
         throw new Error("Port is required");
 
     const prepareFile = async (url) => {
-        const paths = [STATIC_PATH, url];
-        if (url.endsWith("/"))
+        const paths = [STATIC_PATH, url.pathname];
+        if (url.pathname.endsWith("/"))
             paths.push("index.html");
         const filePath = path.join(...paths);
         const pathTraversal = !filePath.startsWith(STATIC_PATH);
@@ -38,7 +39,8 @@ export default function serve(port) {
 
     const server = http
         .createServer(async (req, res) => {
-            const file = await prepareFile(req.url);
+            const url = new URL(`http://${process.env.HOST ?? "localhost"}${req.url}`);
+            const file = await prepareFile(url);
             const statusCode = file.found ? 200 : 404;
             const mimeType = MIME_TYPES[file.ext] || MIME_TYPES.default;
             res.writeHead(statusCode, { "Content-Type": mimeType });