From 4cdde7b80cef9e882db29ad0bdb0a30c45e49a6f Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 7 Jan 2025 16:24:32 +0100 Subject: [PATCH 1/7] pre-format --- .github/workflows/test.yml | 4 ++ package.json | 12 +++-- resources/main.mjs | 2 + resources/shared/benchmark.mjs | 1 + tests/{run.mjs => helper.mjs} | 76 +++++--------------------- tests/run-end2end.mjs | 97 ++++++++++++++++++++++++++++++++++ tests/run-unittests.mjs | 57 ++++++++++++++++++++ tests/server.mjs | 8 +-- 8 files changed, 188 insertions(+), 69 deletions(-) rename tests/{run.mjs => helper.mjs} (52%) create mode 100644 tests/run-end2end.mjs create mode 100644 tests/run-unittests.mjs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f4e4d985d..60e46ec6b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,3 +57,7 @@ jobs: run: | echo "Running in $BROWSER" npm run test:${{ matrix.browser }} + - name: Run end2end + run: | + echo "Running in $BROWSER" + npm run testEnd2End:${{ matrix.browser }} diff --git a/package.json b/package.json index 22f5a1c1f..f09db96a9 100644 --- a/package.json +++ b/package.json @@ -22,10 +22,14 @@ "pretty:check": "prettier --check ./", "pretty:fix": "prettier --write ./", "format": "npm run pretty:fix && npm run lint:fix", - "test:chrome": "node tests/run.mjs --browser chrome", - "test:firefox": "node tests/run.mjs --browser firefox", - "test:safari": "node tests/run.mjs --browser safari", - "test:edge": "node tests/run.mjs --browser edge" + "test:chrome": "node tests/run-unittests.mjs --browser chrome", + "test:firefox": "node tests/run-unittests.mjs --browser firefox", + "test:safari": "node tests/run-unittests.mjs --browser safari", + "test:edge": "node tests/run-unittests.mjs --browser edge", + "test-e2e:chrome": "node tests/run-end2end.mjs --browser chrome", + "test-e2e:firefox": "node tests/run-end2end.mjs --browser firefox", + "test-e2e:safari": "node tests/run-end2end.mjs --browser safari", + "test-e2e:edge": "node tests/run-end2end.mjs --browser edge" }, "devDependencies": { "@babel/core": "^7.21.3", diff --git a/resources/main.mjs b/resources/main.mjs index bbae96597..d644f7d94 100644 --- a/resources/main.mjs +++ b/resources/main.mjs @@ -23,6 +23,7 @@ class MainBenchmarkClient { constructor() { window.addEventListener("DOMContentLoaded", () => this.prepareUI()); this._showSection(window.location.hash); + globalThis.dispatchEvent(new Event("SpeedometerReady")); } start() { @@ -150,6 +151,7 @@ class MainBenchmarkClient { this.showResultsDetails(); else this.showResultsSummary(); + globalThis.dispatchEvent(new Event("SpeedometerDone")); } handleError(error) { diff --git a/resources/shared/benchmark.mjs b/resources/shared/benchmark.mjs index b691e86ac..0f9b57837 100644 --- a/resources/shared/benchmark.mjs +++ b/resources/shared/benchmark.mjs @@ -55,6 +55,7 @@ export class BenchmarkSuite { const result = await test.runAndRecord(params, this, test, this.record); measuredValues.tests[test.name] = result; measuredValues.total += result.total; + measuredValues.totalPrepare += result. onProgress?.(test.name); } diff --git a/tests/run.mjs b/tests/helper.mjs similarity index 52% rename from tests/run.mjs rename to tests/helper.mjs index 8d3461d09..3ea011693 100644 --- a/tests/run.mjs +++ b/tests/helper.mjs @@ -1,10 +1,9 @@ -#! /usr/bin/env node -/* eslint-disable-next-line no-unused-vars */ + +import commandLineUsage from "command-line-usage"; +import commandLineArgs from "command-line-args"; import serve from "./server.mjs"; + import { Builder, Capabilities } from "selenium-webdriver"; -import commandLineArgs from "command-line-args"; -import commandLineUsage from "command-line-usage"; -import assert from "assert"; const optionDefinitions = [ { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." }, @@ -65,63 +64,9 @@ switch (BROWSER) { } } -const PORT = options.port; +export const PORT = options.port; const server = serve(PORT); - -let driver; - -function printTree(node) { - console.log(node.title); - - for (const test of node.tests) { - console.group(); - if (test.state === "passed") { - console.log("\x1b[32m✓", `\x1b[0m${test.title}`); - } else { - console.log("\x1b[31m✖", `\x1b[0m${test.title}`); - console.group(); - console.log(`\x1b[31m${test.error.name}: ${test.error.message}`); - console.groupEnd(); - } - console.groupEnd(); - } - - for (const suite of node.suites) { - console.group(); - printTree(suite); - console.groupEnd(); - } -} - -async function test() { - driver = await new Builder().withCapabilities(capabilities).build(); - try { - await driver.get(`http://localhost:${PORT}/tests/index.html`); - - const { testResults, stats } = await driver.executeAsyncScript(function (callback) { - const returnResults = () => - callback({ - stats: globalThis.testRunner.stats, - testResults: globalThis.testResults, - }); - if (window.testResults) - returnResults(); - else - window.addEventListener("test-complete", returnResults, { once: true }); - }); - - printTree(testResults); - - console.log("\nChecking for passed tests..."); - assert(stats.passes > 0); - console.log("Checking for failed tests..."); - assert(stats.failures === 0); - } finally { - console.log("\nTests complete!"); - driver.quit(); - server.close(); - } -} +export let driver; process.on("unhandledRejection", (err) => { console.error(err); @@ -131,5 +76,12 @@ process.once("uncaughtException", (err) => { console.error(err); process.exit(1); }); +process.on("exit", () => stop()); -setImmediate(test); +driver = await new Builder().withCapabilities(capabilities).build(); + +export function stop() { + server.close(); + if (driver) + driver.close(); +} diff --git a/tests/run-end2end.mjs b/tests/run-end2end.mjs new file mode 100644 index 000000000..ae7a7399c --- /dev/null +++ b/tests/run-end2end.mjs @@ -0,0 +1,97 @@ +#! /usr/bin/env node +/* eslint-disable-next-line no-unused-vars */ +import assert from "assert"; +import { driver, PORT, stop } from "./helper.mjs"; + +import { Suites } from "../resources/tests.mjs"; + +async function testPage(url) { + console.log(`Testing: ${url}`); + await driver.get(`http://localhost:${PORT}/${url}`); + await driver.executeAsyncScript((callback) => { + if (globalThis.benchmarkClient) + callback(); + else + window.addEventListener("SpeedometerReady", () => callback(), { once: true }); + }); + console.log(" - Awaiting Benchmark"); + const metrics = await driver.executeAsyncScript((callback) => { + window.addEventListener("SpeedometerDone", () => callback(globalThis.benchmarkClient.metrics)); + globalThis.benchmarkClient.start(); + }); + validateMetrics(metrics); + return metrics; +} + +function validateMetrics(metrics) { + for (const [name, metric] of Object.entries(metrics)) + validateMetric(name, metric); + assert(metrics.Geomean.mean > 0); + assert(metrics.Score.mean > 0); +} + +function validateMetric(name, metric) { + assert(metric.name === name); + assert(metric.mean >= 0); +} + + +async function testIterations() { + const metrics = await testPage("index.html?iterationCount=3"); + Suites.forEach((suite) => { + if (!suite.disabled) { + const metric = metrics[suite.name]; + assert(metric.values.length === 3); + } + }); + assert(metrics.Geomean.values.length === 3); + assert(metrics.Score.values.length === 3); +} + +async function testAll() { + const metrics = await testPage("index.html?iterationCount=1&tags=all"); + Suites.forEach((suite) => { + assert(suite.name in metrics); + const metric = metrics[suite.name]; + assert(metric.values.length === 1); + }); + assert(metrics.Geomean.values.length === 1); + assert(metrics.Score.values.length === 1); +} + +async function testDeveloperMode() { + const params = [ + "developerMode", + "iterationCount=1", + "warmupBeforeSync=2", + "waitBeforeSync=2", + "shuffleSeed=123", + "suites=Perf-Dashboard" + ]; + const metrics = await testPage(`index.html?${params.join("&")}`); + Suites.forEach((suite) => { + if (suite.name === "Perf-Dashboard") { + const metric = metrics[suite.name]; + assert(metric.values.length === 1); + } else { + assert(!(suite.name in metrics)); + } + }); +} + +async function test() { + try { + await driver.manage().setTimeouts({ script: 60000 }); + await testIterations(); + await testAll(); + await testDeveloperMode(); + console.log("\nTests complete!"); + } catch (e) { + console.error("\nTests failed!"); + throw e; + } finally { + stop(); + } +} + +setImmediate(test); diff --git a/tests/run-unittests.mjs b/tests/run-unittests.mjs new file mode 100644 index 000000000..8f052cb1f --- /dev/null +++ b/tests/run-unittests.mjs @@ -0,0 +1,57 @@ +#! /usr/bin/env node +/* eslint-disable-next-line no-unused-vars */ +import assert from "assert"; +import { driver, PORT, stop} from "./helper.mjs"; + +function printTree(node) { + console.log(node.title); + + for (const test of node.tests) { + console.group(); + if (test.state === "passed") { + console.log("\x1b[32m✓", `\x1b[0m${test.title}`); + } else { + console.log("\x1b[31m✖", `\x1b[0m${test.title}`); + console.group(); + console.log(`\x1b[31m${test.error.name}: ${test.error.message}`); + console.groupEnd(); + } + console.groupEnd(); + } + + for (const suite of node.suites) { + console.group(); + printTree(suite); + console.groupEnd(); + } +} + +async function test() { + try { + await driver.get(`http://localhost:${PORT}/tests/index.html`); + + const { testResults, stats } = await driver.executeAsyncScript(function (callback) { + const returnResults = () => + callback({ + stats: globalThis.testRunner.stats, + testResults: globalThis.testResults, + }); + if (window.testResults) + returnResults(); + else + window.addEventListener("test-complete", returnResults, { once: true }); + }); + + printTree(testResults); + + console.log("\nChecking for passed tests..."); + assert(stats.passes > 0); + console.log("Checking for failed tests..."); + assert(stats.failures === 0); + } finally { + console.log("\nTests complete!"); + stop(); + } +} + +setImmediate(test); diff --git a/tests/server.mjs b/tests/server.mjs index 0b209b897..d113796c3 100644 --- a/tests/server.mjs +++ b/tests/server.mjs @@ -16,6 +16,7 @@ const MIME_TYPES = { }; const STATIC_PATH = path.join(process.cwd(), "./"); +console.log(STATIC_PATH); const toBool = [() => true, () => false]; export default function serve(port) { @@ -23,8 +24,8 @@ export default function serve(port) { throw new Error("Port is required"); const prepareFile = async (url) => { - const paths = [STATIC_PATH, url]; - if (url.endsWith("/")) + const paths = [STATIC_PATH, url.pathname]; + if (url.pathname.endsWith("/")) paths.push("index.html"); const filePath = path.join(...paths); const pathTraversal = !filePath.startsWith(STATIC_PATH); @@ -38,7 +39,8 @@ export default function serve(port) { const server = http .createServer(async (req, res) => { - const file = await prepareFile(req.url); + const url = new URL(`http://${process.env.HOST ?? "localhost"}${req.url}`); + const file = await prepareFile(url); const statusCode = file.found ? 200 : 404; const mimeType = MIME_TYPES[file.ext] || MIME_TYPES.default; res.writeHead(statusCode, { "Content-Type": mimeType }); From 60a3e222d86384d81acc2305f8de2728d8b4b285 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 7 Jan 2025 16:24:53 +0100 Subject: [PATCH 2/7] formatting --- resources/shared/benchmark.mjs | 3 +-- tests/helper.mjs | 1 - tests/run-end2end.mjs | 10 +--------- tests/run-unittests.mjs | 2 +- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/resources/shared/benchmark.mjs b/resources/shared/benchmark.mjs index 0f9b57837..25f839458 100644 --- a/resources/shared/benchmark.mjs +++ b/resources/shared/benchmark.mjs @@ -55,8 +55,7 @@ export class BenchmarkSuite { const result = await test.runAndRecord(params, this, test, this.record); measuredValues.tests[test.name] = result; measuredValues.total += result.total; - measuredValues.totalPrepare += result. - onProgress?.(test.name); + measuredValues.totalPrepare += result.onProgress?.(test.name); } performance.mark(suiteEndLabel); diff --git a/tests/helper.mjs b/tests/helper.mjs index 3ea011693..c408cd890 100644 --- a/tests/helper.mjs +++ b/tests/helper.mjs @@ -1,4 +1,3 @@ - import commandLineUsage from "command-line-usage"; import commandLineArgs from "command-line-args"; import serve from "./server.mjs"; diff --git a/tests/run-end2end.mjs b/tests/run-end2end.mjs index ae7a7399c..d5754a487 100644 --- a/tests/run-end2end.mjs +++ b/tests/run-end2end.mjs @@ -35,7 +35,6 @@ function validateMetric(name, metric) { assert(metric.mean >= 0); } - async function testIterations() { const metrics = await testPage("index.html?iterationCount=3"); Suites.forEach((suite) => { @@ -60,14 +59,7 @@ async function testAll() { } async function testDeveloperMode() { - const params = [ - "developerMode", - "iterationCount=1", - "warmupBeforeSync=2", - "waitBeforeSync=2", - "shuffleSeed=123", - "suites=Perf-Dashboard" - ]; + const params = ["developerMode", "iterationCount=1", "warmupBeforeSync=2", "waitBeforeSync=2", "shuffleSeed=123", "suites=Perf-Dashboard"]; const metrics = await testPage(`index.html?${params.join("&")}`); Suites.forEach((suite) => { if (suite.name === "Perf-Dashboard") { diff --git a/tests/run-unittests.mjs b/tests/run-unittests.mjs index 8f052cb1f..472faf91b 100644 --- a/tests/run-unittests.mjs +++ b/tests/run-unittests.mjs @@ -1,7 +1,7 @@ #! /usr/bin/env node /* eslint-disable-next-line no-unused-vars */ import assert from "assert"; -import { driver, PORT, stop} from "./helper.mjs"; +import { driver, PORT, stop } from "./helper.mjs"; function printTree(node) { console.log(node.title); From f0052f977bf6c08d44c35230fad732b1e117a0a5 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 7 Jan 2025 16:27:34 +0100 Subject: [PATCH 3/7] fix test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60e46ec6b..899bf73d7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,4 +60,4 @@ jobs: - name: Run end2end run: | echo "Running in $BROWSER" - npm run testEnd2End:${{ matrix.browser }} + npm run test-e2e:${{ matrix.browser }} From 2771dd8ab4de3ba2406236cdceee7065dab914bf Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Tue, 7 Jan 2025 18:00:12 +0100 Subject: [PATCH 4/7] revert accidental change --- resources/shared/benchmark.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/shared/benchmark.mjs b/resources/shared/benchmark.mjs index 25f839458..b691e86ac 100644 --- a/resources/shared/benchmark.mjs +++ b/resources/shared/benchmark.mjs @@ -55,7 +55,7 @@ export class BenchmarkSuite { const result = await test.runAndRecord(params, this, test, this.record); measuredValues.tests[test.name] = result; measuredValues.total += result.total; - measuredValues.totalPrepare += result.onProgress?.(test.name); + onProgress?.(test.name); } performance.mark(suiteEndLabel); From b32cf69316a4e48de68fba0395a35e5cd4e23586 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 13 Jan 2025 16:21:13 +0100 Subject: [PATCH 5/7] fixes --- package-lock.json | 86 ++++++++------ package.json | 2 +- resources/main.mjs | 3 +- tests/helper.mjs | 106 +++++++++--------- tests/index.html | 2 +- tests/run-end2end.mjs | 48 ++++++-- tests/run-unittests.mjs | 55 +++++---- tests/server.mjs | 3 +- .../benchmark-runner.mjs} | 8 +- 9 files changed, 186 insertions(+), 127 deletions(-) rename tests/{benchmark-runner-tests.mjs => unittests/benchmark-runner.mjs} (97%) diff --git a/package-lock.json b/package-lock.json index 0e5b02985..cdd1e17ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,7 +26,7 @@ "http-server": "^14.1.1", "mocha": "^10.2.0", "prettier": "^2.8.3", - "selenium-webdriver": "^4.8.0", + "selenium-webdriver": "^4.27.0", "sinon": "^17.0.1", "typescript": "^5.0.4" }, @@ -722,6 +722,12 @@ "node": ">=6.9.0" } }, + "node_modules/@bazel/runfiles": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@bazel/runfiles/-/runfiles-6.3.1.tgz", + "integrity": "sha512-1uLNT5NZsUVIGS4syuHwTzZ8HycMPyr6POA3FCE4GbMtc4rhoJk8aZKtNIRthJYfL+iioppi+rTfH3olMPr9nA==", + "dev": true + }, "node_modules/@ember-data/rfc395-data": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/@ember-data/rfc395-data/-/rfc395-data-0.0.4.tgz", @@ -5538,17 +5544,28 @@ "dev": true }, "node_modules/selenium-webdriver": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.8.0.tgz", - "integrity": "sha512-s/HL8WNwy1ggHR244+tAhjhyKMJnZLt1HKJ6Gn7nQgVjB/ybDF+46Uui0qI2J7AjPNJzlUmTncdC/jg/kKkn0A==", + "version": "4.27.0", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.27.0.tgz", + "integrity": "sha512-LkTJrNz5socxpPnWPODQ2bQ65eYx9JK+DQMYNihpTjMCqHwgWGYQnQTCAAche2W3ZP87alA+1zYPvgS8tHNzMQ==", "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/SeleniumHQ" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/selenium" + } + ], "dependencies": { - "jszip": "^3.10.0", - "tmp": "^0.2.1", - "ws": ">=8.11.0" + "@bazel/runfiles": "^6.3.1", + "jszip": "^3.10.1", + "tmp": "^0.2.3", + "ws": "^8.18.0" }, "engines": { - "node": ">= 14.20.0" + "node": ">= 14.21.0" } }, "node_modules/semver": { @@ -5956,15 +5973,12 @@ } }, "node_modules/tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", + "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", "dev": true, - "dependencies": { - "rimraf": "^3.0.0" - }, "engines": { - "node": ">=8.17.0" + "node": ">=14.14" } }, "node_modules/to-fast-properties": { @@ -6446,9 +6460,9 @@ "dev": true }, "node_modules/ws": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz", - "integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "dev": true, "engines": { "node": ">=10.0.0" @@ -7073,6 +7087,12 @@ "to-fast-properties": "^2.0.0" } }, + "@bazel/runfiles": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@bazel/runfiles/-/runfiles-6.3.1.tgz", + "integrity": "sha512-1uLNT5NZsUVIGS4syuHwTzZ8HycMPyr6POA3FCE4GbMtc4rhoJk8aZKtNIRthJYfL+iioppi+rTfH3olMPr9nA==", + "dev": true + }, "@ember-data/rfc395-data": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/@ember-data/rfc395-data/-/rfc395-data-0.0.4.tgz", @@ -10750,14 +10770,15 @@ "dev": true }, "selenium-webdriver": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.8.0.tgz", - "integrity": "sha512-s/HL8WNwy1ggHR244+tAhjhyKMJnZLt1HKJ6Gn7nQgVjB/ybDF+46Uui0qI2J7AjPNJzlUmTncdC/jg/kKkn0A==", + "version": "4.27.0", + "resolved": "https://registry.npmjs.org/selenium-webdriver/-/selenium-webdriver-4.27.0.tgz", + "integrity": "sha512-LkTJrNz5socxpPnWPODQ2bQ65eYx9JK+DQMYNihpTjMCqHwgWGYQnQTCAAche2W3ZP87alA+1zYPvgS8tHNzMQ==", "dev": true, "requires": { - "jszip": "^3.10.0", - "tmp": "^0.2.1", - "ws": ">=8.11.0" + "@bazel/runfiles": "^6.3.1", + "jszip": "^3.10.1", + "tmp": "^0.2.3", + "ws": "^8.18.0" } }, "semver": { @@ -11087,13 +11108,10 @@ "dev": true }, "tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dev": true, - "requires": { - "rimraf": "^3.0.0" - } + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", + "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", + "dev": true }, "to-fast-properties": { "version": "2.0.0", @@ -11457,9 +11475,9 @@ "dev": true }, "ws": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.0.tgz", - "integrity": "sha512-kU62emKIdKVeEIOIKVegvqpXMSTAMLJozpHZaJNDYqBjzlSYXQGviYwN1osDLJ9av68qHd4a2oSjd7yD4pacig==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", "dev": true, "requires": {} }, diff --git a/package.json b/package.json index f09db96a9..59a220303 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "http-server": "^14.1.1", "mocha": "^10.2.0", "prettier": "^2.8.3", - "selenium-webdriver": "^4.8.0", + "selenium-webdriver": "^4.27.0", "sinon": "^17.0.1", "typescript": "^5.0.4" } diff --git a/resources/main.mjs b/resources/main.mjs index d644f7d94..237f596b7 100644 --- a/resources/main.mjs +++ b/resources/main.mjs @@ -23,7 +23,7 @@ class MainBenchmarkClient { constructor() { window.addEventListener("DOMContentLoaded", () => this.prepareUI()); this._showSection(window.location.hash); - globalThis.dispatchEvent(new Event("SpeedometerReady")); + window.dispatchEvent(new Event("SpeedometerReady")); } start() { @@ -161,6 +161,7 @@ class MainBenchmarkClient { this._metrics = Object.create(null); this._populateInvalidScore(); this.showResultsSummary(); + throw error; } _populateValidScore(scoreResults) { diff --git a/tests/helper.mjs b/tests/helper.mjs index c408cd890..0c34651b4 100644 --- a/tests/helper.mjs +++ b/tests/helper.mjs @@ -2,7 +2,7 @@ import commandLineUsage from "command-line-usage"; import commandLineArgs from "command-line-args"; import serve from "./server.mjs"; -import { Builder, Capabilities } from "selenium-webdriver"; +import { Builder, Capabilities, logging } from "selenium-webdriver"; const optionDefinitions = [ { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." }, @@ -10,7 +10,7 @@ const optionDefinitions = [ { name: "help", alias: "h", description: "Print this help text." }, ]; -function printHelp(message = "") { +function printHelp(message = "", exitStatus = 0) { const usage = commandLineUsage([ { header: "Run all tests", @@ -20,67 +20,71 @@ function printHelp(message = "") { optionList: optionDefinitions, }, ]); - if (!message) { - console.log(usage); - process.exit(0); - } else { + if (message) { console.error(message); console.error(); - console.error(usage); - process.exit(1); } + console.log(usage); + process.exit(exitStatus); } -const options = commandLineArgs(optionDefinitions); +export default async function testSetup(helpText) { + const options = commandLineArgs(optionDefinitions); -if ("help" in options) - printHelp(); + if ("help" in options) + printHelp(helpText); -const BROWSER = options?.browser; -if (!BROWSER) - printHelp("No browser specified, use $BROWSER or --browser"); + const BROWSER = options?.browser; + if (!BROWSER) + printHelp("No browser specified, use $BROWSER or --browser", 1); -let capabilities; -switch (BROWSER) { - case "safari": - capabilities = Capabilities.safari(); - break; + let capabilities; + switch (BROWSER) { + case "safari": + capabilities = Capabilities.safari(); + break; - case "firefox": { - capabilities = Capabilities.firefox(); - break; - } - case "chrome": { - capabilities = Capabilities.chrome(); - break; - } - case "edge": { - capabilities = Capabilities.edge(); - break; - } - default: { - printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`); + case "firefox": { + capabilities = Capabilities.firefox(); + break; + } + case "chrome": { + capabilities = Capabilities.chrome(); + break; + } + case "edge": { + capabilities = Capabilities.edge(); + break; + } + default: { + printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`); + } } -} + const prefs = new logging.Preferences(); + prefs.setLevel(logging.Type.BROWSER, logging.Level.ALL); // Capture all log levels + capabilities.setLoggingPrefs(prefs); -export const PORT = options.port; -const server = serve(PORT); -export let driver; + const PORT = options.port; + const server = serve(PORT); + let driver; -process.on("unhandledRejection", (err) => { - console.error(err); - process.exit(1); -}); -process.once("uncaughtException", (err) => { - console.error(err); - process.exit(1); -}); -process.on("exit", () => stop()); + process.on("unhandledRejection", (err) => { + console.error(err); + process.exit(1); + }); + process.once("uncaughtException", (err) => { + console.error(err); + process.exit(1); + }); + process.on("exit", () => stop()); -driver = await new Builder().withCapabilities(capabilities).build(); + driver = await new Builder().withCapabilities(capabilities).build(); + driver.manage().window().setRect({ width: 1200, height: 1000 }); -export function stop() { - server.close(); - if (driver) - driver.close(); + function stop() { + server.close(); + if (driver) + driver.close(); + } + return { driver, PORT, stop }; } diff --git a/tests/index.html b/tests/index.html index c2ff47da8..ecc0ec41a 100644 --- a/tests/index.html +++ b/tests/index.html @@ -27,7 +27,7 @@ }, }); - await import("./benchmark-runner-tests.mjs"); + await import("./unittests/benchmark-runner.mjs"); globalThis.testResults = undefined; globalThis.testRunner = mocha.run(); diff --git a/tests/run-end2end.mjs b/tests/run-end2end.mjs index d5754a487..85fe550e0 100644 --- a/tests/run-end2end.mjs +++ b/tests/run-end2end.mjs @@ -1,24 +1,51 @@ #! /usr/bin/env node -/* eslint-disable-next-line no-unused-vars */ + import assert from "assert"; -import { driver, PORT, stop } from "./helper.mjs"; +import testSetup from "./helper.mjs"; import { Suites } from "../resources/tests.mjs"; +const HELP = ` +This script runs end2end tests by invoking the benchmark via the main +Speedometer page in /index.html. +`.trim(); + +const { driver, PORT, stop } = await testSetup(HELP); + async function testPage(url) { console.log(`Testing: ${url}`); await driver.get(`http://localhost:${PORT}/${url}`); + await driver.executeAsyncScript((callback) => { if (globalThis.benchmarkClient) callback(); else - window.addEventListener("SpeedometerReady", () => callback(), { once: true }); + globalThis.addEventListener("SpeedometerReady", () => callback(), { once: true }); }); + console.log(" - Awaiting Benchmark"); - const metrics = await driver.executeAsyncScript((callback) => { - window.addEventListener("SpeedometerDone", () => callback(globalThis.benchmarkClient.metrics)); + const { error, metrics } = await driver.executeAsyncScript((callback) => { + globalThis.addEventListener("SpeedometerDone", () => callback({ metrics: globalThis.benchmarkClient.metrics }), { once: true }); + // Install error handlers to report page errors back to selenium. + globalThis.addEventListener("error", (message, source, lineno, colno, error) => + callback({ + error: { message, source, lineno, colno, error }, + }) + ); + globalThis.addEventListener("unhandledrejection", (e) => { + callback({ + error: { + message: e.reason.toString(), + stack: e.reason?.stack, + }, + }); + }); globalThis.benchmarkClient.start(); }); + + if (error) + throw new Error(error.message + (error?.stack ?? "")); + validateMetrics(metrics); return metrics; } @@ -36,15 +63,18 @@ function validateMetric(name, metric) { } async function testIterations() { - const metrics = await testPage("index.html?iterationCount=3"); + const iterationCount = 2; + const metrics = await testPage(`index.html?iterationCount=${iterationCount}`); Suites.forEach((suite) => { if (!suite.disabled) { const metric = metrics[suite.name]; - assert(metric.values.length === 3); + assert(metric.values.length === iterationCount); + } else { + assert(!(suite.name in metrics)); } }); - assert(metrics.Geomean.values.length === 3); - assert(metrics.Score.values.length === 3); + assert(metrics.Geomean.values.length === iterationCount); + assert(metrics.Score.values.length === iterationCount); } async function testAll() { diff --git a/tests/run-unittests.mjs b/tests/run-unittests.mjs index 472faf91b..428a10f7d 100644 --- a/tests/run-unittests.mjs +++ b/tests/run-unittests.mjs @@ -1,30 +1,14 @@ #! /usr/bin/env node -/* eslint-disable-next-line no-unused-vars */ + import assert from "assert"; -import { driver, PORT, stop } from "./helper.mjs"; +import testSetup from "./helper.mjs"; -function printTree(node) { - console.log(node.title); +const HELP = ` +This script runs the unittests located in tests/unittests/* +through the mocha web interface located at tests/index.html +`.trim(); - for (const test of node.tests) { - console.group(); - if (test.state === "passed") { - console.log("\x1b[32m✓", `\x1b[0m${test.title}`); - } else { - console.log("\x1b[31m✖", `\x1b[0m${test.title}`); - console.group(); - console.log(`\x1b[31m${test.error.name}: ${test.error.message}`); - console.groupEnd(); - } - console.groupEnd(); - } - - for (const suite of node.suites) { - console.group(); - printTree(suite); - console.groupEnd(); - } -} +const { driver, PORT, stop } = testSetup(HELP); async function test() { try { @@ -39,7 +23,7 @@ async function test() { if (window.testResults) returnResults(); else - window.addEventListener("test-complete", returnResults, { once: true }); + globalThis.addEventListener("test-complete", returnResults, { once: true }); }); printTree(testResults); @@ -54,4 +38,27 @@ async function test() { } } +function printTree(node) { + console.log(node.title); + + for (const test of node.tests) { + console.group(); + if (test.state === "passed") { + console.log("\x1b[32m✓", `\x1b[0m${test.title}`); + } else { + console.log("\x1b[31m✖", `\x1b[0m${test.title}`); + console.group(); + console.log(`\x1b[31m${test.error.name}: ${test.error.message}`); + console.groupEnd(); + } + console.groupEnd(); + } + + for (const suite of node.suites) { + console.group(); + printTree(suite); + console.groupEnd(); + } +} + setImmediate(test); diff --git a/tests/server.mjs b/tests/server.mjs index d113796c3..8f73e7067 100644 --- a/tests/server.mjs +++ b/tests/server.mjs @@ -16,7 +16,6 @@ const MIME_TYPES = { }; const STATIC_PATH = path.join(process.cwd(), "./"); -console.log(STATIC_PATH); const toBool = [() => true, () => false]; export default function serve(port) { @@ -39,7 +38,7 @@ export default function serve(port) { const server = http .createServer(async (req, res) => { - const url = new URL(`http://${process.env.HOST ?? "localhost"}${req.url}`); + const url = new URL(`http://localhost${req.url}`); const file = await prepareFile(url); const statusCode = file.found ? 200 : 404; const mimeType = MIME_TYPES[file.ext] || MIME_TYPES.default; diff --git a/tests/benchmark-runner-tests.mjs b/tests/unittests/benchmark-runner.mjs similarity index 97% rename from tests/benchmark-runner-tests.mjs rename to tests/unittests/benchmark-runner.mjs index 449ceb266..000382ce3 100644 --- a/tests/benchmark-runner-tests.mjs +++ b/tests/unittests/benchmark-runner.mjs @@ -1,7 +1,7 @@ -import { BenchmarkRunner } from "../resources/benchmark-runner.mjs"; -import { SuiteRunner } from "../resources/suite-runner.mjs"; -import { TestRunner } from "../resources/shared/test-runner.mjs"; -import { defaultParams } from "../resources/shared/params.mjs"; +import { BenchmarkRunner } from "../../resources/benchmark-runner.mjs"; +import { SuiteRunner } from "../../resources/suite-runner.mjs"; +import { TestRunner } from "../../resources/shared/test-runner.mjs"; +import { defaultParams } from "../../resources/shared/params.mjs"; function TEST_FIXTURE(name) { return { From 18c64bb545a4c0a69b69794c9b8dfbf878a72a6a Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 13 Jan 2025 16:57:09 +0100 Subject: [PATCH 6/7] cleanup --- tests/run-end2end.mjs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/run-end2end.mjs b/tests/run-end2end.mjs index 85fe550e0..b7e4886cd 100644 --- a/tests/run-end2end.mjs +++ b/tests/run-end2end.mjs @@ -25,7 +25,14 @@ async function testPage(url) { console.log(" - Awaiting Benchmark"); const { error, metrics } = await driver.executeAsyncScript((callback) => { - globalThis.addEventListener("SpeedometerDone", () => callback({ metrics: globalThis.benchmarkClient.metrics }), { once: true }); + globalThis.addEventListener( + "SpeedometerDone", + () => + callback({ + metrics: globalThis.benchmarkClient.metrics, + }), + { once: true } + ); // Install error handlers to report page errors back to selenium. globalThis.addEventListener("error", (message, source, lineno, colno, error) => callback({ From c8621b427445f8bf8784f286fb250b15dc114e86 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Mon, 13 Jan 2025 16:59:38 +0100 Subject: [PATCH 7/7] missing await --- tests/run-unittests.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run-unittests.mjs b/tests/run-unittests.mjs index 428a10f7d..5fd7fb86d 100644 --- a/tests/run-unittests.mjs +++ b/tests/run-unittests.mjs @@ -8,7 +8,7 @@ This script runs the unittests located in tests/unittests/* through the mocha web interface located at tests/index.html `.trim(); -const { driver, PORT, stop } = testSetup(HELP); +const { driver, PORT, stop } = await testSetup(HELP); async function test() { try {