feat(test-runner-webdriver): initial implementation (modernweb-dev#1033)

.changeset/modern-rats-nail.md

@@ -0,0 +1,5 @@
+---
+'@web/test-runner-webdriver': patch
+---
+
+Initial implementation of WebdriverIO launcher

docs/docs/test-runner/browser-launchers/webdriver.md

@@ -0,0 +1,38 @@
+# Test Runner >> Browser Launchers >> Webdriver ||80
+
+Run tests using [WebdriverIO](https://webdriver.io).
+
+## Usage
+
+1. Make sure you have a selenium server running, either locally or remote.
+
+2. Add the Webdriver launcher to your test runner config and specify relevant [options](https://webdriver.io/docs/options.html):
+
+```js
+import { webdriverLauncher } from '@web/test-runner-webdriver';
+
+module.exports = {
+  browsers: [
+    webdriverLauncher({
+      automationProtocol: 'webdriver',
+      path: '/wd/hub/',
+      capabilities: {
+        browserName: 'chrome',
+        'goog:chromeOptions': {
+          args: ['headless', 'disable-gpu']
+        }
+      }
+    })
+    webdriverLauncher({
+      automationProtocol: 'webdriver',
+      path: '/wd/hub/',
+      capabilities: {
+        browserName: 'firefox',
+        'moz:firefoxOptions': {
+          args: ['-headless']
+        }
+      }
+    })
+  ]
+};
+```

packages/test-runner-webdriver/index.d.ts

@@ -0,0 +1,2 @@
+// this file is autogenerated with the generate-mjs-dts-entrypoints script
+export * from './dist/index';

packages/test-runner-webdriver/index.mjs

@@ -0,0 +1,6 @@
+// this file is autogenerated with the generate-mjs-dts-entrypoints script
+import cjsEntrypoint from './dist/index.js';
+
+const { webdriverIOLauncher, WebdriverIOLauncher } = cjsEntrypoint;
+
+export { webdriverIOLauncher, WebdriverIOLauncher };

packages/test-runner-webdriver/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@web/test-runner-webdriver",
+  "version": "0.0.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "Webdriver browser launcher for Web Test Runner",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/modernweb-dev/web.git",
+    "directory": "packages/test-runner-webdriver"
+  },
+  "author": "modern-web",
+  "homepage": "https://github.com/modernweb-dev/web/tree/master/packages/test-runner-webdriver",
+  "main": "dist/index.js",
+  "exports": {
+    ".": {
+      "import": "./index.mjs",
+      "require": "./dist/index.js"
+    }
+  },
+  "engines": {
+    "node": ">=10.0.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "mocha test/**/*.test.ts --require ts-node/register",
+    "test:watch": "mocha test/**/*.test.ts --require ts-node/register --watch --watch-files src,test"
+  },
+  "files": [
+    "*.d.ts",
+    "*.js",
+    "*.mjs",
+    "dist",
+    "src"
+  ],
+  "keywords": [
+    "web",
+    "test",
+    "runner",
+    "testrunner",
+    "webdriver",
+    "webdriverio",
+    "wdio",
+    "browser",
+    "launcher"
+  ],
+  "dependencies": {
+    "@web/test-runner-core": "^0.8.11",
+    "webdriverio": "^6.10.0"
+  },
+  "devDependencies": {
+    "@types/selenium-standalone": "^6.15.2",
+    "selenium-standalone": "^6.23.0"
+  }
+}

packages/test-runner-webdriver/src/IFrameManager.ts

@@ -0,0 +1,172 @@
+import { TestRunnerCoreConfig } from '@web/test-runner-core';
+import { BrowserObject, Element } from 'webdriverio';
+import { validateBrowserResult } from './coverage';
+
+/**
+ * Manages tests to be executed in iframes on a page.
+ */
+export class IFrameManager {
+  private config: TestRunnerCoreConfig;
+  private driver: BrowserObject;
+  private framePerSession = new Map<string, string>();
+  private inactiveFrames: string[] = [];
+  private frameCount = 0;
+  private initialized = false;
+  private initializePromise?: Promise<void>;
+  private locked?: Promise<unknown>;
+  private isIE: boolean;
+
+  constructor(config: TestRunnerCoreConfig, driver: BrowserObject, isIE: boolean) {
+    this.config = config;
+    this.driver = driver;
+    this.isIE = isIE;
+  }
+
+  private async _initialize(url: string) {
+    const pageUrl = `${new URL(url).origin}/?mode=iframe`;
+    await this.driver.navigateTo(pageUrl);
+  }
+
+  isActive(id: string) {
+    return this.framePerSession.has(id);
+  }
+
+  async getBrowserUrl(sessionId: string): Promise<string | undefined> {
+    const frameId = this.getFrameId(sessionId);
+
+    const returnValue = (await this.driver.execute(`
+      try {
+        var iframe = document.getElementById("${frameId}");
+        return iframe.contentWindow.location.href;
+      } catch (_) {
+        return undefined;
+      }
+    `)) as string | undefined;
+
+    return returnValue;
+  }
+
+  private getFrameId(sessionId: string): string {
+    const frameId = this.framePerSession.get(sessionId);
+    if (!frameId) {
+      throw new Error(
+        `Something went wrong while running tests, there is no frame id for session ${sessionId}`,
+      );
+    }
+    return frameId;
+  }
+
+  private async scheduleCommand<T>(fn: () => Promise<T>) {
+    if (!this.isIE) {
+      return fn();
+    }
+
+    while (this.locked) {
+      await this.locked;
+    }
+
+    const fnPromise = fn();
+    this.locked = fnPromise;
+    const result = await fnPromise;
+    this.locked = undefined;
+    return result;
+  }
+
+  async queueStartSession(id: string, url: string) {
+    if (!this.initializePromise && !this.initialized) {
+      this.initializePromise = this._initialize(url);
+    }
+
+    if (this.initializePromise) {
+      await this.initializePromise;
+      this.initializePromise = undefined;
+      this.initialized = true;
+    }
+
+    this.scheduleCommand(() => this.startSession(id, url));
+  }
+
+  private async startSession(id: string, url: string) {
+    let frameId: string;
+    if (this.inactiveFrames.length > 0) {
+      frameId = this.inactiveFrames.pop()!;
+      await this.driver.execute(`
+        var iframe = document.getElementById("${frameId}");
+        iframe.src = "${url}";
+      `);
+    } else {
+      this.frameCount += 1;
+      frameId = `wtr-test-frame-${this.frameCount}`;
+      await this.driver.execute(`
+        var iframe = document.createElement("iframe");
+        iframe.id = "${frameId}";
+        iframe.src = "${url}";
+        document.body.appendChild(iframe);
+      `);
+    }
+
+    this.framePerSession.set(id, frameId);
+  }
+
+  async queueStopSession(id: string) {
+    return this.scheduleCommand(() => this.stopSession(id));
+  }
+
+  async stopSession(id: string) {
+    const frameId = this.getFrameId(id);
+
+    // Retrieve test results from iframe. Note: IIFE is used to prevent
+    // WebdriverIO from crashing failure with Puppeteer (default mode):
+    // Error: Evaluation failed: SyntaxError: Illegal return statement
+    // See https://github.com/webdriverio/webdriverio/pull/4829
+    const returnValue = await this.driver.execute(`
+      return (function() {
+        var iframe = document.getElementById("${frameId}");
+        var testCoverage;
+        try {
+          testCoverage = iframe.contentWindow.__coverage__;
+        } catch (error) {
+          // iframe can throw a cross-origin error if the test navigated
+        }
+
+        // set src after retrieving values to avoid the iframe from navigating away
+        iframe.src = "data:,";
+        return { testCoverage: testCoverage };
+      })();
+    `);
+
+    if (!validateBrowserResult(returnValue)) {
+      throw new Error();
+    }
+
+    const { testCoverage } = returnValue;
+
+    this.inactiveFrames.push(frameId);
+
+    return { testCoverage: this.config.coverage ? testCoverage : undefined };
+  }
+
+  async takeScreenshot(sessionId: string, locator: string): Promise<Buffer> {
+    const frameId = this.getFrameId(sessionId);
+
+    const frame = await this.driver.$(`iframe#${frameId}`);
+
+    await this.driver.switchToFrame(frame);
+
+    const elementData = (await this.driver.execute(locator, [])) as Element;
+
+    const element = await this.driver.$(elementData);
+
+    let base64 = '';
+
+    try {
+      base64 = await this.driver.takeElementScreenshot(element.elementId);
+    } catch (err) {
+      console.log('Failed to take a screenshot:', err);
+    }
+
+    await this.driver.switchToParentFrame();
+
+    return Buffer.from(base64, 'base64');
+  }
+}

packages/test-runner-webdriver/src/SessionManager.ts

@@ -0,0 +1,98 @@
+import { TestRunnerCoreConfig } from '@web/test-runner-core';
+import { BrowserObject, Element } from 'webdriverio';
+import { validateBrowserResult } from './coverage';
+
+/**
+ * Manages tests to be executed in one session (concurrency: 1).
+ */
+export class SessionManager {
+  private config: TestRunnerCoreConfig;
+  private driver: BrowserObject;
+  private locked?: Promise<unknown>;
+  private isIE: boolean;
+  private urlMap = new Map<string, string>();
+
+  constructor(config: TestRunnerCoreConfig, driver: BrowserObject, isIE: boolean) {
+    this.config = config;
+    this.driver = driver;
+    this.isIE = isIE;
+  }
+
+  isActive(id: string) {
+    return this.urlMap.has(id);
+  }
+
+  async getBrowserUrl(id: string): Promise<string | undefined> {
+    return this.urlMap.get(id);
+  }
+
+  private async scheduleCommand<T>(fn: () => Promise<T>) {
+    if (!this.isIE) {
+      return fn();
+    }
+
+    while (this.locked) {
+      await this.locked;
+    }
+
+    const fnPromise = fn();
+    this.locked = fnPromise;
+    const result = await fnPromise;
+    this.locked = undefined;
+    return result;
+  }
+
+  async queueStartSession(id: string, url: string) {
+    this.scheduleCommand(() => this.startSession(id, url));
+  }
+
+  private async startSession(id: string, url: string) {
+    this.urlMap.set(id, url);
+    await this.driver.navigateTo(url);
+  }
+
+  async queueStopSession(id: string) {
+    return this.scheduleCommand(() => this.stopSession(id));
+  }
+
+  async stopSession(id: string) {
+    // Retrieve test results from iframe. Note: IIFE is used to prevent
+    // WebdriverIO from crashing failure with Puppeteer (default mode):
+    // Error: Evaluation failed: SyntaxError: Illegal return statement
+    // See https://github.com/webdriverio/webdriverio/pull/4829
+    const returnValue = await this.driver.execute(`
+      return (function() {
+        return { testCoverage: window.__coverage__ };
+      })();
+    `);
+
+    if (!validateBrowserResult(returnValue)) {
+      throw new Error();
+    }
+
+    const { testCoverage } = returnValue;
+
+    // navigate to an empty page to kill any running code on the page
+    await this.driver.navigateTo('data:,');
+
+    this.urlMap.delete(id);
+
+    return { testCoverage: this.config.coverage ? testCoverage : undefined };
+  }
+
+  async takeScreenshot(_: string, locator: string): Promise<Buffer> {
+    const elementData = (await this.driver.execute(locator, [])) as Element;
+
+    const element = await this.driver.$(elementData);
+
+    let base64 = '';
+
+    try {
+      base64 = await this.driver.takeElementScreenshot(element.elementId);
+    } catch (err) {
+      console.log('Failed to take a screenshot:', err);
+    }
+
+    return Buffer.from(base64, 'base64');
+  }
+}

packages/test-runner-webdriver/src/coverage.ts

@@ -0,0 +1,14 @@
+import { CoverageMapData } from '@web/test-runner-core';
+
+export interface BrowserResult {
+  testCoverage?: CoverageMapData;
+  url: string;
+}
+
+export function validateBrowserResult(result: any): result is BrowserResult {
+  if (typeof result !== 'object') throw new Error('Browser did not return an object');
+  if (result.testCoverage != null && typeof result.testCoverage !== 'object')
+    throw new Error('Browser returned non-object testCoverage');
+
+  return true;
+}