From 05094dcff95837b33485d4614e67f397424841fa Mon Sep 17 00:00:00 2001 From: Lars den Bakker Date: Sun, 6 Sep 2020 08:34:01 +0200 Subject: [PATCH] feat(test-runner-visual-regression): first implementation --- .changeset/fifty-bobcats-grin.md | 5 + .gitignore | 2 + packages/dev-server-cli/package.json | 4 +- packages/dev-server/package.json | 4 +- packages/test-runner-saucelabs/tsconfig.json | 3 + .../test-runner-visual-regression/README.md | 273 ++++++++++++++++++ .../browser/commands.d.ts | 1 + .../browser/commands.mjs | 44 +++ .../test-runner-visual-regression/index.d.ts | 1 + .../test-runner-visual-regression/index.mjs | 6 + .../package.json | 63 ++++ .../Chrome/baseline/my-element.png | Bin 0 -> 1537 bytes .../Chrome/baseline/my-failed-element.png | Bin 0 -> 1537 bytes .../Firefox/baseline/my-element.png | Bin 0 -> 1322 bytes .../Firefox/baseline/my-failed-element.png | Bin 0 -> 1322 bytes .../Webkit/baseline/my-element.png | Bin 0 -> 1848 bytes .../Webkit/baseline/my-failed-element.png | Bin 0 -> 1848 bytes .../src/VisualRegressionError.ts | 1 + .../src/config.ts | 124 ++++++++ .../test-runner-visual-regression/src/fs.ts | 15 + .../src/index.ts | 1 + .../src/pixelMatchDiff.ts | 31 ++ .../src/visualDiffCommand.ts | 78 +++++ .../src/visualRegressionPlugin.ts | 115 ++++++++ .../test/diff-fail-test.js | 10 + .../test/diff-pass-test.js | 10 + .../test/visualRegressionPlugin.test.ts | 100 +++++++ .../tsconfig.json | 46 +++ tsconfig.json | 9 +- workspace-packages.mjs | 1 + yarn.lock | 33 +++ 31 files changed, 975 insertions(+), 5 deletions(-) create mode 100644 .changeset/fifty-bobcats-grin.md create mode 100644 packages/test-runner-visual-regression/README.md create mode 100644 packages/test-runner-visual-regression/browser/commands.d.ts create mode 100644 packages/test-runner-visual-regression/browser/commands.mjs create mode 100644 packages/test-runner-visual-regression/index.d.ts create mode 100644 packages/test-runner-visual-regression/index.mjs create mode 100644 packages/test-runner-visual-regression/package.json create mode 100644 packages/test-runner-visual-regression/screenshots/Chrome/baseline/my-element.png create mode 100644 packages/test-runner-visual-regression/screenshots/Chrome/baseline/my-failed-element.png create mode 100644 packages/test-runner-visual-regression/screenshots/Firefox/baseline/my-element.png create mode 100644 packages/test-runner-visual-regression/screenshots/Firefox/baseline/my-failed-element.png create mode 100644 packages/test-runner-visual-regression/screenshots/Webkit/baseline/my-element.png create mode 100644 packages/test-runner-visual-regression/screenshots/Webkit/baseline/my-failed-element.png create mode 100644 packages/test-runner-visual-regression/src/VisualRegressionError.ts create mode 100644 packages/test-runner-visual-regression/src/config.ts create mode 100644 packages/test-runner-visual-regression/src/fs.ts create mode 100644 packages/test-runner-visual-regression/src/index.ts create mode 100644 packages/test-runner-visual-regression/src/pixelMatchDiff.ts create mode 100644 packages/test-runner-visual-regression/src/visualDiffCommand.ts create mode 100644 packages/test-runner-visual-regression/src/visualRegressionPlugin.ts create mode 100644 packages/test-runner-visual-regression/test/diff-fail-test.js create mode 100644 packages/test-runner-visual-regression/test/diff-pass-test.js create mode 100644 packages/test-runner-visual-regression/test/visualRegressionPlugin.test.ts create mode 100644 packages/test-runner-visual-regression/tsconfig.json diff --git a/.changeset/fifty-bobcats-grin.md b/.changeset/fifty-bobcats-grin.md new file mode 100644 index 000000000..b8e8b22fe --- /dev/null +++ b/.changeset/fifty-bobcats-grin.md @@ -0,0 +1,5 @@ +--- +'@web/test-runner-visual-regression': patch +--- + +first implementation diff --git a/.gitignore b/.gitignore index c17b57099..8f6c5528e 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,5 @@ _site stats.html local.log + +packages/*/screenshots/*/failed \ No newline at end of file diff --git a/packages/dev-server-cli/package.json b/packages/dev-server-cli/package.json index ee0067384..1c86b121e 100644 --- a/packages/dev-server-cli/package.json +++ b/packages/dev-server-cli/package.json @@ -22,7 +22,9 @@ "test": "mocha \"test/**/*.test.ts\" --require ts-node/register --reporter dot", "test:watch": "mocha \"test/**/*.test.ts\" --require ts-node/register --watch --watch-files src,test --reporter dot" }, - "files": ["dist"], + "files": [ + "dist" + ], "keywords": [ "web", "dev", diff --git a/packages/dev-server/package.json b/packages/dev-server/package.json index 0572de2bf..07511e82c 100644 --- a/packages/dev-server/package.json +++ b/packages/dev-server/package.json @@ -33,7 +33,9 @@ "test": "mocha \"test/**/*.test.mjs\" --reporter dot", "test:watch": "mocha \"test/**/*.test.mjs\" --watch --watch-files src,test --reporter dot" }, - "files": ["dist"], + "files": [ + "dist" + ], "keywords": [ "web", "dev", diff --git a/packages/test-runner-saucelabs/tsconfig.json b/packages/test-runner-saucelabs/tsconfig.json index b6ade14c0..b1e8b4147 100644 --- a/packages/test-runner-saucelabs/tsconfig.json +++ b/packages/test-runner-saucelabs/tsconfig.json @@ -16,6 +16,9 @@ { "path": "../test-runner-core/tsconfig.json" }, + { + "path": "../dev-server-esbuild/tsconfig.json" + }, { "path": "../test-runner-selenium/tsconfig.json" }, diff --git a/packages/test-runner-visual-regression/README.md b/packages/test-runner-visual-regression/README.md new file mode 100644 index 000000000..f78397328 --- /dev/null +++ b/packages/test-runner-visual-regression/README.md @@ -0,0 +1,273 @@ +# Test Runner Visual Regression + +Plugin for visual regression testing with Web Test Runner. + +> This project is experimental. We are testing out different approaches and gathering feedback, let us know what you think and [join the discussion!](https://github.com/modernweb-dev/web/discussions/427). + +## Usage + +Install the package: + +``` +npm i --save-dev @web/test-runner-visual-regression +``` + +Add the plugin to you `web-test-runner.config.mjs`: + +```js +import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; + +export default { + plugins: [ + visualRegressionPlugin({ + update: process.argv.includes('--update-visual-baseline'), + }), + ], +}; +``` + +Run a visual diff in your browser test: + +```js +import { visualDiff } from '@web/test-runner-visual-regression'; + +it('can diff an element', async () => { + const element = document.createElement('p'); + element.textContent = 'Hello world'; + element.style.color = 'blue'; + document.body.appendChild(element); + + await visualDiff(element, 'my-element'); +}); +``` + +## How it works + +### Element + +You call the `visualDiff` function with a reference to a DOM element. The element must be connected to the DOM, in the same document the tests are run in. You can also take a screenshot of the whole body for a full-page screenshot by passing in `document.body`. The element can also be in shadow DOM. + +### Diffing + +When you run a diff test for the first time, a baseline image is saved to `screenshots/baseline/${browser}/${name}.png`. Afterward, every time you do a diff it is compared to this baseline images. + +If the difference between the two images is larger than the configured threshold, the test fails. The failed screenshot is saved as `screenshots/${browser}/failed/${name}.png` and an image illustrating the differences between the two images is saved as `screenshots/${browser}/failed/${name}-diff.png`. + +### Updating diffs + +When tests are run with the `update` option set to `true`, the new image will be saved as a baseline and the test will always pass. + +In the example config, we read the command line args for a `--update-visual-baseline` flag, you can use this when running tests: + +``` +web-test-runner test/**/*.test.js --update-visual-baseline +``` + +## Configuration + +These are all the possible configuration options. All options are optional: + +```ts +import pixelmatch from 'pixelmatch'; + +type PixelMatchParams = Parameters; +type PixelMatchOptions = PixelMatchParams[5]; + +export interface GetNameArgs { + browser: string; + name: string; +} + +export interface ImageArgs { + filePath: string; + baseDir: string; + name: string; +} + +export interface SaveImageArgs extends ImageArgs { + content: Buffer; +} + +export type OptionalImage = Buffer | undefined | Promise; + +export interface DiffResult { + diffPercentage: number; + diffImage: Buffer; +} + +export interface DiffArgs { + name: string; + baselineImage: Buffer; + image: Buffer; + options: PixelMatchOptions; +} + +export interface VisualRegressionPluginOptions { + /** + * Whether to update the baseline image instead of comparing + * the image with the current baseline. + */ + update: boolean; + /** + * The base directory to write images to. + */ + baseDir: string; + /** + * Options to use when diffing images. + */ + diffOptions: PixelMatchOptions; + + /** + * Returns the name of the baseline image file. The name + * is a path relative to the baseDir + */ + getBaselineName: (args: GetNameArgs) => string; + /** + * Returns the name of the image file representing the difference + * between the baseline and the new image. The name is a path + * relative to the baseDir + */ + getDiffName: (args: GetNameArgs) => string; + /** + * Returns the name of the failed image file. The name is a path + * relative to the baseDir + */ + getFailedName: (args: GetNameArgs) => string; + + /** + * Returns the baseline image. + */ + getBaseline: (args: ImageArgs) => OptionalImage; + /** + * Saves the baseline image. + */ + saveBaseline: (args: SaveImageArgs) => void | Promise; + + /** + * Saves the image representing the difference between the + * baseline and the new image. + */ + saveDiff: (args: SaveImageArgs) => void | Promise; + /** + * Saves the failed image file. + */ + saveFailed: (args: SaveImageArgs) => void | Promise; + + /** + * Gets the difference between two images. + */ + getImageDiff: (args: DiffArgs) => DiffResult | Promise; +} +``` + +### Diffing options + +We use the [pixelmatch](https://www.npmjs.com/package/pixelmatch) library for diffing images. You can configure the diffing options in the config. + +```js +import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; +import path from 'path'; + +export default { + plugins: [ + visualRegressionPlugin({ + diffOptions: { + threshold: 0.2, + includeAA: false, + }, + }), + ], +}; +``` + +### Names and directories + +By default images are saved to disk to the `screenshots/${browser}/baseline` and `screenshots/${browser}/failed` directories. You can configure different directories or name patterns. + +```js +import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; +import path from 'path'; + +export default { + plugins: [ + visualRegressionPlugin({ + update: process.argv.includes('--update-visual-baseline'), + // configure the directory to output screenshots into + // can also be an absolute path + baseDir: 'screenshots', + + // configure the path relative to the basedir where to store individual screenshots + // this can be used to configure different directories, or to change the names + getBaselineName: ({ browser, name }) => path.join(browser, 'baseline', name), + getDiffName: ({ browser, name }) => path.join(browser, 'failed', `${name}-diff`), + getFailedName: ({ browser, name }) => path.join(browser, 'failed', name), + }), + ], +}; +``` + +### Storing images externally + +By default, we write files on disk, but you can configure this behavior from the config. This way you can, for example, upload images to an external service. + +```js +import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; +import path from 'path'; + +export default { + plugins: [ + visualRegressionPlugin({ + update: process.argv.includes('--update-visual-baseline'), + + getBaseline({ filePath, baseDir, name }) { + // read the baseline image from somewhere. this function can be async, and should + // return a Buffer with the image data + }, + saveBaseline({ filePath, content, baseDir, name }) { + // save the baseline image somewhere. this function can be async. + }, + saveDiff({ filePath, content, baseDir, name }) { + // save the diff image somewhere. this function can be async. + }, + saveFailed({ filePath, content, baseDir, name }) { + // save the failed image somewhere. this function can be async. + }, + }), + ], +}; +``` + +### Custom diffing + +You can implement custom image diffing logic in the config. Use this to implement different diffing libraries. + +```js +import { visualRegressionPlugin } from '@web/test-runner-visual-regression/plugin'; +import path from 'path'; + +export default { + plugins: [ + visualRegressionPlugin({ + update: process.argv.includes('--update-visual-baseline'), + + getImageDiff({ + options: VisualRegressionPluginOptions, + image: Buffer, + browser: string, + name: string, + }) { + // read the baseline image from somewhere. this function can be async, and should + // return a Buffer with the image data + + return { + // return the diff percentage as a number between 0 and 100 + diffPercentage, + // return the image representing the diff between the two images + // this helps the user with debugging + diffImage, + }; + }, + }), + ], +}; +``` diff --git a/packages/test-runner-visual-regression/browser/commands.d.ts b/packages/test-runner-visual-regression/browser/commands.d.ts new file mode 100644 index 000000000..b3cf3a317 --- /dev/null +++ b/packages/test-runner-visual-regression/browser/commands.d.ts @@ -0,0 +1 @@ +export function visualDiff(element: Node, name: string): Promise; diff --git a/packages/test-runner-visual-regression/browser/commands.mjs b/packages/test-runner-visual-regression/browser/commands.mjs new file mode 100644 index 000000000..a4b5d0d54 --- /dev/null +++ b/packages/test-runner-visual-regression/browser/commands.mjs @@ -0,0 +1,44 @@ +import { executeServerCommand } from '@web/test-runner-commands'; + +let i = 0; + +const elements = {}; +window.__WTR_VISUAL_REGRESSION__ = elements; + +export async function visualDiff(element, name) { + if (!(element instanceof Node)) { + throw new Error('Element to diff must be a Node.'); + } + + if (!element.isConnected) { + throw new Error('Element must be connected to the DOM.'); + } + + if (element.ownerDocument !== document) { + throw new Error('Element must belong to the same document the tests are run in.'); + } + + if (typeof name !== 'string') { + throw new Error('You must provide a name to diff'); + } + + i += 1; + elements[i] = element; + try { + const result = await executeServerCommand('visual-diff', { id: String(i), name }); + if (!result) { + throw new Error('Failed to execute visual diff.'); + } + + if (result.passed) { + return; + } + + if (typeof result.errorMessage === 'string') { + throw new Error(result.errorMessage); + } + throw new Error('Failed to execute visual diff.'); + } finally { + delete elements[i]; + } +} diff --git a/packages/test-runner-visual-regression/index.d.ts b/packages/test-runner-visual-regression/index.d.ts new file mode 100644 index 000000000..2b8395cdb --- /dev/null +++ b/packages/test-runner-visual-regression/index.d.ts @@ -0,0 +1 @@ +export * from './dist/index.js'; diff --git a/packages/test-runner-visual-regression/index.mjs b/packages/test-runner-visual-regression/index.mjs new file mode 100644 index 000000000..8dece0ddf --- /dev/null +++ b/packages/test-runner-visual-regression/index.mjs @@ -0,0 +1,6 @@ +// this file is autogenerated with the update-esm-entrypoints script +import cjsEntrypoint from './dist/index.js'; + +const { visualRegressionPlugin } = cjsEntrypoint; + +export { visualRegressionPlugin }; diff --git a/packages/test-runner-visual-regression/package.json b/packages/test-runner-visual-regression/package.json new file mode 100644 index 000000000..bb11aeeb6 --- /dev/null +++ b/packages/test-runner-visual-regression/package.json @@ -0,0 +1,63 @@ +{ + "name": "@web/test-runner-visual-regression", + "version": "0.1.3", + "publishConfig": { + "access": "public" + }, + "description": "Web test runner visual regression", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/modernweb-dev/web.git", + "directory": "packages/test-runner-visual-regression" + }, + "author": "modern-web", + "homepage": "https://github.com/modernweb-dev/web/tree/master/packages/test-runner-visual-regression", + "main": "browser/commands.mjs", + "engines": { + "node": ">=10.0.0" + }, + "scripts": { + "build": "tsc", + "test": "mocha test/**/*.test.ts --require ts-node/register", + "test:watch": "mocha test/**/*.test.ts --require ts-node/register --watch --watch-files src,test" + }, + "files": [ + "*.d.ts", + "*.js", + "*.mjs", + "dist", + "src" + ], + "keywords": [ + "web", + "test", + "runner", + "testrunner", + "visual-regression" + ], + "dependencies": { + "@types/mkdirp": "^1.0.1", + "@types/pixelmatch": "^5.2.1", + "@types/pngjs": "^3.4.2", + "@web/test-runner-commands": "^0.1.3", + "@web/test-runner-core": "^0.7.4", + "mkdirp": "^1.0.4", + "pixelmatch": "^5.2.1", + "pngjs": "^5.0.0" + }, + "devDependencies": { + "@web/test-runner-chrome": "^0.6.4", + "@web/test-runner-playwright": "^0.5.4", + "mocha": "^8.1.1" + }, + "exports": { + ".": { + "import": "./browser/commands.mjs" + }, + "./plugins": { + "import": "./index.mjs", + "require": "./dist/index.js" + } + } +} diff --git a/packages/test-runner-visual-regression/screenshots/Chrome/baseline/my-element.png b/packages/test-runner-visual-regression/screenshots/Chrome/baseline/my-element.png new file mode 100644 index 0000000000000000000000000000000000000000..1609d7a5ba430c87b89b48bbc96681a7b790fb42 GIT binary patch literal 1537 zcmV+c2LAbpP)Px)yGcYrRCt{2-FZkAT^I)N=h~*NR+N=#5mADqMU*XCwvg?RMMi;zn5jun5=DfG zMP(pVP!uMG7Hx!W7MW>QP!MDWrBR8I8C^5VEpxly9}k!D=5`Z5zrK8aa5CKNhdlr~5fxm%> zh=_3g!-MfLkdz;_Cb0>qVva-&dmxzdnh{&JB0Wfat`}S=*V8{>#5%zL&vhDZm z+1{yju73Sm-|y(yIr9(^5fKsjleMb`A~hB8^#xp9nELbfZOda#rl#8-JpzJ)m@?(- z+j8nqMg}lz7C)y>0omEW!iDSy2Ln}Az>_E3wsa|vsjcN|!GZ-`If;zqH~`4X;tEM{ zFc25Vz$hc5`D@qJ0Xud8lP3d<7Xx?h*#G;8h=_=Y{H?k-?!0>kaz))V!dWMM;Ikq{G zk-(cbEUd1sW}S$Lh=_T^9G+_8};PPerYXkP} z113!ZJUo~d6c@+zp*?$=A0Hmh_SP-7D^{?*ew}SW0pRY=V(t8V;KT_aB!v6Cy@Bo9 z8Jyg|&s7gkPX-+c3BZE~z@kM>ubH3EdXS(X1}TFE0jpNo_x(gfL_|dXW;QDjAtAt; zHMRq+H|kb7IZOi@Igg z`ZR-)ix-)a1TdWm0EP`Sj`#C3=I!qfbnC`*HkVpuWqq8hzrV4~+1bADCn6#uBJwx; z?HUNPneTS+_cuN>XEG3IOKmNWfBxKd`-l--g=p!fe*Jm|2L%Our^MB(JkQ;`3~&Mh zIPT*|wt0ERu`Vube=Y8AO>wdDT}4DhL`1}36F^ujO$XG}0QL1o-^+_d)QwY?Vq^K} z*0%KM0gM|5JbMOIRvO#O%el(XQUI}hIopjJ0UsY=*)pzxgoOc9rU1QqaleNL+ow;B zW3AVjI<@WjS-H8f&CIm_7!eT>5s|LI0pN-P0DS*$yR*I?C@Qku9u~&d&ks0rh6U8c z#el0Tw^vr$1{c;Fc}Mcn7$Mh)$$~roq@G$c|#u&!8E7PP|j`LI%C|D zB`o+pf8MxmS{j2BFE93MYJk$xrq?JhH;!4jkZn>DS3Qc0Suasu&cf=(0hNe|h=@p6 z(#_R1HZ~@PH}bi;TqW@FVH!|a7@uHNTFM~9(~~JnW;2VLuU+HciHX3#fxx+Qz<~om zNeNdooSax3T~Y#k_`t`n`uXt&JUW`!*t-{qi2+uxZhfMTj=X{2yqSGhR|W_1@y5zU z_wHP+NJ-)6#tq=b3*gEXzT+Y;j&m$1;QXvlecHCoNRe`P2c}PF0d`^{aP%nOL_J^t z@a-GxG-l7{?<683A|le&m@Jm&5|`RmS_*vm0*oEY!sOpoRmGsf-s9`)__)@_kCZJc zGK#r>RanTniot__e{NPke(*eF#_;?0@8A5(_C!QPL`1rZb_Wn5A|fIp()G0KP1GVH nA|fKvH7S4)5fKp)5xe~Zczw%vHA`pg00000NkvXXu0mjfV`|%% literal 0 HcmV?d00001 diff --git a/packages/test-runner-visual-regression/screenshots/Chrome/baseline/my-failed-element.png b/packages/test-runner-visual-regression/screenshots/Chrome/baseline/my-failed-element.png new file mode 100644 index 0000000000000000000000000000000000000000..1609d7a5ba430c87b89b48bbc96681a7b790fb42 GIT binary patch literal 1537 zcmV+c2LAbpP)Px)yGcYrRCt{2-FZkAT^I)N=h~*NR+N=#5mADqMU*XCwvg?RMMi;zn5jun5=DfG zMP(pVP!uMG7Hx!W7MW>QP!MDWrBR8I8C^5VEpxly9}k!D=5`Z5zrK8aa5CKNhdlr~5fxm%> zh=_3g!-MfLkdz;_Cb0>qVva-&dmxzdnh{&JB0Wfat`}S=*V8{>#5%zL&vhDZm z+1{yju73Sm-|y(yIr9(^5fKsjleMb`A~hB8^#xp9nELbfZOda#rl#8-JpzJ)m@?(- z+j8nqMg}lz7C)y>0omEW!iDSy2Ln}Az>_E3wsa|vsjcN|!GZ-`If;zqH~`4X;tEM{ zFc25Vz$hc5`D@qJ0Xud8lP3d<7Xx?h*#G;8h=_=Y{H?k-?!0>kaz))V!dWMM;Ikq{G zk-(cbEUd1sW}S$Lh=_T^9G+_8};PPerYXkP} z113!ZJUo~d6c@+zp*?$=A0Hmh_SP-7D^{?*ew}SW0pRY=V(t8V;KT_aB!v6Cy@Bo9 z8Jyg|&s7gkPX-+c3BZE~z@kM>ubH3EdXS(X1}TFE0jpNo_x(gfL_|dXW;QDjAtAt; zHMRq+H|kb7IZOi@Igg z`ZR-)ix-)a1TdWm0EP`Sj`#C3=I!qfbnC`*HkVpuWqq8hzrV4~+1bADCn6#uBJwx; z?HUNPneTS+_cuN>XEG3IOKmNWfBxKd`-l--g=p!fe*Jm|2L%Our^MB(JkQ;`3~&Mh zIPT*|wt0ERu`Vube=Y8AO>wdDT}4DhL`1}36F^ujO$XG}0QL1o-^+_d)QwY?Vq^K} z*0%KM0gM|5JbMOIRvO#O%el(XQUI}hIopjJ0UsY=*)pzxgoOc9rU1QqaleNL+ow;B zW3AVjI<@WjS-H8f&CIm_7!eT>5s|LI0pN-P0DS*$yR*I?C@Qku9u~&d&ks0rh6U8c z#el0Tw^vr$1{c;Fc}Mcn7$Mh)$$~roq@G$c|#u&!8E7PP|j`LI%C|D zB`o+pf8MxmS{j2BFE93MYJk$xrq?JhH;!4jkZn>DS3Qc0Suasu&cf=(0hNe|h=@p6 z(#_R1HZ~@PH}bi;TqW@FVH!|a7@uHNTFM~9(~~JnW;2VLuU+HciHX3#fxx+Qz<~om zNeNdooSax3T~Y#k_`t`n`uXt&JUW`!*t-{qi2+uxZhfMTj=X{2yqSGhR|W_1@y5zU z_wHP+NJ-)6#tq=b3*gEXzT+Y;j&m$1;QXvlecHCoNRe`P2c}PF0d`^{aP%nOL_J^t z@a-GxG-l7{?<683A|le&m@Jm&5|`RmS_*vm0*oEY!sOpoRmGsf-s9`)__)@_kCZJc zGK#r>RanTniot__e{NPke(*eF#_;?0@8A5(_C!QPL`1rZb_Wn5A|fIp()G0KP1GVH nA|fKvH7S4)5fKp)5xe~Zczw%vHA`pg00000NkvXXu0mjfV`|%% literal 0 HcmV?d00001 diff --git a/packages/test-runner-visual-regression/screenshots/Firefox/baseline/my-element.png b/packages/test-runner-visual-regression/screenshots/Firefox/baseline/my-element.png new file mode 100644 index 0000000000000000000000000000000000000000..b14878d00c6c917478698fad71a528e5d3080afd GIT binary patch literal 1322 zcmV+_1=aeAP)NklchKD(zvM`Ly`{E@!Te;~3&^s`m>9@44sPGiO|#&)jq7dIA6d z0DvNS|Mvg@0002|IhH*@Oif86Bd$C>E#1Do-0!md#>b@>FBYy(r==%PR&G21002NK z^18aDvNFl@>~BwxRXl$#H8j|H&yzN7k|rh=?kJs>cJH>f_4RhHzhClwJ6EbECZvrU zZHz~c=5KT9lKpQgD_3@G0000$sqrK!9=8efsZ-&~>1jLf`%)%TT!*!_Egtsn6951JP>MWBI&;P*&=)VxZ{Yh&`{sT1>TrNKc(C*WL^d1# zn@%qrAcEi&jY^To%8U;H001aeOAZj(taRdp-D&WBskvGD{MpqP6Cf^JkXEg-K5A;D zYuEA*Zel{JsVu^R90Gf+culrkB&;^`xYW{O$Jei0 zeNdJb4-f+bHgSLV&dSG*S^eN({sEpoZS|w0Rz82;$`2n}Ih(cdrBc%O?{>U@ztv4m z+1`an#6rTkb8b&ZM~Ad_tu#D5KS0dR+H$3M+{zz6T0H>*00030x=?_qtFv$Rb6>du zVrIrB|7~qjU!T2#&DVj&0z_Gvv~#B`fBQB+q~uB_t^U9PD<={z>`3;DPtDDC9E>NC zaMvF?WaqA2v2sO)jWO5U0CD-Um2-cUX~PBr1ONa4u%snSV`J8?uTQ#h!@X|abnOcT zh&ONSxU)0-9(8qger6{BAe)+O2~j2^_4c~Zl1@vpm~{WXwQFgyx^&uId+Sy>fK*r8 zGNxQ}14L`9eePh7gv1^Q0000$T4I2B@W9#)4=>zdp#U*CX~#`X;cJ44|CcZM`wT+B zojW#(9~^Yco+1%9(T~OE-#G|EOMAPOiwO{$H(UABC)Xw#wf!0Z006+h%#+-=`uPGx z?((9yZ>=1ShTC*^yGeVY0FlX9Ajw_wBuVSm+0vqg%+A^!i1Km^8^OK@-n`tiepKyPp1fp&C+?=Ez8&EM~kOj=*{_3r0a zySpuT1k0l;DlD+%3ieq9o3EcevvN(1ZT6m=v_5=adi=QXH~|0vfWMg+kGrQumz7B; zPg=#;n6ze%3lI$r((BhYnT|xl=aWghE6~>F&d1|+Cm?vrbu=n{_+WvdwbjmdcG~sr z?RJ-;kXNsy_3K?Zk+6-;b3J-wA)&F+#t@0vXUSxwUAx@)_U*GVClc1y^DHD>y=wjR z_q(>0mDW#ft#tP6vi#ox004kL(r-WFEs?OFN&Q`^l=Spz_!$z5ncH@uJrkLXG(Il9 zd>P&dy;SM6`;4j7@Am@$008_gO6(cY00000N*MtH0000$(FhO#0001rMt}eS002-l g0t5g60Dz+X0tXTPHpeAaOaK4?07*qoM6N<$f}ZSf3IG5A literal 0 HcmV?d00001 diff --git a/packages/test-runner-visual-regression/screenshots/Firefox/baseline/my-failed-element.png b/packages/test-runner-visual-regression/screenshots/Firefox/baseline/my-failed-element.png new file mode 100644 index 0000000000000000000000000000000000000000..b14878d00c6c917478698fad71a528e5d3080afd GIT binary patch literal 1322 zcmV+_1=aeAP)NklchKD(zvM`Ly`{E@!Te;~3&^s`m>9@44sPGiO|#&)jq7dIA6d z0DvNS|Mvg@0002|IhH*@Oif86Bd$C>E#1Do-0!md#>b@>FBYy(r==%PR&G21002NK z^18aDvNFl@>~BwxRXl$#H8j|H&yzN7k|rh=?kJs>cJH>f_4RhHzhClwJ6EbECZvrU zZHz~c=5KT9lKpQgD_3@G0000$sqrK!9=8efsZ-&~>1jLf`%)%TT!*!_Egtsn6951JP>MWBI&;P*&=)VxZ{Yh&`{sT1>TrNKc(C*WL^d1# zn@%qrAcEi&jY^To%8U;H001aeOAZj(taRdp-D&WBskvGD{MpqP6Cf^JkXEg-K5A;D zYuEA*Zel{JsVu^R90Gf+culrkB&;^`xYW{O$Jei0 zeNdJb4-f+bHgSLV&dSG*S^eN({sEpoZS|w0Rz82;$`2n}Ih(cdrBc%O?{>U@ztv4m z+1`an#6rTkb8b&ZM~Ad_tu#D5KS0dR+H$3M+{zz6T0H>*00030x=?_qtFv$Rb6>du zVrIrB|7~qjU!T2#&DVj&0z_Gvv~#B`fBQB+q~uB_t^U9PD<={z>`3;DPtDDC9E>NC zaMvF?WaqA2v2sO)jWO5U0CD-Um2-cUX~PBr1ONa4u%snSV`J8?uTQ#h!@X|abnOcT zh&ONSxU)0-9(8qger6{BAe)+O2~j2^_4c~Zl1@vpm~{WXwQFgyx^&uId+Sy>fK*r8 zGNxQ}14L`9eePh7gv1^Q0000$T4I2B@W9#)4=>zdp#U*CX~#`X;cJ44|CcZM`wT+B zojW#(9~^Yco+1%9(T~OE-#G|EOMAPOiwO{$H(UABC)Xw#wf!0Z006+h%#+-=`uPGx z?((9yZ>=1ShTC*^yGeVY0FlX9Ajw_wBuVSm+0vqg%+A^!i1Km^8^OK@-n`tiepKyPp1fp&C+?=Ez8&EM~kOj=*{_3r0a zySpuT1k0l;DlD+%3ieq9o3EcevvN(1ZT6m=v_5=adi=QXH~|0vfWMg+kGrQumz7B; zPg=#;n6ze%3lI$r((BhYnT|xl=aWghE6~>F&d1|+Cm?vrbu=n{_+WvdwbjmdcG~sr z?RJ-;kXNsy_3K?Zk+6-;b3J-wA)&F+#t@0vXUSxwUAx@)_U*GVClc1y^DHD>y=wjR z_q(>0mDW#ft#tP6vi#ox004kL(r-WFEs?OFN&Q`^l=Spz_!$z5ncH@uJrkLXG(Il9 zd>P&dy;SM6`;4j7@Am@$008_gO6(cY00000N*MtH0000$(FhO#0001rMt}eS002-l g0t5g60Dz+X0tXTPHpeAaOaK4?07*qoM6N<$f}ZSf3IG5A literal 0 HcmV?d00001 diff --git a/packages/test-runner-visual-regression/screenshots/Webkit/baseline/my-element.png b/packages/test-runner-visual-regression/screenshots/Webkit/baseline/my-element.png new file mode 100644 index 0000000000000000000000000000000000000000..70540398bb875d91b8059579a10a6fb340c050ec GIT binary patch literal 1848 zcmZuydo8aquRz?>M=qoCJ~RuYt*BiA*-}VN2=<1B_mP2Ri&df z+D60;A#Q7m5?Vz?t2!PHDwwHPTGt~kCZrWgkP$k?eQB82?LOjnCiNFz+jQ~K` zVgTrti%L)`QH66r07RuhTWcKfU%L<;4&*Nz(A{z@(^E%8-ow|UyiIjO>l zy|~1>spHEg%J|;jTZ}w>)x7S)SLQ+FVc|YM zmYraEs%>*=lF`q-5(``SdMoEH3-vx^R~J~_VEO+fWj+slRBF*2P>!bG{)%z-C4G5{ zfs2oS1f}DVjF1>ii38MfOMqbc7j%S3BRmi!}T-hkUB3 zw}&6qK&ekFj`1x96zh;2jHp`sOC$L4bPDa776_sSK1cxT9Rinof!0?-!&g&QE&CcC zmWL z#%mTM;uZa9@X3SyB@n@cu^mO+ln4oH0Y6OueOj?q6Hj@QmLrXbZ1_QQ2pkX{?c0C< zlm@5a-RdRrZ%Dcpz%ELY)7g#;A`5U2KFtTRcWWzy#2cv`f;MT&ODuLJwPz`B`rklg z(Lba8mWzPFS!F6Ea^)&l<5#zKXLn>!wH*ywvM|O%Ik>9@{c7o7>m6I@G&26p#jp>K z=A`ov`ZpCr6ZMts#~0;7u6t;9|IO&&>)Yc;M^b5m*M-1cF>p&ZC)QS2Z9;-xPjZ^HU4nU=?#v&4%ZMjd-A+ z`=`k%DiAM znwg|{%1(_es4yea2|qVCNDYo+Ul*S90S0d0!GL%^^){99Qzv(~MH%iXu?l#e{3e8$ zJ^gc8i1&wEhtG&k9;X_PRR?W51cc18Wnpe z>p5e?TO5xOyyLx_?Y|rj3NO7xsXCHhpK>`t#)~c-7DdG=7HJi}X|kl=Im`}bmk#6= z7Zfl%wFQ^B)P+LDa0@L>{2o7&FmRtE6A({~gtD_t3b=L|gE{vO@Z(By|7oY0vhjOn zPc90CVRfk!O`L%Q{gBw}Mn64ElLS;&;Ru;VA6>7+clh+%b~mmp#!k-GNR13OtG}-+ z)t&hb(b> zPjNC{xzk|f-Snlp8^u!Z(VRE`PJZ}qO%nFGajIIwW)OHS9=$pAl03enU!%dT1oVBz xYD@5J%#Om*VLZ{vgCOs=@`n!ZV3G1>aHJn=rbDmPNbe*wIM7(@U7 literal 0 HcmV?d00001 diff --git a/packages/test-runner-visual-regression/screenshots/Webkit/baseline/my-failed-element.png b/packages/test-runner-visual-regression/screenshots/Webkit/baseline/my-failed-element.png new file mode 100644 index 0000000000000000000000000000000000000000..70540398bb875d91b8059579a10a6fb340c050ec GIT binary patch literal 1848 zcmZuydo8aquRz?>M=qoCJ~RuYt*BiA*-}VN2=<1B_mP2Ri&df z+D60;A#Q7m5?Vz?t2!PHDwwHPTGt~kCZrWgkP$k?eQB82?LOjnCiNFz+jQ~K` zVgTrti%L)`QH66r07RuhTWcKfU%L<;4&*Nz(A{z@(^E%8-ow|UyiIjO>l zy|~1>spHEg%J|;jTZ}w>)x7S)SLQ+FVc|YM zmYraEs%>*=lF`q-5(``SdMoEH3-vx^R~J~_VEO+fWj+slRBF*2P>!bG{)%z-C4G5{ zfs2oS1f}DVjF1>ii38MfOMqbc7j%S3BRmi!}T-hkUB3 zw}&6qK&ekFj`1x96zh;2jHp`sOC$L4bPDa776_sSK1cxT9Rinof!0?-!&g&QE&CcC zmWL z#%mTM;uZa9@X3SyB@n@cu^mO+ln4oH0Y6OueOj?q6Hj@QmLrXbZ1_QQ2pkX{?c0C< zlm@5a-RdRrZ%Dcpz%ELY)7g#;A`5U2KFtTRcWWzy#2cv`f;MT&ODuLJwPz`B`rklg z(Lba8mWzPFS!F6Ea^)&l<5#zKXLn>!wH*ywvM|O%Ik>9@{c7o7>m6I@G&26p#jp>K z=A`ov`ZpCr6ZMts#~0;7u6t;9|IO&&>)Yc;M^b5m*M-1cF>p&ZC)QS2Z9;-xPjZ^HU4nU=?#v&4%ZMjd-A+ z`=`k%DiAM znwg|{%1(_es4yea2|qVCNDYo+Ul*S90S0d0!GL%^^){99Qzv(~MH%iXu?l#e{3e8$ zJ^gc8i1&wEhtG&k9;X_PRR?W51cc18Wnpe z>p5e?TO5xOyyLx_?Y|rj3NO7xsXCHhpK>`t#)~c-7DdG=7HJi}X|kl=Im`}bmk#6= z7Zfl%wFQ^B)P+LDa0@L>{2o7&FmRtE6A({~gtD_t3b=L|gE{vO@Z(By|7oY0vhjOn zPc90CVRfk!O`L%Q{gBw}Mn64ElLS;&;Ru;VA6>7+clh+%b~mmp#!k-GNR13OtG}-+ z)t&hb(b> zPjNC{xzk|f-Snlp8^u!Z(VRE`PJZ}qO%nFGajIIwW)OHS9=$pAl03enU!%dT1oVBz xYD@5J%#Om*VLZ{vgCOs=@`n!ZV3G1>aHJn=rbDmPNbe*wIM7(@U7 literal 0 HcmV?d00001 diff --git a/packages/test-runner-visual-regression/src/VisualRegressionError.ts b/packages/test-runner-visual-regression/src/VisualRegressionError.ts new file mode 100644 index 000000000..ec11b981a --- /dev/null +++ b/packages/test-runner-visual-regression/src/VisualRegressionError.ts @@ -0,0 +1 @@ +export class VisualRegressionError extends Error {} diff --git a/packages/test-runner-visual-regression/src/config.ts b/packages/test-runner-visual-regression/src/config.ts new file mode 100644 index 000000000..e7eecda0d --- /dev/null +++ b/packages/test-runner-visual-regression/src/config.ts @@ -0,0 +1,124 @@ +import path from 'path'; +import mkdirp from 'mkdirp'; +import pixelmatch from 'pixelmatch'; + +import { readFile, writeFile, fileExists } from './fs'; +import { pixelMatchDiff } from './pixelMatchDiff'; + +type PixelMatchParams = Parameters; +type PixelMatchOptions = PixelMatchParams[5]; + +export interface GetNameArgs { + browser: string; + name: string; +} + +export interface ImageArgs { + filePath: string; + baseDir: string; + name: string; +} + +export interface SaveImageArgs extends ImageArgs { + content: Buffer; +} + +export type OptionalImage = Buffer | undefined | Promise; + +export interface DiffResult { + diffPercentage: number; + diffImage: Buffer; +} + +export interface DiffArgs { + name: string; + baselineImage: Buffer; + image: Buffer; + options: PixelMatchOptions; +} + +export interface VisualRegressionPluginOptions { + /** + * Whether to update the baseline image instead of comparing + * the image with the current baseline. + */ + update: boolean; + /** + * The base directory to write images to. + */ + baseDir: string; + /** + * Options to use when diffing images. + */ + diffOptions: PixelMatchOptions; + + /** + * Returns the name of the baseline image file. The name + * is a path relative to the baseDir + */ + getBaselineName: (args: GetNameArgs) => string; + /** + * Returns the name of the image file representing the difference + * between the baseline and the new image. The name is a path + * relative to the baseDir + */ + getDiffName: (args: GetNameArgs) => string; + /** + * Returns the name of the failed image file. The name is a path + * relative to the baseDir + */ + getFailedName: (args: GetNameArgs) => string; + + /** + * Returns the baseline image. + */ + getBaseline: (args: ImageArgs) => OptionalImage; + /** + * Saves the baseline image. + */ + saveBaseline: (args: SaveImageArgs) => void | Promise; + + /** + * Saves the image representing the difference between the + * baseline and the new image. + */ + saveDiff: (args: SaveImageArgs) => void | Promise; + /** + * Saves the failed image file. + */ + saveFailed: (args: SaveImageArgs) => void | Promise; + + /** + * Gets the difference between two images. + */ + getImageDiff: (args: DiffArgs) => DiffResult | Promise; +} + +async function getImage({ filePath }: ImageArgs) { + if (await fileExists(filePath)) { + return readFile(filePath); + } +} + +async function saveImage({ filePath, content }: SaveImageArgs) { + await mkdirp(path.dirname(filePath)); + await writeFile(filePath, content); +} + +export const defaultOptions: VisualRegressionPluginOptions = { + update: false, + baseDir: 'screenshots', + diffOptions: {}, + + getBaselineName: ({ browser, name }) => path.join(browser, 'baseline', name), + getDiffName: ({ browser, name }) => path.join(browser, 'failed', `${name}-diff`), + getFailedName: ({ browser, name }) => path.join(browser, 'failed', name), + + getBaseline: getImage, + saveBaseline: saveImage, + + saveDiff: saveImage, + saveFailed: saveImage, + + getImageDiff: pixelMatchDiff, +}; diff --git a/packages/test-runner-visual-regression/src/fs.ts b/packages/test-runner-visual-regression/src/fs.ts new file mode 100644 index 000000000..3217d7802 --- /dev/null +++ b/packages/test-runner-visual-regression/src/fs.ts @@ -0,0 +1,15 @@ +import fs from 'fs'; +import { promisify } from 'util'; + +export const readFile = promisify(fs.readFile); +export const writeFile = promisify(fs.writeFile); +export const fsAccess = promisify(fs.access); + +export async function fileExists(filePath: string) { + try { + await fsAccess(filePath); + return true; + } catch { + return false; + } +} diff --git a/packages/test-runner-visual-regression/src/index.ts b/packages/test-runner-visual-regression/src/index.ts new file mode 100644 index 000000000..dc7c13cae --- /dev/null +++ b/packages/test-runner-visual-regression/src/index.ts @@ -0,0 +1 @@ +export { visualRegressionPlugin } from './visualRegressionPlugin'; diff --git a/packages/test-runner-visual-regression/src/pixelMatchDiff.ts b/packages/test-runner-visual-regression/src/pixelMatchDiff.ts new file mode 100644 index 000000000..36b2f3081 --- /dev/null +++ b/packages/test-runner-visual-regression/src/pixelMatchDiff.ts @@ -0,0 +1,31 @@ +import pixelmatch from 'pixelmatch'; +import { PNG } from 'pngjs'; + +import { DiffArgs, DiffResult } from './config'; +import { VisualRegressionError } from './VisualRegressionError'; + +export function pixelMatchDiff({ baselineImage, image, options }: DiffArgs): DiffResult { + const basePng = PNG.sync.read(baselineImage); + const png = PNG.sync.read(image); + + if (basePng.width !== png.width || basePng.height !== png.height) { + throw new VisualRegressionError( + `Screenshot is not the same width and height as the baseline. ` + + `Baseline: { width: ${basePng.width}, height: ${basePng.height} }` + + `Screenshot: { width: ${png.width}, height: ${png.height} }`, + ); + } + + const width = png.width; + const height = png.height; + + const diff = new PNG({ width, height }); + + const numDiffPixels = pixelmatch(basePng.data, png.data, diff.data, width, height, options); + const diffPercentage = (numDiffPixels / (width * height)) * 100; + + return { + diffImage: diff.data, + diffPercentage, + }; +} diff --git a/packages/test-runner-visual-regression/src/visualDiffCommand.ts b/packages/test-runner-visual-regression/src/visualDiffCommand.ts new file mode 100644 index 000000000..20acb2bc3 --- /dev/null +++ b/packages/test-runner-visual-regression/src/visualDiffCommand.ts @@ -0,0 +1,78 @@ +import path from 'path'; + +import { VisualRegressionPluginOptions } from './config'; + +function resolveImagePath(baseDir: string, name: string) { + const finalName = path.extname(name) ? name : `${name}.png`; + if (path.isAbsolute(finalName)) { + return finalName; + } + return path.join(baseDir, finalName); +} + +export interface VisualDiffCommandResult { + errorMessage?: string; + diffPercentage: number; + passed: boolean; +} + +export async function visualDiffCommand( + options: VisualRegressionPluginOptions, + image: Buffer, + browser: string, + name: string, +): Promise { + const baseDir = path.resolve(options.baseDir); + const baselineName = options.getBaselineName({ browser, name }); + + const baselineImage = await options.getBaseline({ + filePath: resolveImagePath(baseDir, baselineName), + baseDir, + name: baselineName, + }); + + if (!baselineImage || options.update) { + await options.saveBaseline({ + filePath: resolveImagePath(baseDir, baselineName), + baseDir, + name: baselineName, + content: image, + }); + return { diffPercentage: -1, passed: true }; + } + + const diffName = options.getDiffName({ browser, name }); + const failedName = options.getFailedName({ browser, name }); + + const { diffImage, diffPercentage } = await options.getImageDiff({ + name, + baselineImage, + image, + options: options.diffOptions, + }); + const passed = diffPercentage === 0; + + if (!passed) { + await options.saveDiff({ + filePath: resolveImagePath(baseDir, diffName), + baseDir, + name: diffName, + content: diffImage, + }); + + await options.saveFailed({ + filePath: resolveImagePath(baseDir, failedName), + baseDir, + name: failedName, + content: diffImage, + }); + } + + return { + errorMessage: !passed + ? `Visual diff failed. New screenshot is ${diffPercentage.toFixed(2)} % different.` + : undefined, + diffPercentage: -1, + passed, + }; +} diff --git a/packages/test-runner-visual-regression/src/visualRegressionPlugin.ts b/packages/test-runner-visual-regression/src/visualRegressionPlugin.ts new file mode 100644 index 000000000..6a412f014 --- /dev/null +++ b/packages/test-runner-visual-regression/src/visualRegressionPlugin.ts @@ -0,0 +1,115 @@ +import { TestRunnerPlugin } from '@web/test-runner-core'; +import type { ChromeLauncher } from '@web/test-runner-chrome'; + +import { defaultOptions, VisualRegressionPluginOptions } from './config'; +import { visualDiffCommand, VisualDiffCommandResult } from './visualDiffCommand'; +import { VisualRegressionError } from './VisualRegressionError'; +import type { PlaywrightLauncher } from '@web/test-runner-playwright'; + +interface Payload { + id: string; + name: string; +} + +function validatePayload(payload: any): payload is Payload { + if (payload == null || typeof payload !== 'object') { + throw new Error('Command visual-diff requires a payload with an id and name'); + } + + if (typeof payload.id !== 'string') { + throw new Error('Command visual-diff is missing an id in payload'); + } + + if (typeof payload.name !== 'string') { + throw new Error('Command visual-diff is missing a name in payload'); + } + return true; +} + +export function visualRegressionPlugin( + options: Partial = {}, +): TestRunnerPlugin { + const mergedOptions = { + ...defaultOptions, + ...options, + diffOptions: { + ...defaultOptions.diffOptions, + ...options.diffOptions, + }, + }; + + return { + name: 'visual-regression', + + async executeCommand({ command, session, payload }): Promise { + if (command === 'visual-diff') { + try { + if (!validatePayload(payload)) { + return; + } + + if (session.browser.type === 'puppeteer') { + const browser = session.browser as ChromeLauncher; + const page = browser.getPage(session.id); + + const handle = await page.evaluateHandle(function findElement(elementId) { + return ( + (window as any).__WTR_VISUAL_REGRESSION__ && + (window as any).__WTR_VISUAL_REGRESSION__[elementId] + ); + }, payload.id); + const element = handle.asElement(); + if (!element) { + throw new VisualRegressionError( + 'Something went wrong diffing element, the browser could not find it.', + ); + } + + const screenshot = await element.screenshot({ encoding: 'binary' }); + return visualDiffCommand(mergedOptions, screenshot, session.browser.name, payload.name); + } + + if (session.browser.type === 'playwright') { + const browser = session.browser as PlaywrightLauncher; + const page = browser.getPage(session.id); + + const handle = await page.evaluateHandle(function findElement(elementId) { + return ( + (window as any).__WTR_VISUAL_REGRESSION__ && + (window as any).__WTR_VISUAL_REGRESSION__[elementId] + ); + }, payload.id); + const element = handle.asElement(); + if (!element) { + throw new VisualRegressionError( + 'Something went wrong diffing element, the browser could not find it.', + ); + } + + const screenshot = await element.screenshot(); + return visualDiffCommand(mergedOptions, screenshot, session.browser.name, payload.name); + } + + throw new Error( + `Browser type ${session.browser.type} is not supported for visual diffing.`, + ); + } catch (error: unknown) { + if (error instanceof VisualRegressionError) { + return { + errorMessage: `Something went wrong while executing creating visual diff: ${error.message}`, + diffPercentage: -1, + passed: false, + }; + } + + console.error(error); + return { + errorMessage: 'Something went wrong while creating visual diff.', + diffPercentage: -1, + passed: false, + }; + } + } + }, + }; +} diff --git a/packages/test-runner-visual-regression/test/diff-fail-test.js b/packages/test-runner-visual-regression/test/diff-fail-test.js new file mode 100644 index 000000000..176e6f47c --- /dev/null +++ b/packages/test-runner-visual-regression/test/diff-fail-test.js @@ -0,0 +1,10 @@ +import { visualDiff } from '../browser/commands.mjs'; + +it('can diff an image', async () => { + const element = document.createElement('p'); + element.textContent = 'Hello world'; + element.style.color = 'red'; + document.body.appendChild(element); + + await visualDiff(element, 'my-failed-element'); +}); diff --git a/packages/test-runner-visual-regression/test/diff-pass-test.js b/packages/test-runner-visual-regression/test/diff-pass-test.js new file mode 100644 index 000000000..2d99ce86a --- /dev/null +++ b/packages/test-runner-visual-regression/test/diff-pass-test.js @@ -0,0 +1,10 @@ +import { visualDiff } from '../browser/commands.mjs'; + +it('can diff an image', async () => { + const element = document.createElement('p'); + element.textContent = 'Hello world'; + element.style.color = 'blue'; + document.body.appendChild(element); + + await visualDiff(element, 'my-element'); +}); diff --git a/packages/test-runner-visual-regression/test/visualRegressionPlugin.test.ts b/packages/test-runner-visual-regression/test/visualRegressionPlugin.test.ts new file mode 100644 index 000000000..51dcadc53 --- /dev/null +++ b/packages/test-runner-visual-regression/test/visualRegressionPlugin.test.ts @@ -0,0 +1,100 @@ +import path from 'path'; +import { runTests } from '@web/test-runner-core/test-helpers'; +import { expect } from 'chai'; +import { chromeLauncher } from '@web/test-runner-chrome'; + +import { visualRegressionPlugin } from '../src/visualRegressionPlugin'; +import { fileExists } from '../src/fs'; +import { playwrightLauncher } from '@web/test-runner-playwright'; + +describe('visualRegressionPlugin', function test() { + this.timeout(20000); + + it.skip('can run a passing test', async () => { + await runTests( + { + browsers: [ + chromeLauncher(), + playwrightLauncher({ product: 'firefox' }), + playwrightLauncher({ product: 'webkit' }), + ], + plugins: [ + { + name: 'resolve-commands', + resolveImport({ source }) { + if (source === '@web/test-runner-commands') { + return '/packages/test-runner-commands/browser/commands.mjs'; + } + }, + }, + visualRegressionPlugin({ + update: process.argv.includes('--update-visual-diffs'), + }), + ], + }, + [path.join(__dirname, 'diff-pass-test.js')], + ); + }); + + it.skip('can run a failed test', async () => { + const runner = await runTests( + { + browsers: [ + chromeLauncher(), + playwrightLauncher({ product: 'firefox' }), + playwrightLauncher({ product: 'webkit' }), + ], + plugins: [ + { + name: 'resolve-commands', + resolveImport({ source }) { + if (source === '@web/test-runner-commands') { + return '/packages/test-runner-commands/browser/commands.mjs'; + } + }, + }, + visualRegressionPlugin({ + update: process.argv.includes('--update-visual-diffs'), + }), + ], + }, + [path.join(__dirname, 'diff-fail-test.js')], + { allowFailure: true, reportErrors: false }, + ); + + const sessions = Array.from(runner.sessions.all()); + expect(sessions.length).to.equal(3); + + for (const session of sessions) { + expect(session.passed).to.equal(false); + expect(session.testResults!.tests.length).to.equal(1); + expect(session.testResults!.tests[0].error!.message).to.include( + 'Visual diff failed. New screenshot is ', + ); + expect( + await fileExists( + path.resolve( + __dirname, + '..', + 'screenshots', + session.browser.name, + 'failed', + 'my-failed-element.png', + ), + ), + ).to.equal(true); + expect( + await fileExists( + path.resolve( + __dirname, + '..', + 'screenshots', + session.browser.name, + 'failed', + 'my-failed-element-diff.png', + ), + ), + ).to.equal(true); + } + }); +}); diff --git a/packages/test-runner-visual-regression/tsconfig.json b/packages/test-runner-visual-regression/tsconfig.json new file mode 100644 index 000000000..ab992c2ce --- /dev/null +++ b/packages/test-runner-visual-regression/tsconfig.json @@ -0,0 +1,46 @@ +// Don't edit this file directly. It is generated by /scripts/update-package-configs.ts + +{ + "extends": "../../tsconfig.node-base.json", + "compilerOptions": { + "module": "commonjs", + "outDir": "./dist", + "rootDir": "./src", + "composite": true, + "allowJs": true + }, + "references": [ + { + "path": "../dev-server-core/tsconfig.json" + }, + { + "path": "../test-runner-core/tsconfig.json" + }, + { + "path": "../browser-logs/tsconfig.json" + }, + { + "path": "../test-runner-coverage-v8/tsconfig.json" + }, + { + "path": "../test-runner-mocha/tsconfig.json" + }, + { + "path": "../test-runner-chrome/tsconfig.json" + }, + { + "path": "../test-runner-playwright/tsconfig.json" + }, + { + "path": "../test-runner-commands/tsconfig.json" + } + ], + "include": [ + "src" + ], + "exclude": [ + "src/browser", + "tests", + "dist" + ] +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 0ab00acb5..ec7bc3905 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -49,6 +49,9 @@ { "path": "./packages/test-runner-selenium/tsconfig.json" }, + { + "path": "./packages/dev-server-esbuild/tsconfig.json" + }, { "path": "./packages/rollup-plugin-copy/tsconfig.json" }, @@ -58,9 +61,6 @@ { "path": "./packages/dev-server/tsconfig.json" }, - { - "path": "./packages/dev-server-esbuild/tsconfig.json" - }, { "path": "./packages/dev-server-import-maps/tsconfig.json" }, @@ -75,6 +75,9 @@ }, { "path": "./packages/test-runner-saucelabs/tsconfig.json" + }, + { + "path": "./packages/test-runner-visual-regression/tsconfig.json" } ] } \ No newline at end of file diff --git a/workspace-packages.mjs b/workspace-packages.mjs index efe8d0556..032b37db8 100644 --- a/workspace-packages.mjs +++ b/workspace-packages.mjs @@ -23,6 +23,7 @@ const packages = [ { name: 'test-runner-junit-reporter', type: 'ts', environment: 'node' }, { name: 'test-runner-mocha', type: 'ts', environment: 'browser' }, { name: 'test-runner-saucelabs', type: 'ts', environment: 'node' }, + { name: 'test-runner-visual-regression', type: 'ts', environment: 'node' }, ]; export { packages }; diff --git a/yarn.lock b/yarn.lock index 94ceeca3a..89b640e25 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1993,6 +1993,13 @@ resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.0.tgz#69a23a3ad29caf0097f06eda59b361ee2f0639f6" integrity sha1-aaI6OtKcrwCX8G7aWbNh7i8GOfY= +"@types/mkdirp@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-1.0.1.tgz#0930b948914a78587de35458b86c907b6e98bbf6" + integrity sha512-HkGSK7CGAXncr8Qn/0VqNtExEE+PHMWb+qlR1faHMao7ng6P3tAaoWWBMdva0gL5h4zprjIO89GJOLXsMcDm1Q== + dependencies: + "@types/node" "*" + "@types/mocha@^8.0.01", "@types/mocha@^8.0.1": version "8.0.3" resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-8.0.3.tgz#51b21b6acb6d1b923bbdc7725c38f9f455166402" @@ -2036,6 +2043,20 @@ resolved "https://registry.yarnpkg.com/@types/picomatch/-/picomatch-2.2.1.tgz#f9e5a5e6ad03996832975ab7eadfa35791ca2a8f" integrity sha512-26/tQcDmJXYHiaWAAIjnTVL5nwrT+IVaqFZIbBImAuKk/r/j1r/1hmZ7uaOzG6IknqP3QHcNNQ6QO8Vp28lUoA== +"@types/pixelmatch@^5.2.1": + version "5.2.1" + resolved "https://registry.yarnpkg.com/@types/pixelmatch/-/pixelmatch-5.2.1.tgz#c82843e0c08417b45a233ddd267852c3c1b7afaa" + integrity sha512-4KPEw8fz139KyHfBiNHI9+nmA93gcH/Dj2Q+10JF34XO0Sb20IIGsdj+9Ol+dgJUWN5RIa7NRsbEXFFd5MiG0Q== + dependencies: + "@types/node" "*" + +"@types/pngjs@^3.4.2": + version "3.4.2" + resolved "https://registry.yarnpkg.com/@types/pngjs/-/pngjs-3.4.2.tgz#8dc49b45fbcf18a5873179e3664f049388e39ecf" + integrity sha512-LJVPDraJ5YFEnMHnzxTN4psdWz1M61MtaAAWPn3qnDk5fvs7BAmmQ9pd3KPlrdrvozMyne4ktanD4pg0L7x1Pw== + dependencies: + "@types/node" "*" + "@types/puppeteer-core@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/puppeteer-core/-/puppeteer-core-2.0.0.tgz#3b7fbbac53d56b566f5ef096116e1d60d504aa45" @@ -9179,6 +9200,13 @@ pinkie@^2.0.0: resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" integrity sha1-clVrgM+g1IqXToDnckjoDtT3+HA= +pixelmatch@^5.2.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.2.1.tgz#9e4e4f4aa59648208a31310306a5bed5522b0d65" + integrity sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ== + dependencies: + pngjs "^4.0.1" + pkg-dir@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3" @@ -9221,6 +9249,11 @@ pluralize@^8.0.0: resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1" integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA== +pngjs@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-4.0.1.tgz#f803869bb2fc1bfe1bf99aa4ec21c108117cfdbe" + integrity sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg== + pngjs@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-5.0.0.tgz#e79dd2b215767fd9c04561c01236df960bce7fbb"