diff --git a/cspell-wordlist.txt b/cspell-wordlist.txt index 513030ec9..785969053 100644 --- a/cspell-wordlist.txt +++ b/cspell-wordlist.txt @@ -37,3 +37,5 @@ transpiling unshift usingtheplatform webapps +wdio +webdriverio \ No newline at end of file diff --git a/docs/config/01-overview.md b/docs/config/01-overview.md index 0f12cc833..8d35ca59b 100644 --- a/docs/config/01-overview.md +++ b/docs/config/01-overview.md @@ -63,7 +63,7 @@ bundles: [ *default: '.stencil'* The directory where sub-directories will be created for caching when [`enableCache`](#enablecache) is set `true` or if using -[Stencil's Screenshot Connector](../testing/screenshot-connector.md). +[Stencil's Screenshot Connector](../testing/stencil-testrunner/07-screenshot-connector.md). A Stencil config like the following: @@ -409,7 +409,7 @@ taskQueue: 'async' ## testing -Please see the [testing config docs](../testing/config.md). +Please see the [testing config docs](../testing/stencil-testrunner/02-config.md). ## transformAliasedImportPaths diff --git a/docs/testing/01-overview.md b/docs/testing/01-overview.md index 59c7cd288..76227afb9 100644 --- a/docs/testing/01-overview.md +++ b/docs/testing/01-overview.md @@ -25,136 +25,15 @@ choose when to use each type of testing: together. For example, when `my-component` has the `X` attribute, the child component then renders the text `Y`, and expects to receive the event `Z`. -Both types of testing use [Jest](https://jestjs.io/) as the JavaScript testing solution. End-to-end tests also use -[Puppeteer](https://pptr.dev/) instead of a Node environment. This allows end-to-end tests to run within an actual -browser in order to provide more realistic results. - ## Library Support -Stencil uses [Jest](https://jestjs.io/) and [Puppeteer](https://pptr.dev/) as its testing libraries, and allows -developers to install both libraries using their preferred package manager. - -If you created a project using `npm init stencil`, these libraries were installed for you. Depending on when your -project was created, you may or may not have the latest supported version installed. - -To view current version support for both Jest and Puppeteer, please see the -[Stencil support policy for testing libraries](../reference/support-policy.md#testing-libraries). - -## Testing Commands - -Stencil tests are run using the command `stencil test`, followed by one or more optional flags: -- `--spec` to run unit tests -- `--e2e` to run end-to-end tests -- `--watchAll` to watch the file system for changes, and rerun tests when changes are detected - -When the `--spec` and/or `--e2e` flags are provided, Stencil will automatically run the tests associated with each flag. - -Below a series of example `npm` scripts which can be added to the project's `package.json` file to run Stencil tests: - -```json -{ - "scripts": { - "test": "stencil test --spec", - "test.watch": "stencil test --spec --watchAll", - "test.end-to-end": "stencil test --e2e" - } -} -``` - -Each command above begins with `stencil test`, which tells Stencil to run tests. Note that each `stencil test` command -in example above is followed one or more of the optional flags. Looking at each script, one at a time: -- the `test` script runs unit tests for our Stencil project. -- the `test.watch` script runs unit tests for our Stencil project. It watches the filesystem for changes, and reruns -tests when changes are detected. -- the `test.end-to-end` script runs the end-to-end tests for our Stencil project. - -If you created a project using `npm init stencil`, these scripts are provided to you automatically. - -Stencil does not prescribe any specific naming convention for the names of your scripts. The `test.watch` script could -as easily be named `test-watch`, `test.incremental`, etc. As long as the script itself uses the `stencil test` command, -your tests should be run. - -### Testing Configuration - -Stencil will apply defaults from data it has already gathered. For example, Stencil already knows what directories to look through, and what files are spec and e2e files. Jest can still be configured using the same config names, but now using the stencil config `testing` property. Please see the [Testing Config docs](./config.md) for more info. - -```tsx -import { Config } from '@stencil/core'; - -export const config: Config = { - testing: { - testPathIgnorePatterns: [...] - } -}; -``` - -### Command Line Arguments - -While the Stencil CLI offers a certain set of command line flags to specify e.g. which types of tests to run, you also have access to all Jest options through the CLI. For example to specify a single test, you can pass in a positional argument to Jest by adding a `--`, e.g.: - -```sh -# run a single unit test -npx stencil test --spec -- src/components/my-component/my-component.spec.ts -# run a single e2e test -npx stencil test --e2e -- src/components/my-component/my-component.e2e.ts -``` - -Next to positional arguments, Stencil also passes along [certain](https://github.com/ionic-team/stencil/blob/54d4ee252768e1d225baababee0093fdb0562b83/src/cli/config-flags.ts#L38-L85) Jest specific flags, e.g.: - -```sh -# enable code coverage -npx stencil test --spec --coverage -``` - -You can find more information about [Jest CLI options](https://jestjs.io/docs/cli) in the project documentation. - -## Running and Debugging Tests in VS Code - -Adding the following configurations to `.vscode/launch.json` will allow you to use the VS Code Debugger to run the Stencil test runner for the currently active file in your editor. - -To use the below configuration: -1. Ensure the test file you want to run is open and in the current active window in VS Code. -2. Select the debug configuration to run: - 1. 'E2E Test Current File' will run the end-to-end tests in the active test file - 2. 'Spec Test Current File' will run the spec tests in the active test file -3. Hit the play button to start the test. - -```json title=".vscode/launch.json" -{ - "configurations": [ - { - "type": "node", - "request": "launch", - "name": "E2E Test Current File", - "cwd": "${workspaceFolder}", - "program": "${workspaceFolder}/node_modules/.bin/stencil", - "args": ["test", "--e2e", "--", "--maxWorkers=0", "${fileBasename}"], - "console": "integratedTerminal", - "internalConsoleOptions": "neverOpen" - }, - { - "type": "node", - "request": "launch", - "name": "Spec Test Current File", - "cwd": "${workspaceFolder}", - "program": "${workspaceFolder}/node_modules/.bin/stencil", - "args": ["test", "--spec", "--", "${fileBasename}"], - "console": "integratedTerminal", - "internalConsoleOptions": "neverOpen" - } - ] -} -``` +Stencil currently supports the following tools for testing components: -:::tip -Windows users: The `program` value should be set to `"${workspaceFolder}/node_modules/bin/stencil"`. -If that value does not work, please try`"${workspaceFolder}/node_modules/@stencil/core/bin/stencil"`. -::: +- [Stencil Test Runner](./stencil-testrunner/01-overview.md): a built-in test runner based on Jest for unit and end-to-end testing with Puppeteer to run within an actual browser in order to provide more realistic results. +- [WebdriverIO](./webdriverio/01-overview.md): a browser and mobile automation test framework for Node.js that allows you to run component and end-to-end tests across all browsers. -The configuration above makes use of special VS Code variables, such as `${workspaceFolder}`. -These variables are substituted with actual values upon starting the tests. -For more information regarding the values these variables resolve to, please see VS Code's [Variables Reference documentation](https://code.visualstudio.com/docs/editor/variables-reference). +:::info -## Other Resources +We are actively working to support a wider range of testing tools, including Playwright. Stay tuned for updates! -- [The Basics of Unit Testing in StencilJS](https://eliteionic.com/tutorials/the-basics-of-unit-testing-in-stencil-js/) +::: \ No newline at end of file diff --git a/docs/testing/config.md b/docs/testing/config.md deleted file mode 100644 index eb58cc5f6..000000000 --- a/docs/testing/config.md +++ /dev/null @@ -1,160 +0,0 @@ ---- -title: Testing Config -sidebar_label: Config -description: Testing Config -slug: /testing-config ---- - -# Testing Config - -The `testing` config setting in `stencil.config.ts` specifies an object that corresponds to the jest configuration that should be used in your tests. Stencil provides a default configuration, which you likely won't need to edit, however it can be extended with the same configuration options as Jest. See the [Configuring Jest Guide](https://jestjs.io/docs/en/configuration.html) for configuration details. - -:::note -Keep in mind that the usual way of configuring Jest (`package.json` and `jest.config.js`) is not used with the `stencil testing` command. Jest can still be used, but configuring the presets, transpilation and setting up the correct commands must be done by the project. -::: - -Some additional Stencil specific options may be set here as well for configuring the e2e tests: - -```tsx -export interface TestingConfig extends JestConfig { - /** - * The `allowableMismatchedPixels` value is used to determine an acceptable - * number of pixels that can be mismatched before the image is considered - * to have changes. Realistically, two screenshots representing the same - * content may have a small number of pixels that are not identical due to - * anti-aliasing, which is perfectly normal. If the `allowableMismatchedRatio` - * is provided it will take precedence, otherwise `allowableMismatchedPixels` - * will be used. - */ - allowableMismatchedPixels?: number; - - /** - * The `allowableMismatchedRatio` ranges from `0` to `1` and is used to - * determine an acceptable ratio of pixels that can be mismatched before - * the image is considered to have changes. Realistically, two screenshots - * representing the same content may have a small number of pixels that - * are not identical due to anti-aliasing, which is perfectly normal. The - * `allowableMismatchedRatio` is the number of pixels that were mismatched, - * divided by the total number of pixels in the screenshot. For example, - * a ratio value of `0.06` means 6% of the pixels can be mismatched before - * the image is considered to have changes. If the `allowableMismatchedRatio` - * is provided it will take precedence, otherwise `allowableMismatchedPixels` - * will be used. - */ - allowableMismatchedRatio?: number; - - /** - * Matching threshold while comparing two screenshots. Value ranges from `0` to `1`. - * Smaller values make the comparison more sensitive. The `pixelmatchThreshold` - * value helps to ignore anti-aliasing. Default: `0.1` - */ - pixelmatchThreshold?: number; - - /** - * Additional arguments to pass to the browser instance. - */ - browserArgs?: string[]; - - /** - * Path to a Chromium or Chrome executable to run instead of the bundled Chromium. - */ - browserExecutablePath?: string; - - /** - * Whether to run browser e2e tests in headless mode. - * - * `headless` is an argument passed through to Puppeteer (which is passed to Chrome) for - * end-to-end testing. Prior to Chrome v112, `headless` was treated like a boolean flag. - * Starting with Chrome v112, 'new' is an accepted option to support Chrome's new - * headless mode. - * - * The following values are accepted: - * - "new" - enables the "new" headless mode, starting with Chrome 112 - * - `true` - enables the "old" headless mode, prior to Chrome 112 - * - `false` - enables the "headful" mode - * - * Stencil will default to `true` (the old headless mode) if no value is provided. - * - * In the future, Chrome will enable the new headless mode by default, even when `true` - * is provided. - * - * {@see https://developer.chrome.com/articles/new-headless/} - */ - browserHeadless?: boolean; - - /** - * Whether to auto-open a DevTools panel for each tab. - * If this option is true, the headless option will be set false - */ - browserDevtools?: boolean; - - /** - * Slows down e2e browser operations by the specified amount of milliseconds. - * Useful so that you can see what is going on. - */ - browserSlowMo?: number; - - /** - * Array of browser emulations to be used during _screenshot_ tests. A full screenshot - * test is run for each emulation. - * - * To emulate a device display for your e2e tests, use the `setViewport` method on a test's E2E page. - */ - emulate?: EmulateConfig[]; - - /** - * Path to the Screenshot Connector module. - */ - screenshotConnector?: string; -} - -export interface EmulateConfig { - /** - * Predefined device descriptor name, such as "iPhone X" or "Nexus 10". - * For a complete list please see: https://github.com/puppeteer/puppeteer/blob/main/src/DeviceDescriptors.ts - */ - device?: string; - - /** - * User-Agent to be used. Defaults to the user-agent of the installed Puppeteer version. - */ - userAgent?: string; - - viewport?: EmulateViewport; -} - - -export interface EmulateViewport { - - /** - * Page width in pixels. - */ - width: number; - - /** - * page height in pixels. - */ - height: number; - - /** - * Specify device scale factor (can be thought of as dpr). Defaults to 1. - */ - deviceScaleFactor?: number; - - /** - * Whether the meta viewport tag is taken into account. Defaults to false. - */ - isMobile?: boolean; - - /** - * Specifies if viewport supports touch events. Defaults to false - */ - hasTouch?: boolean; - - /** - * Specifies if viewport is in landscape mode. Defaults to false. - */ - isLandscape?: boolean; - -} -``` diff --git a/docs/testing/stencil-testrunner/01-overview.md b/docs/testing/stencil-testrunner/01-overview.md new file mode 100644 index 000000000..bb1fa0a90 --- /dev/null +++ b/docs/testing/stencil-testrunner/01-overview.md @@ -0,0 +1,127 @@ +--- +title: Stencil Test Runner Overview +sidebar_label: Overview +--- + +# Overview + +Stencil has a built-in test runner that uses [Jest](https://jestjs.io/) and [Puppeteer](https://pptr.dev/) as its testing libraries, and allows developers to install both libraries using their preferred package manager. + +If you created a project using `npm init stencil`, these libraries were installed for you. Depending on when your project was created, you may or may not have the latest supported version installed. + +To view current version support for both Jest and Puppeteer, please see the [Stencil support policy for testing libraries](../../reference/support-policy.md#testing-libraries). + +## Testing Commands + +Stencil tests are run using the command `stencil test`, followed by one or more optional flags: +- `--spec` to run unit tests +- `--e2e` to run end-to-end tests +- `--watchAll` to watch the file system for changes, and rerun tests when changes are detected + +When the `--spec` and/or `--e2e` flags are provided, Stencil will automatically run the tests associated with each flag. + +Below a series of example `npm` scripts which can be added to the project's `package.json` file to run Stencil tests: + +```json +{ + "scripts": { + "test": "stencil test --spec", + "test.watch": "stencil test --spec --watchAll", + "test.end-to-end": "stencil test --e2e" + } +} +``` + +Each command above begins with `stencil test`, which tells Stencil to run tests. Note that each `stencil test` command +in the example above is followed by one or more of the optional flags. Looking at each script, one at a time: +- the `test` script runs unit tests for our Stencil project. +- the `test.watch` script runs unit tests for our Stencil project. It watches the filesystem for changes, and reruns +tests when changes are detected. +- the `test.end-to-end` script runs the end-to-end tests for our Stencil project. + +If you created a project using `npm init stencil`, these scripts are provided to you automatically. + +Stencil does not prescribe any specific naming convention for the names of your scripts. The `test.watch` script could as easily be named `test-watch`, `test.incremental`, etc. As long as the script itself uses the `stencil test` command, your tests should be run. + +### Testing Configuration + +Stencil will apply defaults from data it has already gathered. For example, Stencil already knows what directories to look through, and what files are spec and e2e files. Jest can still be configured using the same config names, but now using the stencil config `testing` property. Please see the [Testing Config docs](./02-config.md) for more info. + +```tsx +import { Config } from '@stencil/core'; + +export const config: Config = { + testing: { + testPathIgnorePatterns: [...] + } +}; +``` + +### Command Line Arguments + +While the Stencil CLI offers a certain set of command line flags to specify e.g. which types of tests to run, you also have access to all Jest options through the CLI. For example to specify a single test, you can pass in a positional argument to Jest by adding a `--`, e.g.: + +```sh +# run a single unit test +npx stencil test --spec -- src/components/my-component/my-component.spec.ts +# run a single e2e test +npx stencil test --e2e -- src/components/my-component/my-component.e2e.ts +``` + +Next to positional arguments, Stencil also passes along [certain](https://github.com/ionic-team/stencil/blob/54d4ee252768e1d225baababee0093fdb0562b83/src/cli/config-flags.ts#L38-L85) Jest specific flags, e.g.: + +```sh +# enable code coverage +npx stencil test --spec --coverage +``` + +You can find more information about [Jest CLI options](https://jestjs.io/docs/cli) in the project documentation. + +## Running and Debugging Tests in VS Code + +Adding the following configurations to `.vscode/launch.json` will allow you to use the VS Code Debugger to run the Stencil test runner for the currently active file in your editor. + +To use the below configuration: +1. Ensure the test file you want to run is open and in the current active window in VS Code. +2. Select the debug configuration to run: + 1. 'E2E Test Current File' will run the end-to-end tests in the active test file + 2. 'Spec Test Current File' will run the spec tests in the active test file +3. Hit the play button to start the test. + +```json title=".vscode/launch.json" +{ + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "E2E Test Current File", + "cwd": "${workspaceFolder}", + "program": "${workspaceFolder}/node_modules/.bin/stencil", + "args": ["test", "--e2e", "--", "--maxWorkers=0", "${fileBasename}"], + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen" + }, + { + "type": "node", + "request": "launch", + "name": "Spec Test Current File", + "cwd": "${workspaceFolder}", + "program": "${workspaceFolder}/node_modules/.bin/stencil", + "args": ["test", "--spec", "--", "${fileBasename}"], + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen" + } + ] +} +``` + +:::tip +Windows users: The `program` value should be set to `"${workspaceFolder}/node_modules/bin/stencil"`. +If that value does not work, please try`"${workspaceFolder}/node_modules/@stencil/core/bin/stencil"`. +::: + +The configuration above makes use of special VS Code variables, such as `${workspaceFolder}`. These variables are substituted with actual values upon starting the tests. For more information regarding the values these variables resolve to, please see VS Code's [Variables Reference documentation](https://code.visualstudio.com/docs/editor/variables-reference). + +## Other Resources + +- [The Basics of Unit Testing in StencilJS](https://eliteionic.com/tutorials/the-basics-of-unit-testing-in-stencil-js/) diff --git a/docs/testing/stencil-testrunner/02-config.md b/docs/testing/stencil-testrunner/02-config.md new file mode 100644 index 000000000..1671bba4f --- /dev/null +++ b/docs/testing/stencil-testrunner/02-config.md @@ -0,0 +1,20 @@ +--- +title: Testing Config +sidebar_label: Config +description: Testing Config +slug: /testing-config +--- + +# Testing Config + +The `testing` config setting in `stencil.config.ts` specifies an object that corresponds to the jest configuration that should be used in your tests. Stencil provides a default configuration, which you likely won't need to edit, however it can be extended with the same configuration options as Jest. See the [Configuring Jest Guide](https://jestjs.io/docs/en/configuration.html) for configuration details. + +:::note +Keep in mind that the usual way of configuring Jest (`package.json` and `jest.config.js`) is not used with the `stencil testing` command. Jest can still be used, but configuring the presets, transpilation and setting up the correct commands must be done by the project. +::: + +Some additional Stencil specific options may be set here as well for configuring the e2e tests. + +```tsx reference title="" +https://github.com/ionic-team/stencil/blob/278c336166e27d329fffb686f2ecb3185d156696/src/declarations/stencil-public-compiler.ts#L1817-L1963 +``` diff --git a/docs/testing/unit-testing.md b/docs/testing/stencil-testrunner/03-unit-testing.md similarity index 98% rename from docs/testing/unit-testing.md rename to docs/testing/stencil-testrunner/03-unit-testing.md index 94d5b5177..904c76a60 100644 --- a/docs/testing/unit-testing.md +++ b/docs/testing/stencil-testrunner/03-unit-testing.md @@ -14,7 +14,7 @@ To run unit tests, run `stencil test --spec`. Files ending in `.spec.ts` will be ## newSpecPage() -In order to unit test a component as rendered HTML, tests can use `newSpecPage()` imported from `@stencil/core/testing`. This testing utility method is similar to `newE2EPage()`, however, `newSpecPage()` is much faster since it does not require a full [Puppeteer](https://pptr.dev/) instance to be running. Please see the [newE2EPage()](./e2e-testing.md) docs on more information about complete End-to-end testing with Puppeteer. +In order to unit test a component as rendered HTML, tests can use `newSpecPage()` imported from `@stencil/core/testing`. This testing utility method is similar to `newE2EPage()`, however, `newSpecPage()` is much faster since it does not require a full [Puppeteer](https://pptr.dev/) instance to be running. Please see the [newE2EPage()](./05-e2e-testing.md) docs for more information about complete End-to-end testing with Puppeteer. Below is a simple example where `newSpecPage()` is given one component class which was imported, and the initial HTML to use for the test. In this example, when the component `MyCmp` renders it sets its text content as "Success!". The matcher `toEqualHtml()` is then used to ensure the component renders as expected. diff --git a/docs/testing/mocking.md b/docs/testing/stencil-testrunner/04-mocking.md similarity index 100% rename from docs/testing/mocking.md rename to docs/testing/stencil-testrunner/04-mocking.md diff --git a/docs/testing/e2e-testing.md b/docs/testing/stencil-testrunner/05-e2e-testing.md similarity index 98% rename from docs/testing/e2e-testing.md rename to docs/testing/stencil-testrunner/05-e2e-testing.md index 2dc3461e5..b5c0f1897 100644 --- a/docs/testing/e2e-testing.md +++ b/docs/testing/stencil-testrunner/05-e2e-testing.md @@ -244,4 +244,4 @@ export const config: Config = { }; ``` -Check [the testing config docs](./config.md) to learn more about the possibilities on this matter. +Check [the testing config docs](./02-config.md) to learn more about the possibilities on this matter. diff --git a/docs/testing/screenshot-visual-diff.md b/docs/testing/stencil-testrunner/06-screenshot-visual-diff.md similarity index 100% rename from docs/testing/screenshot-visual-diff.md rename to docs/testing/stencil-testrunner/06-screenshot-visual-diff.md diff --git a/docs/testing/screenshot-connector.md b/docs/testing/stencil-testrunner/07-screenshot-connector.md similarity index 51% rename from docs/testing/screenshot-connector.md rename to docs/testing/stencil-testrunner/07-screenshot-connector.md index cec526494..e62d24527 100644 --- a/docs/testing/screenshot-connector.md +++ b/docs/testing/stencil-testrunner/07-screenshot-connector.md @@ -35,24 +35,8 @@ For a good reference on how this can be done, have a look at the default `Stenci ## Methods The base connector which can be imported and extended from stencil has the following methods which can be overwritten: -```tsx -export interface ScreenshotConnector { - initBuild(opts: ScreenshotConnectorOptions): Promise; - - pullMasterBuild(): Promise; - - getMasterBuild(): Promise; - - getScreenshotCache(): Promise; - - completeBuild(masterBuild: ScreenshotBuild): Promise; - - publishBuild(buildResults: ScreenshotBuildResults): Promise; - - updateScreenshotCache(screenshotCache: ScreenshotCache, buildResults: ScreenshotBuildResults): Promise; - - generateJsonpDataUris(build: ScreenshotBuild): Promise; -} +```tsx reference title="ScreenshotConnector" +https://github.com/ionic-team/stencil/blob/a2e119d059ba0d0fa6155dbd3d82c17612630828/src/declarations/stencil-private.ts#L1631-L1645 ``` For references to the interfaces, [see here](#interfaces) @@ -87,193 +71,31 @@ At the end of the whole run, the screenshot cache should be updated with this me ## Interfaces -```tsx -export interface ScreenshotConnectorOptions { - buildId: string; - - buildMessage: string; - - buildAuthor?: string; - - buildUrl?: string; - - previewUrl?: string; - - appNamespace: string; - - buildTimestamp: number; - - logger: Logger; - - rootDir: string; - - cacheDir: string; - - packageDir: string; - - screenshotDirName?: string; - - imagesDirName?: string; - - buildsDirName?: string; - - currentBuildDir?: string; - - updateMaster?: boolean; - - allowableMismatchedPixels?: number; - - allowableMismatchedRatio?: number; - - pixelmatchThreshold?: number; - - waitBeforeScreenshot?: number; - - pixelmatchModulePath?: string; -} - -export interface ScreenshotBuild { - id: string; - - message: string; - - author?: string; - - url?: string; - - previewUrl?: string; - - appNamespace: string; - - timestamp: number; - - screenshots: Screenshot[]; -} - -export interface ScreenshotBuildResults { - appNamespace: string; - - masterBuild: ScreenshotBuild; - - currentBuild: ScreenshotBuild; - - compare: ScreenshotCompareResults; -} - -export interface ScreenshotCompareResults { - id: string; - a: { - id: string; - message: string; - author: string; - url: string; - previewUrl: string; - }; - - b: { - id: string; - message: string; - author: string; - url: string; - previewUrl: string; - }; - - timestamp: number; - - url: string; - - appNamespace: string; - - diffs: ScreenshotDiff[]; -} - -export interface ScreenshotCache { - timestamp?: number; - - lastBuildId?: string; - - size?: number; - - items?: { - /** - * Cache key - */ - key: string; - - /** - * Timestamp used to remove the oldest data - */ - ts: number; - - /** - * Mismatched pixels - */ - mp: number; - }[]; -} - -export interface Screenshot { - id: string; - - desc?: string; - - image: string; - - device?: string; - - userAgent?: string; - - width?: number; - - height?: number; - - deviceScaleFactor?: number; - - hasTouch?: boolean; - - isLandscape?: boolean; - - isMobile?: boolean; - - testPath?: string; - - diff?: ScreenshotDiff; -} - -export interface ScreenshotDiff { - mismatchedPixels: number; - - id?: string; - - desc?: string; - - imageA?: string; - - imageB?: string; - - device?: string; - - userAgent?: string; - - width?: number; - - height?: number; - - deviceScaleFactor?: number; - - hasTouch?: boolean; +```tsx reference title="ScreenshotConnectorOptions" +https://github.com/ionic-team/stencil/blob/a2e119d059ba0d0fa6155dbd3d82c17612630828/src/declarations/stencil-private.ts#L1676-L1698 +``` - isLandscape?: boolean; +```tsx reference title="ScreenshotBuild" +https://github.com/ionic-team/stencil/blob/a2e119d059ba0d0fa6155dbd3d82c17612630828/src/declarations/stencil-private.ts#L1725-L1734 +``` - isMobile?: boolean; +```tsx reference title="ScreenshotBuildResults" +https://github.com/ionic-team/stencil/blob/a2e119d059ba0d0fa6155dbd3d82c17612630828/src/declarations/stencil-private.ts#L1647-L1652 +``` - allowableMismatchedPixels: number; +```tsx reference title="ScreenshotCompareResults" +https://github.com/ionic-team/stencil/blob/a2e119d059ba0d0fa6155dbd3d82c17612630828/src/declarations/stencil-private.ts#L1654-L1674 +``` - allowableMismatchedRatio: number; +```ts reference title="ScreenshotCache" +https://github.com/ionic-team/stencil/blob/a2e119d059ba0d0fa6155dbd3d82c17612630828/src/declarations/stencil-private.ts#L1736-L1756 +``` - testPath?: string; +```ts reference title="Screenshot" +https://github.com/ionic-team/stencil/blob/a2e119d059ba0d0fa6155dbd3d82c17612630828/src/declarations/stencil-private.ts#L1758-L1772 +``` - cacheKey?: string; -} +```ts reference title="ScreenshotDiff" +https://github.com/ionic-team/stencil/blob/a2e119d059ba0d0fa6155dbd3d82c17612630828/src/declarations/stencil-private.ts#L1774-L1792 ``` diff --git a/docs/testing/stencil-testrunner/_category_.json b/docs/testing/stencil-testrunner/_category_.json new file mode 100644 index 000000000..08ddc1c23 --- /dev/null +++ b/docs/testing/stencil-testrunner/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Stencil Test Runner", + "position": 9 +} diff --git a/docs/testing/webdriverio/01-overview.md b/docs/testing/webdriverio/01-overview.md new file mode 100644 index 000000000..cd10595dc --- /dev/null +++ b/docs/testing/webdriverio/01-overview.md @@ -0,0 +1,89 @@ +--- +title: WebdriverIO Overview +sidebar_label: Overview +--- + +# Overview + +WebdriverIO is a progressive automation framework built to automate modern web and mobile applications. It simplifies the interaction with your project and provides a set of plugins that help you create a scalable, robust and stable test suite. + +Testing with WebdriverIO has the following advantages: + +- __Cross Browser Support__: WebdriverIO is designed to support all platforms, either on desktop or mobile. You can run tests on actual browser your users are using, including covering different versions of them. +- __Real User Interaction__: Interaction with elements in WebdriverIO through the WebDriver protocol is much closer to native user-triggered interactions than what can be achieved with emulated DOM environments (such as JSDom or Stencil's own Mock-Doc). +- __Web Platform Support__: Running tests in actual browsers allows you to tap into the latest Web Platform features for testing your components, often not available when using virtual DOM environments. +- __Real Environments__: Run your tests in an environment that your users are using and not somewhere that re-implements web standards with polyfills like JSDOM. + +## Set Up + +To get started with WebdriverIO, all you need to do is to run their project starter: + +```bash npm2yarn +npm init wdio@latest . +``` + +This will initiate WebdriverIO's configuration wizard that walks you through the setup. Make sure you select the following options when walking through it: + +- __What type of testing would you like to do?__ Select either: + - `Component or Unit Testing - in the browser` if you are interested adding unit tests for your components + - `E2E Testing - of Web or Mobile Applications` if you like to test your whole application + + You can always add either of them later on +- __Which framework do you use for building components?__: if you select _Component or Unit Testing_ make sure to select `StencilJS` as preset so WebdriverIO knows how to compile your components properly + +The following questions can be answered as desired. Once setup the wizard has created a `wdio.conf.ts` file and a `wdio` script to run your tests. + +:::info CJS vs. ESM + +WebdriverIO's generated config and test files use ESM syntax for imports. If you generated a project via the [`create-stencil`](https://www.npmjs.com/package/create-stencil) starter package your project is likely setup for CommonJS. To avoid any incompatibility issues, we recommend to rename your `wdio.conf.ts` to `wdio.conf.mts` and update the `wdio` script in your `package.json`. + +::: + +:::info Type Clashes + +It's possible that you run into TypeScript issues as WebdriverIO uses Mocha for component testing and Stencil Jest. Both register the same globals, e.g. `it` which causes type clashes. To fix these we recommend to add the following to your `tsconfig.json`: + +```json + "types": ["jest"] +``` + +This will ensure that Jest types will be preferred. + +::: + +You should be able to run your first test on the auto-generated test file via: + +```bash npm2yarn +npm run wdio +``` + +More information on setting up WebdriverIO can be found in their [documentation](https://webdriver.io/docs/component-testing/stencil). + +## Integration with Stencil + +If you have been using Stencil's test runner for unit or end-to-end tests to can continue to do so. For basic implementation details that don't require any web platform features, running tests through the Stencil test runner might still be the faster choice, since no browser needs to be spawned. However you can also migrate over to only one single framework one test at a time. + +We recommend to create a new NPM script for running both, Stencil and WebdriverIO tests, starting with Stencil tests first as they are likely to run faster. In your `package.json` this can be structured like so: + +```json title="package.json" +{ + "scripts:": { + "test.e2e": "stencil test && wdio run wdio.conf.ts" + } +} +``` + +Make sure that each test runner picks up their respective tests by defining the `testRegex` property in your Stencil config, e.g.: + +```ts title="stencil.config.ts" +import { Config } from '@stencil/core'; + +export const config: Config = { + // ... + testing: { + testRegex: '(/__tests__/.*|\\.?(spec))\\.(ts|js)$', + }, +}; +``` + +This will make Stencil pick up all files ending with `.spec.ts` or `.spec.js` while WebdriverIO picks up tests ending with `.test.ts`. \ No newline at end of file diff --git a/docs/testing/webdriverio/02-unit-testing.md b/docs/testing/webdriverio/02-unit-testing.md new file mode 100644 index 000000000..e86faf3f0 --- /dev/null +++ b/docs/testing/webdriverio/02-unit-testing.md @@ -0,0 +1,69 @@ +--- +title: Unit Testing +sidebar_label: Unit Testing +description: Unit Testing +slug: /testing/webdriverio/unit-testing +--- + +# Unit Testing + +WebdriverIO makes it easy to unit test components and app utility functions in the browser. Unit tests validate the code in isolation. Well written tests are fast, repeatable, and easy to reason about. It tries to follow a simple guiding principle: the more your tests resemble the way your software is used, the more confidence they can give you. + +### Test Setup + +For a test to resemble as much as possible how your component is used in an actual application we need to render it into an actual DOM tree. WebdriverIO provides a helper package for this that you can use called `@wdio/browser-runner/stencil`. It exports a `render` method that allows us to mount our component to the DOM. + +For example, given the following component: + +```ts reference title="/src/components/my-component/my-component.tsx" +https://github.com/webdriverio/component-testing-examples/blob/main/stencil-component-starter/src/components/my-component/my-component.tsx +``` + +We import the component into our test to render it in the browser: + +```ts reference title="/src/components/my-component/my-component.test.tsx" +https://github.com/webdriverio/component-testing-examples/blob/main/stencil-component-starter/src/components/my-component/my-component.test.tsx#L2-L18 +``` + +If your component under test uses other Stencil components make sure you add these to the `components` list as well. For example, let's say `ComponentA` uses `ComponentB` which also imports `ComponentC` and `ComponentD`. In this case you will have to import and pass in all components you'd like to have rendered, e.g.: + +```ts +render({ + components: [ComponentA, ComponentB, ComponentC, ComponentD], + template: () => +}); +``` + +While this seems tedious at first, it gives you the flexibility to leave out components that are not relevant for your test, or have side effects that can cause flakiness. + +Find more information about the `render` method option and its return object in the WebdriverIO [documentation](https://webdriver.io/docs/component-testing/stencil#render-options). + +### Handling Asynchronicity + +Instead of directly working on DOM objects, with WebdriverIO you interact with references to DOM nodes through asynchronous WebDriver commands. Make sure you always use an `await` to ensure that all commands and assertion are executed sequentially. + +:::info + +Missing an `await` can be a simple oversight and can cause us long hours of debugging. To avoid this and ensure promises are handled properly, it is recommended to use an ESLint rule called [`require-await`](https://eslint.org/docs/latest/rules/require-await). + +::: + +### Matchers + +WebdriverIO provides their own matchers to make various assertions about an element. We recommend these over synchronous matchers like `toBe` or `toEqual` as they allow for retries and make your tests more resilient against flakiness. + +For example, instead of asserting the content of a component like this: + +```ts +expect(await $('my-component').getText()) + .toBe(`Hello, World! I'm Stencil 'Don't call me a framework' JS`) +``` + +It is better to use WebdriverIOs matchers for asserting text: + +```ts +await expect($('my-component')) + .toHaveText(`Hello, World! I'm Stencil 'Don't call me a framework' JS`) +``` + +You can read more about WebdriverIOs specific matchers, in the project [documentation](https://webdriver.io/docs/api/expect-webdriverio). \ No newline at end of file diff --git a/docs/testing/webdriverio/03-mocking.md b/docs/testing/webdriverio/03-mocking.md new file mode 100644 index 000000000..057da10f8 --- /dev/null +++ b/docs/testing/webdriverio/03-mocking.md @@ -0,0 +1,34 @@ +--- +title: Mocking +sidebar_label: Mocking +description: Mocking +label: Mocking +--- + +# Mocking + +WebdriverIO has support for file based module mocking as well as mocking of entire dependencies of your project. The framework provides a set of primitives for mocking as documented in the project [documentation](https://webdriver.io/docs/component-testing/mocking): + +```ts +import { mock, fn, unmock } from '@wdio/browser-runner' +``` + +To create a mock you can either create a file with the name of the module you would like to mock in the `__mocks__` directory, as described in [Manual Mocks](https://webdriver.io/docs/component-testing/mocking#manual-mocks), or mock the file directly as part of your test: + +```ts +import { mock, fn } from '@wdio/browser-runner' +import { format } from './utils/utils.ts' + +// mock files within the project +mock('./utils/utils.ts', () => ({ + format: fn().mockReturnValue(42) +})) +// mock whole modules and replace functionality with what is defined in `./__mocks__/leftpad.ts` +mock('leftpad') + +console.log(format()) // returns `42` +``` + +Once a module is mocked, importing it either from your test or your component will give you the mocked version of the module and not the actual one. + +Find more examples and documentation on mocking in the project [documentation](https://webdriver.io/docs/component-testing/mocking). \ No newline at end of file diff --git a/docs/testing/webdriverio/04-visual-testing.md b/docs/testing/webdriverio/04-visual-testing.md new file mode 100644 index 000000000..41c45acb7 --- /dev/null +++ b/docs/testing/webdriverio/04-visual-testing.md @@ -0,0 +1,60 @@ +--- +title: Visual Testing +sidebar_label: Visual Testing +description: Visual Testing +slug: /testing/webdriverio/visual-testing +--- + +# Visual Testing + +WebdriverIO supports [visual testing capabilities](https://webdriver.io/docs/visual-testing) out of the box through a plugin called [`@wdio/visual-service`](https://www.npmjs.com/package/@wdio/visual-service). It uses [ResembleJS](https://github.com/Huddle/Resemble.js) under the hood to do pixel perfect comparisons. + +## Adding Visual Testing to your Setup + +If you don't have a WebdriverIO project set up yet, please take a look at the set up instructions we provide on the [WebdriverIO Overview](./01-overview.md) page. + +Once you are set up, add the visual plugin to your project via: + +```bash npm2yarn +npm install --save-dev @wdio/visual-service +``` + +A plugin, also called [service](https://webdriver.io/docs/customservices) in WebdriverIO, has access to a variety of test lifecycle hooks to enable new functionality or integrate with another platform. To use a service, add it to your services list in your `wdio.conf.ts`: + +```ts reference title="wdio.conf.ts" +https://github.com/webdriverio/component-testing-examples/blob/2de295ab568b5163e67d716156221578b6536d9d/stencil-component-starter/wdio.conf.ts#L119-L126) +``` + +As shown in the [Visual Testing](https://webdriver.io/docs/visual-testing/writing-tests/) WebdriverIO documentation, the service adds 4 new matchers to visually assert your application: + +- `toMatchScreenSnapshot`: captures and compares the whole browser screen +- `toMatchElementSnapshot`: captures and compares the visual difference within the element boundaries +- `toMatchFullPageSnapshot`: captures and compares the whole document +- `toMatchTabbablePageSnapshot`: same as `toMatchFullPageSnapshot` with tab marks for accessibility testing + +In the context of testing StencilJS components the best choice is to use `toMatchElementSnapshot` to verify a single component visually. Such a test may appear as follows: + +```ts reference title="src/components/my-component/my-component.test.tsx" +https://github.com/webdriverio/component-testing-examples/blob/2de295ab568b5163e67d716156221578b6536d9d/stencil-component-starter/src/components/my-component/my-component.test.tsx#L20-L28 +``` + +The screenshots will be generated locally and the baseline should be checked into your project, so that everyone running the tests visually, compare against the same assumptions. If a test is failing, e.g. we set the color of the text to a different color, WebdriverIO will let the test fail with the following message: + +``` +Expected image to have a mismatch percentage of 0%, but was 6.488% +Please compare the images manually and update the baseline if the new screenshot is correct. + +Baseline: /stencil-project/__snapshots__/MyComponent-chrome-1200x1551-dpr-2.png +Actual Screenshot: /stencil-project/__snapshots__/.tmp/actual/MyComponent-chrome-1200x1551-dpr-2.png +Difference: /stencil-project/__snapshots__/.tmp/diff/MyComponent-chrome-1200x1551-dpr-2.png + +See https://webdriver.io/docs/api/visual-regression.html for more information. +``` + +You can see the visual differences highlighted in `/stencil-project/__snapshots__/.tmp/diff` which can look as following: + +![Example of visual difference](/img/testing/diff-example.png) + +If you believe the visual changes are correct, update the baseline by moving the image from `stencil-project/__snapshots__/.tmp/actual` into the baseline directory. + +For further information on Visual Testing in WebdriverIO visit their [documentation page](https://webdriver.io/docs/visual-testing). \ No newline at end of file diff --git a/docs/testing/webdriverio/_category_.json b/docs/testing/webdriverio/_category_.json new file mode 100644 index 000000000..3aec28209 --- /dev/null +++ b/docs/testing/webdriverio/_category_.json @@ -0,0 +1,3 @@ +{ + "label": "WebdriverIO" +} diff --git a/docusaurus.config.js b/docusaurus.config.js index bb09227b7..87ec55132 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -67,6 +67,8 @@ const config = { ], ], + themes: ['docusaurus-theme-github-codeblock'], + themeConfig: /** @type {import('@ionic-docs/preset-classic').ThemeConfig} */ ({ @@ -219,6 +221,11 @@ const config = { contextualSearch: true, }, image: `https://${HOSTNAME}/stencil-og.png`, + codeblock: { + showGithubLink: true, + githubLinkLabel: 'View on GitHub', + showRunmeLink: false, + }, }), }; diff --git a/package-lock.json b/package-lock.json index 8679489de..bc039f6bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@docusaurus/remark-plugin-npm2yarn": "^3.1.1", "@ionic-docs/preset-classic": "^1.1.0", "clsx": "^2.0.0", + "docusaurus-theme-github-codeblock": "^2.0.2", "prism-react-renderer": "^2.3.0", "react": "^18.2.0", "react-dom": "^18.2.0" @@ -6933,6 +6934,14 @@ "sass": "^1.30.0" } }, + "node_modules/docusaurus-theme-github-codeblock": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/docusaurus-theme-github-codeblock/-/docusaurus-theme-github-codeblock-2.0.2.tgz", + "integrity": "sha512-H2WoQPWOLjGZO6KS58Gsd+eUVjTFJemkReiSSu9chqokyLc/3Ih3+zPRYfuEZ/HsDvSMIarf7CNcp+Vt+/G+ig==", + "dependencies": { + "@docusaurus/types": "^3.0.0" + } + }, "node_modules/dom-converter": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz", diff --git a/package.json b/package.json index 8be03cd1e..6b2341621 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@docusaurus/remark-plugin-npm2yarn": "^3.1.1", "@ionic-docs/preset-classic": "^1.1.0", "clsx": "^2.0.0", + "docusaurus-theme-github-codeblock": "^2.0.2", "prism-react-renderer": "^2.3.0", "react": "^18.2.0", "react-dom": "^18.2.0" diff --git a/static/img/testing/diff-example.png b/static/img/testing/diff-example.png new file mode 100644 index 000000000..6d85416c1 Binary files /dev/null and b/static/img/testing/diff-example.png differ