diff --git a/.github/assets/coverage-result.png b/.github/assets/coverage-result.png index 7787afcc..60be71b0 100644 Binary files a/.github/assets/coverage-result.png and b/.github/assets/coverage-result.png differ diff --git a/.github/workflows/test.yml b/.github/workflows/tests-extended.yml similarity index 73% rename from .github/workflows/test.yml rename to .github/workflows/tests-extended.yml index 8eb80ec6..f0f0fc99 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/tests-extended.yml @@ -1,4 +1,4 @@ -name: Test +name: Tests-extended on: [push] @@ -16,15 +16,6 @@ jobs: - name: Install dependencies uses: bahmutov/npm-install@v1 - - name: Run jest tests - run: | - yarn test - - - name: Run test runner (default config) - run: | - yarn build - yarn test-storybook:ci - - name: Run test runner (story store v7) run: | yarn build diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..3f7ed98f --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,31 @@ +name: Tests + +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Use Node.js 14.x + uses: actions/setup-node@v1 + with: + node-version: 14.x + + - name: Install dependencies + uses: bahmutov/npm-install@v1 + + - name: Run jest tests + run: | + yarn test --coverage + + - name: Run test runner + run: | + yarn build + yarn test-storybook:ci-coverage + + - name: Generate code coverage + uses: codecov/codecov-action@v2 + with: + verbose: true diff --git a/.storybook/main.js b/.storybook/main.js index 2b6c85c8..e472b171 100644 --- a/.storybook/main.js +++ b/.storybook/main.js @@ -28,6 +28,7 @@ const addons = [ } : '@storybook/addon-essentials', '@storybook/addon-interactions', + '@storybook/addon-coverage', ]; module.exports = { @@ -37,20 +38,6 @@ module.exports = { storyStoreV7: process.env.STORY_STORE_V7 ? true : false, buildStoriesJson: true, }, - babel: async (options) => { - options.plugins.push([ - 'istanbul', - { - include: ['stories/**'], - exclude: [ - '**/*.d.ts', - '**/*{.,-}{spec,stories,types}.{js,jsx,ts,tsx}', - ], - }, - ]) - - return options - }, core: { disableTelemetry: true } diff --git a/.storybook/test-runner.ts b/.storybook/test-runner.ts index c624eaee..76d89258 100644 --- a/.storybook/test-runner.ts +++ b/.storybook/test-runner.ts @@ -14,7 +14,6 @@ const config: TestRunnerConfig = { const { parameters } = await getStoryContext(page, context); if (parameters?.tests?.disableSnapshots) { - console.log('skipping story ', context.id); return; } diff --git a/README.md b/README.md index 3b3fcd5c..ccd9b970 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,10 @@ Read the announcement: [Interaction Testing with Storybook](https://storybook.js - [1. Running against deployed Storybooks on Github Actions deployment](#1-running-against-deployed-storybooks-on-github-actions-deployment) - [2. Running against locally built Storybooks in CI](#2-running-against-locally-built-storybooks-in-ci) - [Setting up code coverage](#setting-up-code-coverage) - - [Instrument the code](#instrument-the-code) - - [Run tests with `--coverage` flag](#run-tests-with---coverage-flag) + - [1 - Instrument the code](#1---instrument-the-code) + - [Using @storybook/addon-coverage](#using-storybookaddon-coverage) + - [Manually configuring Istanbul](#manually-configuring-istanbul) + - [2 - Run tests with `--coverage` flag](#2---run-tests-with---coverage-flag) - [Experimental test hook API](#experimental-test-hook-api) - [Image snapshot recipe](#image-snapshot-recipe) - [Render lifecycle](#render-lifecycle) @@ -26,7 +28,6 @@ Read the announcement: [Interaction Testing with Storybook](https://storybook.js - [The error output in the CLI is too short](#the-error-output-in-the-cli-is-too-short) - [The test runner seems flaky and keeps timing out](#the-test-runner-seems-flaky-and-keeps-timing-out) - [The test runner reports "No tests found" running on a Windows CI](#the-test-runner-reports-"no-tests-found"-running-on-a-windows-ci) - - [The test runner does not show reports in watch mode](#the-test-runner-does-not-show-reports-in-watch-mode) - [Adding the test runner to other CI environments](#adding-the-test-runner-to-other-ci-environments) - [Future work](#future-work) @@ -264,44 +265,39 @@ jobs: The test runner supports code coverage with the `--coverage` flag or `STORYBOOK_COLLECT_COVERAGE` environment variable. The pre-requisite is that your components are instrumented using [istanbul](https://istanbul.js.org/). -### Instrument the code +### 1 - Instrument the code -Given that your components' code runs in the context of a real browser, they have to be instrumented so that the test runner is able to collect coverage. In order to do so, you have to setup the instrumentation yourself. +Given that your components' code runs in the context of a real browser, they have to be instrumented so that the test runner is able to collect coverage. This is done by configuring [istanbul](https://istanbul.js.org/) in your Storybook. You can achieve that in two different ways: -Install the istanbul babel plugin: +#### Using @storybook/addon-coverage + +For select frameworks (React, Preact, HTML, Web components and Vue) you can use the [@storybook/addon-coverage](https://github.com/storybookjs/addon-coverage) addon, which will automatically configure the plugin for you. + +Install `@storybook/addon-coverage`: ```sh -yarn add -D babel-plugin-istanbul +yarn add -D @storybook/addon-coverage ``` -Storybook allows code transpilation with babel out of the box by configuring the `babel` function in your `main.js`. Add the `istanbul` plugin: +And register it in your `.storybook/main.js` file: ```js // .storybook/main.js module.exports = { // ...rest of your code here - babel: async (options) => { - options.plugins.push([ - 'istanbul', - { - // provide include patterns if you like - include: ['src/components/**'], - // provide exclude patterns if you like - exclude: [ - '**/*.d.ts', - '**/*{.,-}{spec,stories,types}.{js,jsx,ts,tsx}', - ], - }, - ]); - - return options; - }, + addons: [ + "@storybook/addon-coverage", + ] }; ``` -The babel plugin has default options that might suffice to your project, however if you want to know which options are taken into account you can check them [here](https://github.com/istanbuljs/babel-plugin-istanbul/blob/master/src/index.js#L82). +The addon has default options that might suffice to your project, however if you want to customize the addon you can see how it's done [here](https://github.com/storybookjs/addon-coverage#configuring-the-addon). -### Run tests with --coverage flag +#### Manually configuring istanbul + +Some frameworks or Storybook builders might not automatically accept babel plugins. In that case, you will have to manually configure whatever flavor of [istanbul](https://istanbul.js.org/) (rollup, vite, webpack loader) your project might require. + +### 2 - Run tests with --coverage flag After setting up instrumentation, run Storybook then run the test-runner with `--coverage`: @@ -309,21 +305,48 @@ After setting up instrumentation, run Storybook then run the test-runner with `- yarn test-storybook --coverage ``` -The test runner will report the results in the CLI and generate a `.nyc_output/coverage.json` file which can be used by `nyc`. +The test runner will report the results in the CLI and generate a `coverage/storybook/coverage-storybook.json` file which can be used by `nyc`. ![](.github/assets/coverage-result.png) -Notice that it provides a message telling you that you can get a better, interactive summary of your code by running: +If you want to generate reports with [different reporters](https://istanbul.js.org/docs/advanced/alternative-reporters/), you can use `nyc` and point it to the folder which contains the Storybook coverage file. `nyc` is a dependency of the test runner so you will already have it in your project. + +Here's an example generating an `lcov` report: ``` -npx nyc report --reporter=lcov +npx nyc report --reporter=lcov -t coverage/storybook --report-dir coverage/storybook ``` -This will generate a folder called `coverage`, containing an `index.html` file which can be explored and will show the coverage in detail: +This will generate a more detailed, interactive coverage summary that you can access at `coverage/storybook/index.html` file which can be explored and will show the coverage in detail: ![](.github/assets/coverage-report-html.png) -`nyc` is a dependency of the test runner so you will already have it in your project. In the example above, the `lcov` reporter was used, which generates an output compatible with tools like [Codecov](https://codecov.io/). However, you can configure it to generate different reports, and you can find more information [here](https://istanbul.js.org/docs/advanced/alternative-reporters/). +The `nyc` command will respect [nyc configuration files](https://github.com/istanbuljs/nyc#common-configuration-options) if you have them in your project. + +If you want certain parts of your code to be deliberately ignored, you can use istanbul [parsing hints](https://github.com/istanbuljs/nyc#parsing-hints-ignoring-lines). + +### 3 - Merging code coverage with coverage from other tools + +The test runner reports coverage related to the `coverage/storybook/coverage-storybook.json` file. This is by design, showing you the coverage which is tested while running Storybook. + +Now, you might have other tests (e.g. unit tests) which are _not_ covered in Storybook but are covered when running tests with Jest, which you might also generate coverage files from, for instance. In such cases, if you are using tools like [Codecov](https://codecov.io/) to automate reporting, the coverage files will be detected automatically and if there are multiple files in the coverage folder, they will be merged automatically. + +Alternatively, in case you want to merge coverages from other tools, you should: + +1 - move or copy the `coverage/storybook/coverage-storybook.json` into `coverage/coverage-storybook.json`; +2 - run `nyc report` against the `coverage` folder. + +Here's an example on how to achieve that: + +```json +{ + "scripts": { + "test:coverage": "jest --coverage", + "test-storybook:coverage": "test-storybook --coverage", + "coverage-report": "cp coverage/storybook/coverage-storybook.json coverage/coverage-storybook.json && nyc report --reporter=html -t coverage --report-dir coverage" + } +} +``` ## Experimental test hook API @@ -483,10 +506,6 @@ env: TEMP: ${{ runner.temp }} ``` -#### The test runner does not show reports in watch mode - -Because the displaying of reports and the underlying Jest process are separate, the reports can't be show in watch mode. However, the `.nyc_output/coverage.json` file is being generated, and you can show the reports by running `npx nyc report` in a separate terminal. - #### Adding the test runner to other CI environments As the test runner is based on playwright, depending on your CI setup you might need to use specific docker images or other configuration. In that case, you can refer to the [Playwright CI docs](https://playwright.dev/docs/ci) for more information. diff --git a/bin/test-storybook.js b/bin/test-storybook.js index a514050e..47ac6908 100755 --- a/bin/test-storybook.js +++ b/bin/test-storybook.js @@ -35,11 +35,47 @@ const cleanup = () => { log(`Cleaning up ${storiesJsonTmpDir}`); fs.rmSync(storiesJsonTmpDir, { recursive: true, force: true }); } - process.exit(); }; -process.on('SIGINT', cleanup); -process.on('beforeExit', cleanup); +let isWatchMode = false; +async function reportCoverage() { + if (isWatchMode || process.env.STORYBOOK_COLLECT_COVERAGE !== 'true') { + return + } + + const coverageFolderE2E = path.resolve(process.cwd(), '.nyc_output'); + const coverageFolder = path.resolve(process.cwd(), 'coverage/storybook'); + + // in case something goes wrong and .nyc_output does not exist, bail + if (!fs.existsSync(coverageFolderE2E)) { + return + } + + // if there's no coverage folder, create one + if (!fs.existsSync(coverageFolder)) { + fs.mkdirSync(coverageFolder, { recursive: true }); + } + + // move the coverage files from .nyc_output folder (coming from jest-playwright) to coverage, then delete .nyc_output + fs.renameSync( + `${coverageFolderE2E}/coverage.json`, + `${coverageFolder}/coverage-storybook.json`, + ); + fs.rmSync(coverageFolderE2E, { recursive: true }); + + // --skip-full in case we only want to show not fully covered code + // --check-coverage if we want to break if coverage reaches certain threshold + // .nycrc will be respected for thresholds etc. https://www.npmjs.com/package/nyc#coverage-thresholds + execSync(`npx nyc report --reporter=text -t ${coverageFolder} --report-dir ${coverageFolder}`, { stdio: 'inherit' }) +} + +const onProcessEnd = () => { + cleanup(); + reportCoverage(); +} + +process.on('SIGINT', onProcessEnd); +process.on('exit', onProcessEnd); function sanitizeURL(url) { let finalURL = url; @@ -88,14 +124,6 @@ async function executeJestPlaywright(args) { await jest.run(argv); } -async function printCoverageReport() { - // --skip-full in case we only want to show not fully covered code - // --check-coverage if we want to break if coverage reaches certain threshold - // idea: pass configuration object for thresholds https://www.npmjs.com/package/nyc#coverage-thresholds - execSync('npx nyc report', { stdio: 'inherit' }) - log('For a better, interactive summary of coverage, run: \nnpx nyc report --reporter=lcov\n') -} - async function checkStorybook(url) { try { const res = await fetch(url, { method: 'HEAD' }); @@ -166,6 +194,9 @@ const main = async () => { process.exit(0); } + // set this flag to skip reporting coverage in watch mode + isWatchMode = jestOptions.watch; + const targetURL = sanitizeURL(process.env.TARGET_URL || runnerOptions.url); await checkStorybook(targetURL); @@ -211,10 +242,6 @@ const main = async () => { } await executeJestPlaywright(jestOptions); - - if (process.env.STORYBOOK_COLLECT_COVERAGE === 'true') { - printCoverageReport(); - } }; main().catch((e) => log(e)); diff --git a/package.json b/package.json index 4cc26a1e..ebe01f9a 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "test-storybook:no-cache": "yarn test-storybook --no-cache", "test-storybook:json": "yarn test-storybook --stories-json", "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook\"", + "test-storybook:ci-coverage": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook --coverage\"", "test-storybook:ci-json": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:json\"", "generate-dynamic-stories": "node scripts/generate-dynamic-stories.js" }, @@ -60,6 +61,7 @@ "@babel/template": "^7.14.5", "@babel/types": "^7.14.8", "@jest/types": "^27.0.6", + "@storybook/addon-coverage": "^0.0.1", "@storybook/addon-essentials": "^6.5.0", "@storybook/addon-interactions": "^6.5.0", "@storybook/jest": "^0.0.10", diff --git a/yarn.lock b/yarn.lock index f7c4a4ab..1dd47e1e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2055,6 +2055,14 @@ lodash "^4.17.21" ts-dedent "^2.0.0" +"@storybook/addon-coverage@^0.0.1": + version "0.0.1" + resolved "https://registry.yarnpkg.com/@storybook/addon-coverage/-/addon-coverage-0.0.1.tgz#0eaf343a403c8a12a40839b40268bf51defef06f" + integrity sha512-Ml7GYmksZWqsXJhYig75UlFqEj3SMeKhLyMaHi0r6jCPdPSbl+osmWgmXnzQwdlqkhm7NrAIK+NYFVW/uTcMXA== + dependencies: + "@types/babel__core" "^7.1.19" + babel-plugin-istanbul "^6.1.1" + "@storybook/addon-docs@6.5.0": version "6.5.0" resolved "https://registry.yarnpkg.com/@storybook/addon-docs/-/addon-docs-6.5.0.tgz#54457ddebbe87f40738503f6c939fac44a982926" @@ -3022,7 +3030,7 @@ resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-4.2.2.tgz#ed4e0ad92306a704f9fb132a0cfcf77486dbe2bc" integrity sha512-HnYpAE1Y6kRyKM/XkEuiRQhTHvkzMBurTHnpFLYLBGPIylZNPs9jJcuOOYWxPLJCSEtmZT0Y8rHDokKN7rRTig== -"@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14": +"@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14", "@types/babel__core@^7.1.19": version "7.1.19" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.19.tgz#7b497495b7d1b4812bdb9d02804d0576f43ee460" integrity sha512-WEOTgRsbYkvA/KCsDwVEGkd7WAr1e3g31VHQ8zy5gul/V1qKullU/BU5I68X5v7V3GnB9eotmom4v5a5gjxorw==