Merge pull request #120 from storybookjs/yann/sb-382-test-runner-test…

…-coverage-support

Support code coverage

.gitignore

@@ -6,4 +6,6 @@ build-storybook.log
 .env
 .cache
 stories/atoms/StressTest.stories.js
-yarn-error.log
+yarn-error.log
+.nyc_output
+coverage

.storybook/main.js

@@ -21,11 +21,11 @@ if (process.env.STRESS_TEST) {
 const addons = [
   process.env.WITHOUT_DOCS
     ? {
-        name: '@storybook/addon-essentials',
-        options: {
-          docs: false,
-        },
-      }
+      name: '@storybook/addon-essentials',
+      options: {
+        docs: false,
+      },
+    }
     : '@storybook/addon-essentials',
   '@storybook/addon-interactions',
 ];
@@ -37,6 +37,20 @@ module.exports = {
     storyStoreV7: process.env.STORY_STORE_V7 ? true : false,
     buildStoriesJson: true,
   },
+  babel: async (options) => {
+    options.plugins.push([
+      'istanbul',
+      {
+        include: ['stories/**'],
+        exclude: [
+          '**/*.d.ts',
+          '**/*{.,-}{spec,stories,types}.{js,jsx,ts,tsx}',
+        ],
+      },
+    ])
+
+    return options
+  },
   core: {
     disableTelemetry: true
   }

README.md

@@ -15,11 +15,18 @@ Read the announcement: [Interaction Testing with Storybook](https://storybook.js
 - [Running in CI](#running-in-ci)
   - [1. Running against deployed Storybooks on Github Actions deployment](#1-running-against-deployed-storybooks-on-github-actions-deployment)
   - [2. Running against locally built Storybooks in CI](#2-running-against-locally-built-storybooks-in-ci)
+- [Setting up code coverage](#setting-up-code-coverage)
+  - [Instrument the code](#instrument-the-code)
+  - [Run tests with `--coverage` flag](#run-tests-with---coverage-flag)
 - [Experimental test hook API](#experimental-test-hook-api)
   - [Image snapshot recipe](#image-snapshot-recipe)
   - [Render lifecycle](#render-lifecycle)
 - [Troubleshooting](#troubleshooting)
+  - [Errors with Jest 28](#errors-with-jest-28)
+  - [The error output in the CLI is too short](#the-error-output-in-the-cli-is-too-short)
   - [The test runner seems flaky and keeps timing out](#the-test-runner-seems-flaky-and-keeps-timing-out)
+  - [The test runner reports "No tests found" running on a Windows CI](#the-test-runner-reports-"no-tests-found"-running-on-a-windows-ci)
+  - [The test runner does not show reports in watch mode](#the-test-runner-does-not-show-reports-in-watch-mode)
   - [Adding the test runner to other CI environments](#adding-the-test-runner-to-other-ci-environments)
 - [Future work](#future-work)
 
@@ -33,6 +40,7 @@ Read the announcement: [Interaction Testing with Storybook](https://storybook.js
 - 🐛 Debug them visually and interactively in a live browser with [addon-interactions](https://storybook.js.org/docs/react/essentials/interactions)
 - 🎭 Powered by [Jest](https://jestjs.io/) and [Playwright](https://playwright.dev/)
 - 👀 Watch mode, filters, and the conveniences you'd expect
+- 📔 Code coverage reports
 
 ## Getting started
 
@@ -111,6 +119,7 @@ Usage: test-storybook [options]
 | `--no-stories-json`             | Disables stories json mode <br/>`test-storybook --no-stories-json`                                                               |
 | `-c`, `--config-dir [dir-name]` | Directory where to load Storybook configurations from <br/>`test-storybook -c .storybook`                                        |
 | `--watch`                       | Run in watch mode <br/>`test-storybook --watch`                                                                                  |
+| `--coverage`                    | Indicates that test coverage information should be collected and reported in the output <br/>`test-storybook --coverage`         |
 | `--url`                         | Define the URL to run tests in. Useful for custom Storybook URLs <br/>`test-storybook --url http://the-storybook-url-here.com`   |
 | `--browsers`                    | Define browsers to run tests in. One or multiple of: chromium, firefox, webkit <br/>`test-storybook --browsers firefox chromium` |
 | `--maxWorkers [amount]`         | Specifies the maximum number of workers the worker-pool will spawn for running tests <br/>`test-storybook --maxWorkers=2`        |
@@ -251,6 +260,71 @@ jobs:
 
 > **_NOTE:_** Building Storybook locally makes it simple to test Storybooks that could be available remotely, but are under authentication layers. If you also deploy your Storybooks somewhere (e.g. Chromatic, Vercel, etc.), the Storybook URL can still be useful with the test-runner. You can pass it to the `REFERENCE_URL` environment variable when running the test-storybook command, and if a story fails, the test-runner will provide a helpful message with the link to the story in your published Storybook instead.
 
+## Setting up code coverage
+
+The test runner supports code coverage with the `--coverage` flag or `STORYBOOK_COLLECT_COVERAGE` environment variable. The pre-requisite is that your components are instrumented using [istanbul](https://istanbul.js.org/).
+
+### Instrument the code
+
+Given that your components' code runs in the context of a real browser, they have to be instrumented so that the test runner is able to collect coverage. In order to do so, you have to setup the instrumentation yourself.
+
+Install the istanbul babel plugin:
+
+```sh
+yarn add -D babel-plugin-istanbul
+```
+
+Storybook allows code transpilation with babel out of the box by configuring the `babel` function in your `main.js`. Add the `istanbul` plugin:
+
+```js
+// .storybook/main.js
+module.exports = {
+  // ...rest of your code here
+  babel: async (options) => {
+    options.plugins.push([
+      'istanbul',
+      {
+        // provide include patterns if you like
+        include: ['src/components/**'],
+        // provide exclude patterns if you like
+        exclude: [
+          '**/*.d.ts',
+          '**/*{.,-}{spec,stories,types}.{js,jsx,ts,tsx}',
+        ],
+      },
+    ]);
+
+    return options;
+  },
+};
+```
+
+The babel plugin has default options that might suffice to your project, however if you want to know which options are taken into account you can check them [here](https://github.com/istanbuljs/babel-plugin-istanbul/blob/master/src/index.js#L82).
+
+### Run tests with --coverage flag
+
+After setting up instrumentation, run Storybook then run the test-runner with `--coverage`:
+
+```sh
+yarn test-storybook --coverage
+```
+
+The test runner will report the results in the CLI and generate a `.nyc_output/coverage.json` file which can be used by `nyc`.
+
+![](.github/assets/coverage-result.png)
+
+Notice that it provides a message telling you that you can get a better, interactive summary of your code by running:
+
+```
+npx nyc report --reporter=lcov
+```
+
+This will generate a folder called `coverage`, containing an `index.html` file which can be explored and will show the coverage in detail:
+
+![](.github/assets/coverage-report-html.png)
+
+`nyc` is a dependency of the test runner so you will already have it in your project. In the example above, the `lcov` reporter was used, which generates an output compatible with tools like [Codecov](https://codecov.io/). However, you can configure it to generate different reports, and you can find more information [here](https://istanbul.js.org/docs/advanced/alternative-reporters/).
+
 ## Experimental test hook API
 
 The test runner renders a story and executes its [play function](https://storybook.js.org/docs/react/writing-stories/play-function) if one exists. However, there are certain behaviors that are not possible to achieve via the play function, which executes in the browser. For example, if you want the test runner to take visual snapshots for you, this is something that is possible via Playwright/Jest, but must be executed in Node.
@@ -409,6 +483,10 @@ env:
   TEMP: ${{ runner.temp }}
 ```
 
+#### The test runner does not show reports in watch mode
+
+Because the displaying of reports and the underlying Jest process are separate, the reports can't be show in watch mode. However, the `.nyc_output/coverage.json` file is being generated, and you can show the reports by running `npx nyc report` in a separate terminal.
+
 #### Adding the test runner to other CI environments
 
 As the test runner is based on playwright, depending on your CI setup you might need to use specific docker images or other configuration. In that case, you can refer to the [Playwright CI docs](https://playwright.dev/docs/ci) for more information.
@@ -417,5 +495,4 @@ As the test runner is based on playwright, depending on your CI setup you might
 
 Future plans involve adding support for the following features:
 
-- 🧪 Custom test functions
 - 📄 Run addon reports

bin/test-storybook.js

@@ -2,6 +2,7 @@
 //@ts-check
 'use strict';
 
+const { execSync } = require('child_process');
 const fetch = require('node-fetch');
 const isLocalhostIp = require('is-localhost-ip');
 const fs = require('fs');
@@ -87,6 +88,14 @@ async function executeJestPlaywright(args) {
   await jest.run(argv);
 }
 
+async function printCoverageReport() {
+  // --skip-full in case we only want to show not fully covered code
+  // --check-coverage if we want to break if coverage reaches certain threshold
+  // idea: pass configuration object for thresholds https://www.npmjs.com/package/nyc#coverage-thresholds
+  execSync('npx nyc report', { stdio: 'inherit' })
+  log('For a better, interactive summary of coverage, run: \nnpx nyc report --reporter=lcov\n')
+}
+
 async function checkStorybook(url) {
   try {
     const res = await fetch(url, { method: 'HEAD' });
@@ -162,6 +171,10 @@ const main = async () => {
 
   process.env.TARGET_URL = targetURL;
 
+  if (runnerOptions.coverage) {
+    process.env.STORYBOOK_COLLECT_COVERAGE = 'true';
+  }
+
   if (process.env.REFERENCE_URL) {
     process.env.REFERENCE_URL = sanitizeURL(process.env.REFERENCE_URL);
   }
@@ -198,6 +211,10 @@ const main = async () => {
   }
 
   await executeJestPlaywright(jestOptions);
+
+  if (process.env.STORYBOOK_COLLECT_COVERAGE === 'true') {
+    printCoverageReport();
+  }
 };
 
 main().catch((e) => log(e));

package.json

@@ -74,6 +74,7 @@
     "auto": "^10.3.0",
     "babel-jest": "^27.0.6",
     "babel-loader": "^8.1.0",
+    "babel-plugin-istanbul": "^6.1.1",
     "concurrently": "^7.0.0",
     "jest": "^27.0.6",
     "jest-image-snapshot": "^4.5.1",

playwright/jest-setup.js

@@ -12,3 +12,5 @@ if (testRunnerConfig) {
     setPostRender(testRunnerConfig.postRender);
   }
 }
+
+global.__sbCollectCoverage = process.env.STORYBOOK_COLLECT_COVERAGE === 'true'

src/config/jest-playwright.ts

@@ -1,5 +1,11 @@
 export const getJestConfig = () => {
-  const { TEST_ROOT, TEST_MATCH, STORYBOOK_STORIES_PATTERN, TEST_BROWSERS } = process.env;
+  const {
+    TEST_ROOT,
+    TEST_MATCH,
+    STORYBOOK_STORIES_PATTERN,
+    TEST_BROWSERS,
+    STORYBOOK_COLLECT_COVERAGE,
+  } = process.env;
 
   let config = {
     rootDir: process.cwd(),
@@ -20,12 +26,14 @@ export const getJestConfig = () => {
         browsers: TEST_BROWSERS.split(',')
           .map((p) => p.trim().toLowerCase())
           .filter(Boolean),
+        collectCoverage: STORYBOOK_COLLECT_COVERAGE === 'true',
       },
     },
     watchPlugins: [
       require.resolve('jest-watch-typeahead/filename'),
       require.resolve('jest-watch-typeahead/testname'),
     ],
+    watchPathIgnorePatterns: ['coverage', '.nyc_output', '.cache'],
   };
 
   if (TEST_MATCH) {

src/playwright/transformPlaywright.test.ts

@@ -88,6 +88,10 @@ describe('Playwright', () => {
                   await global.__sbPostRender(page, context);
                 }
         
+                if (global.__sbCollectCoverage) {
+                  await jestPlaywright.saveCoverage(page);
+                }
+        
                 return result;
               };
         
@@ -159,6 +163,10 @@ describe('Playwright', () => {
                   await global.__sbPostRender(page, context);
                 }
         
+                if (global.__sbCollectCoverage) {
+                  await jestPlaywright.saveCoverage(page);
+                }
+        
                 return result;
               };
         
@@ -231,6 +239,10 @@ describe('Playwright', () => {
                   await global.__sbPostRender(page, context);
                 }
         
+                if (global.__sbCollectCoverage) {
+                  await jestPlaywright.saveCoverage(page);
+                }
+        
                 return result;
               };
         

src/playwright/transformPlaywright.ts

@@ -33,6 +33,10 @@ export const testPrefixer = template(
           await global.__sbPostRender(page, context);
         }
 
+        if(global.__sbCollectCoverage) {
+          await jestPlaywright.saveCoverage(page);
+        }
+
         return result;
       };
 

src/playwright/transformPlaywrightJson.test.ts

@@ -83,6 +83,10 @@ describe('Playwright Json', () => {
                 await global.__sbPostRender(page, context);
               }
 
+              if (global.__sbCollectCoverage) {
+                await jestPlaywright.saveCoverage(page);
+              }
+
               return result;
             };
 
@@ -132,6 +136,10 @@ describe('Playwright Json', () => {
                 await global.__sbPostRender(page, context);
               }
 
+              if (global.__sbCollectCoverage) {
+                await jestPlaywright.saveCoverage(page);
+              }
+
               return result;
             };
 
@@ -183,6 +191,10 @@ describe('Playwright Json', () => {
                 await global.__sbPostRender(page, context);
               }
 
+              if (global.__sbCollectCoverage) {
+                await jestPlaywright.saveCoverage(page);
+              }
+
               return result;
             };
 
@@ -272,6 +284,10 @@ describe('Playwright Json', () => {
                 await global.__sbPostRender(page, context);
               }
 
+              if (global.__sbCollectCoverage) {
+                await jestPlaywright.saveCoverage(page);
+              }
+
               return result;
             };
 

src/util/getCliOptions.ts

@@ -7,6 +7,7 @@ type CliOptions = {
     url?: string;
     configDir?: string;
     eject?: boolean;
+    coverage?: boolean;
     browsers?: BrowserType | BrowserType[];
   };
   jestOptions: string[];
@@ -20,6 +21,7 @@ const STORYBOOK_RUNNER_COMMANDS: StorybookRunnerCommand[] = [
   'browsers',
   'eject',
   'url',
+  'coverage',
 ];
 
 export const getCliOptions = () => {

src/util/getParsedCliOptions.ts

@@ -34,6 +34,10 @@ export const getParsedCliOptions = () => {
       '-u, --updateSnapshot',
       'Use this flag to re-record every snapshot that fails during this test run'
     )
+    .option(
+      '--coverage',
+      'Indicates that test coverage information should be collected and reported in the output'
+    )
     .option(
       '--eject',
       'Creates a local configuration file to override defaults of the test-runner. Use it only if you want to have better control over the runner configurations'