Merge pull request #130 from storybookjs/feat/code-coverage-merge

Improve code coverage for coverage merging

.github/workflows/test.yml → .github/workflows/tests-extended.yml

@@ -1,4 +1,4 @@
-name: Test
+name: Tests-extended
 
 on: [push]
 
@@ -16,15 +16,6 @@ jobs:
       - name: Install dependencies
         uses: bahmutov/npm-install@v1
 
-      - name: Run jest tests
-        run: |
-          yarn test
-
-      - name: Run test runner (default config)
-        run: |
-          yarn build
-          yarn test-storybook:ci
-
       - name: Run test runner (story store v7)
         run: |
           yarn build

.github/workflows/tests.yml

@@ -0,0 +1,31 @@
+name: Tests
+
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+
+      - name: Install dependencies
+        uses: bahmutov/npm-install@v1
+
+      - name: Run jest tests
+        run: |
+          yarn test --coverage
+
+      - name: Run test runner
+        run: |
+          yarn build
+          yarn test-storybook:ci-coverage
+      
+      - name: Generate code coverage
+        uses: codecov/codecov-action@v2
+        with:
+          verbose: true

.storybook/main.js

@@ -28,6 +28,7 @@ const addons = [
     }
     : '@storybook/addon-essentials',
   '@storybook/addon-interactions',
+  '@storybook/addon-coverage',
 ];
 
 module.exports = {
@@ -37,20 +38,6 @@ module.exports = {
     storyStoreV7: process.env.STORY_STORE_V7 ? true : false,
     buildStoriesJson: true,
   },
-  babel: async (options) => {
-    options.plugins.push([
-      'istanbul',
-      {
-        include: ['stories/**'],
-        exclude: [
-          '**/*.d.ts',
-          '**/*{.,-}{spec,stories,types}.{js,jsx,ts,tsx}',
-        ],
-      },
-    ])
-
-    return options
-  },
   core: {
     disableTelemetry: true
   }

.storybook/test-runner.ts

@@ -14,7 +14,6 @@ const config: TestRunnerConfig = {
     const { parameters } = await getStoryContext(page, context);
 
     if (parameters?.tests?.disableSnapshots) {
-      console.log('skipping story ', context.id);
       return;
     }
 

README.md

@@ -16,8 +16,10 @@ Read the announcement: [Interaction Testing with Storybook](https://storybook.js
   - [1. Running against deployed Storybooks on Github Actions deployment](#1-running-against-deployed-storybooks-on-github-actions-deployment)
   - [2. Running against locally built Storybooks in CI](#2-running-against-locally-built-storybooks-in-ci)
 - [Setting up code coverage](#setting-up-code-coverage)
-  - [Instrument the code](#instrument-the-code)
-  - [Run tests with `--coverage` flag](#run-tests-with---coverage-flag)
+  - [1 - Instrument the code](#1---instrument-the-code)
+    - [Using @storybook/addon-coverage](#using-storybookaddon-coverage)
+    - [Manually configuring Istanbul](#manually-configuring-istanbul)
+  - [2 - Run tests with `--coverage` flag](#2---run-tests-with---coverage-flag)
 - [Experimental test hook API](#experimental-test-hook-api)
   - [Image snapshot recipe](#image-snapshot-recipe)
   - [Render lifecycle](#render-lifecycle)
@@ -26,7 +28,6 @@ Read the announcement: [Interaction Testing with Storybook](https://storybook.js
   - [The error output in the CLI is too short](#the-error-output-in-the-cli-is-too-short)
   - [The test runner seems flaky and keeps timing out](#the-test-runner-seems-flaky-and-keeps-timing-out)
   - [The test runner reports "No tests found" running on a Windows CI](#the-test-runner-reports-"no-tests-found"-running-on-a-windows-ci)
-  - [The test runner does not show reports in watch mode](#the-test-runner-does-not-show-reports-in-watch-mode)
   - [Adding the test runner to other CI environments](#adding-the-test-runner-to-other-ci-environments)
 - [Future work](#future-work)
 
@@ -264,66 +265,88 @@ jobs:
 
 The test runner supports code coverage with the `--coverage` flag or `STORYBOOK_COLLECT_COVERAGE` environment variable. The pre-requisite is that your components are instrumented using [istanbul](https://istanbul.js.org/).
 
-### Instrument the code
+### 1 - Instrument the code
 
-Given that your components' code runs in the context of a real browser, they have to be instrumented so that the test runner is able to collect coverage. In order to do so, you have to setup the instrumentation yourself.
+Given that your components' code runs in the context of a real browser, they have to be instrumented so that the test runner is able to collect coverage. This is done by configuring [istanbul](https://istanbul.js.org/) in your Storybook. You can achieve that in two different ways: 
 
-Install the istanbul babel plugin:
+#### Using @storybook/addon-coverage
+
+For select frameworks (React, Preact, HTML, Web components and Vue) you can use the [@storybook/addon-coverage](https://github.com/storybookjs/addon-coverage) addon, which will automatically configure the plugin for you.
+
+Install `@storybook/addon-coverage`:
 
 ```sh
-yarn add -D babel-plugin-istanbul
+yarn add -D @storybook/addon-coverage
 ```
 
-Storybook allows code transpilation with babel out of the box by configuring the `babel` function in your `main.js`. Add the `istanbul` plugin:
+And register it in your `.storybook/main.js` file:
 
 ```js
 // .storybook/main.js
 module.exports = {
   // ...rest of your code here
-  babel: async (options) => {
-    options.plugins.push([
-      'istanbul',
-      {
-        // provide include patterns if you like
-        include: ['src/components/**'],
-        // provide exclude patterns if you like
-        exclude: [
-          '**/*.d.ts',
-          '**/*{.,-}{spec,stories,types}.{js,jsx,ts,tsx}',
-        ],
-      },
-    ]);
-
-    return options;
-  },
+  addons: [
+    "@storybook/addon-coverage",
+  ]
 };
 ```
 
-The babel plugin has default options that might suffice to your project, however if you want to know which options are taken into account you can check them [here](https://github.com/istanbuljs/babel-plugin-istanbul/blob/master/src/index.js#L82).
+The addon has default options that might suffice to your project, however if you want to customize the addon you can see how it's done [here](https://github.com/storybookjs/addon-coverage#configuring-the-addon).
 
-### Run tests with --coverage flag
+#### Manually configuring istanbul
+
+Some frameworks or Storybook builders might not automatically accept babel plugins. In that case, you will have to manually configure whatever flavor of [istanbul](https://istanbul.js.org/) (rollup, vite, webpack loader) your project might require.
+
+### 2 - Run tests with --coverage flag
 
 After setting up instrumentation, run Storybook then run the test-runner with `--coverage`:
 
 ```sh
 yarn test-storybook --coverage
 ```
 
-The test runner will report the results in the CLI and generate a `.nyc_output/coverage.json` file which can be used by `nyc`.
+The test runner will report the results in the CLI and generate a `coverage/storybook/coverage-storybook.json` file which can be used by `nyc`.
 
 ![](.github/assets/coverage-result.png)
 
-Notice that it provides a message telling you that you can get a better, interactive summary of your code by running:
+If you want to generate reports with [different reporters](https://istanbul.js.org/docs/advanced/alternative-reporters/), you can use `nyc` and point it to the folder which contains the Storybook coverage file. `nyc` is a dependency of the test runner so you will already have it in your project.
+
+Here's an example generating an `lcov` report:
 
 ```
-npx nyc report --reporter=lcov
+npx nyc report --reporter=lcov -t coverage/storybook --report-dir coverage/storybook
 ```
 
-This will generate a folder called `coverage`, containing an `index.html` file which can be explored and will show the coverage in detail:
+This will generate a more detailed, interactive coverage summary that you can access at `coverage/storybook/index.html` file which can be explored and will show the coverage in detail:
 
 ![](.github/assets/coverage-report-html.png)
 
-`nyc` is a dependency of the test runner so you will already have it in your project. In the example above, the `lcov` reporter was used, which generates an output compatible with tools like [Codecov](https://codecov.io/). However, you can configure it to generate different reports, and you can find more information [here](https://istanbul.js.org/docs/advanced/alternative-reporters/).
+The `nyc` command will respect [nyc configuration files](https://github.com/istanbuljs/nyc#common-configuration-options) if you have them in your project.
+
+If you want certain parts of your code to be deliberately ignored, you can use istanbul [parsing hints](https://github.com/istanbuljs/nyc#parsing-hints-ignoring-lines).
+
+### 3 - Merging code coverage with coverage from other tools
+
+The test runner reports coverage related to the `coverage/storybook/coverage-storybook.json` file. This is by design, showing you the coverage which is tested while running Storybook. 
+
+Now, you might have other tests (e.g. unit tests) which are _not_ covered in Storybook but are covered when running tests with Jest, which you might also generate coverage files from, for instance. In such cases, if you are using tools like [Codecov](https://codecov.io/) to automate reporting, the coverage files will be detected automatically and if there are multiple files in the coverage folder, they will be merged automatically.
+
+Alternatively, in case you want to merge coverages from other tools, you should:
+
+1 - move or copy the `coverage/storybook/coverage-storybook.json` into `coverage/coverage-storybook.json`;
+2 - run `nyc report` against the `coverage` folder.
+
+Here's an example on how to achieve that:
+
+```json
+{
+  "scripts": {
+    "test:coverage": "jest --coverage",
+    "test-storybook:coverage": "test-storybook --coverage",
+    "coverage-report": "cp coverage/storybook/coverage-storybook.json coverage/coverage-storybook.json && nyc report --reporter=html -t coverage --report-dir coverage"
+  }
+}
+```
 
 ## Experimental test hook API
 
@@ -483,10 +506,6 @@ env:
   TEMP: ${{ runner.temp }}
 ```
 
-#### The test runner does not show reports in watch mode
-
-Because the displaying of reports and the underlying Jest process are separate, the reports can't be show in watch mode. However, the `.nyc_output/coverage.json` file is being generated, and you can show the reports by running `npx nyc report` in a separate terminal.
-
 #### Adding the test runner to other CI environments
 
 As the test runner is based on playwright, depending on your CI setup you might need to use specific docker images or other configuration. In that case, you can refer to the [Playwright CI docs](https://playwright.dev/docs/ci) for more information.

bin/test-storybook.js

@@ -35,11 +35,47 @@ const cleanup = () => {
     log(`Cleaning up ${storiesJsonTmpDir}`);
     fs.rmSync(storiesJsonTmpDir, { recursive: true, force: true });
   }
-  process.exit();
 };
 
-process.on('SIGINT', cleanup);
-process.on('beforeExit', cleanup);
+let isWatchMode = false;
+async function reportCoverage() {
+  if (isWatchMode || process.env.STORYBOOK_COLLECT_COVERAGE !== 'true') {
+    return
+  }
+
+  const coverageFolderE2E = path.resolve(process.cwd(), '.nyc_output');
+  const coverageFolder = path.resolve(process.cwd(), 'coverage/storybook');
+
+  // in case something goes wrong and .nyc_output does not exist, bail
+  if (!fs.existsSync(coverageFolderE2E)) {
+    return
+  }
+
+  // if there's no coverage folder, create one
+  if (!fs.existsSync(coverageFolder)) {
+    fs.mkdirSync(coverageFolder, { recursive: true });
+  }
+
+  // move the coverage files from .nyc_output folder (coming from jest-playwright) to coverage, then delete .nyc_output
+  fs.renameSync(
+    `${coverageFolderE2E}/coverage.json`,
+    `${coverageFolder}/coverage-storybook.json`,
+  );
+  fs.rmSync(coverageFolderE2E, { recursive: true });
+
+  // --skip-full in case we only want to show not fully covered code
+  // --check-coverage if we want to break if coverage reaches certain threshold
+  // .nycrc will be respected for thresholds etc. https://www.npmjs.com/package/nyc#coverage-thresholds
+  execSync(`npx nyc report --reporter=text -t ${coverageFolder} --report-dir ${coverageFolder}`, { stdio: 'inherit' })
+} 
+
+const onProcessEnd = () => {
+  cleanup();
+  reportCoverage();
+}
+
+process.on('SIGINT', onProcessEnd);
+process.on('exit', onProcessEnd);
 
 function sanitizeURL(url) {
   let finalURL = url;
@@ -88,14 +124,6 @@ async function executeJestPlaywright(args) {
   await jest.run(argv);
 }
 
-async function printCoverageReport() {
-  // --skip-full in case we only want to show not fully covered code
-  // --check-coverage if we want to break if coverage reaches certain threshold
-  // idea: pass configuration object for thresholds https://www.npmjs.com/package/nyc#coverage-thresholds
-  execSync('npx nyc report', { stdio: 'inherit' })
-  log('For a better, interactive summary of coverage, run: \nnpx nyc report --reporter=lcov\n')
-}
-
 async function checkStorybook(url) {
   try {
     const res = await fetch(url, { method: 'HEAD' });
@@ -166,6 +194,9 @@ const main = async () => {
     process.exit(0);
   }
 
+  // set this flag to skip reporting coverage in watch mode
+  isWatchMode = jestOptions.watch;
+
   const targetURL = sanitizeURL(process.env.TARGET_URL || runnerOptions.url);
   await checkStorybook(targetURL);
 
@@ -211,10 +242,6 @@ const main = async () => {
   }
 
   await executeJestPlaywright(jestOptions);
-
-  if (process.env.STORYBOOK_COLLECT_COVERAGE === 'true') {
-    printCoverageReport();
-  }
 };
 
 main().catch((e) => log(e));

package.json

@@ -45,6 +45,7 @@
     "test-storybook:no-cache": "yarn test-storybook --no-cache",
     "test-storybook:json": "yarn test-storybook --stories-json",
     "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook\"",
+    "test-storybook:ci-coverage": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook --coverage\"",
     "test-storybook:ci-json": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:json\"",
     "generate-dynamic-stories": "node scripts/generate-dynamic-stories.js"
   },
@@ -60,6 +61,7 @@
     "@babel/template": "^7.14.5",
     "@babel/types": "^7.14.8",
     "@jest/types": "^27.0.6",
+    "@storybook/addon-coverage": "^0.0.1",
     "@storybook/addon-essentials": "^6.5.0",
     "@storybook/addon-interactions": "^6.5.0",
     "@storybook/jest": "^0.0.10",

yarn.lock

@@ -2055,6 +2055,14 @@
     lodash "^4.17.21"
     ts-dedent "^2.0.0"
 
+"@storybook/addon-coverage@^0.0.1":
+  version "0.0.1"
+  resolved "https://registry.yarnpkg.com/@storybook/addon-coverage/-/addon-coverage-0.0.1.tgz#0eaf343a403c8a12a40839b40268bf51defef06f"
+  integrity sha512-Ml7GYmksZWqsXJhYig75UlFqEj3SMeKhLyMaHi0r6jCPdPSbl+osmWgmXnzQwdlqkhm7NrAIK+NYFVW/uTcMXA==
+  dependencies:
+    "@types/babel__core" "^7.1.19"
+    babel-plugin-istanbul "^6.1.1"
+
 "@storybook/[email protected]":
   version "6.5.0"
   resolved "https://registry.yarnpkg.com/@storybook/addon-docs/-/addon-docs-6.5.0.tgz#54457ddebbe87f40738503f6c939fac44a982926"
@@ -3022,7 +3030,7 @@
   resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-4.2.2.tgz#ed4e0ad92306a704f9fb132a0cfcf77486dbe2bc"
   integrity sha512-HnYpAE1Y6kRyKM/XkEuiRQhTHvkzMBurTHnpFLYLBGPIylZNPs9jJcuOOYWxPLJCSEtmZT0Y8rHDokKN7rRTig==
 
-"@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14":
+"@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14", "@types/babel__core@^7.1.19":
   version "7.1.19"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.19.tgz#7b497495b7d1b4812bdb9d02804d0576f43ee460"
   integrity sha512-WEOTgRsbYkvA/KCsDwVEGkd7WAr1e3g31VHQ8zy5gul/V1qKullU/BU5I68X5v7V3GnB9eotmom4v5a5gjxorw==