build: Add codecov test analytics for playwright tests (#13654)

https://docs.codecov.com/docs/test-result-ingestion-beta https://playwright.dev/docs/test-reporters Adds codecov test analytics to the repo, specifically for our playwright tests. This works by using the junit reporter with playwright, and then uploading that via the `codecov/test-results-action@v1` GitHub Action.
getsentry · Sep 13, 2024 · f44785c · f44785c
1 parent e8a016e
commit f44785c
Show file tree

Hide file tree

Showing 10 changed files with 32 additions and 7 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -610,6 +610,13 @@ jobs:
           overwrite: true
           retention-days: 7
 
+      - name: Upload test results to Codecov
+        if: cancelled() == false
+        uses: codecov/test-results-action@v1
+        with:
+          directory: dev-packages/browser-integration-tests
+          token: ${{ secrets.CODECOV_TOKEN }}
+
   job_browser_loader_tests:
     name: PW ${{ matrix.bundle }} Tests
     needs: [job_get_metadata, job_build]
@@ -653,6 +660,7 @@ jobs:
         run: |
           cd dev-packages/browser-integration-tests
           yarn test:loader
+
       - name: Upload Playwright Traces
         uses: actions/upload-artifact@v4
         if: failure()
@@ -662,6 +670,13 @@ jobs:
           overwrite: true
           retention-days: 7
 
+      - name: Upload test results to Codecov
+        if: cancelled() == false
+        uses: codecov/test-results-action@v1
+        with:
+          directory: dev-packages/browser-integration-tests
+          token: ${{ secrets.CODECOV_TOKEN }}
+
   job_check_for_faulty_dts:
     name: Check for faulty .d.ts files
     needs: [job_get_metadata, job_build]
@@ -1013,6 +1028,13 @@ jobs:
           overwrite: true
           retention-days: 7
 
+      - name: Upload test results to Codecov
+        if: cancelled() == false
+        uses: codecov/test-results-action@v1
+        with:
+          directory: dev-packages/e2e-tests
+          token: ${{ secrets.CODECOV_TOKEN }}
+
   job_optional_e2e_tests:
     name: E2E ${{ matrix.label || matrix.test-application }} Test
     # We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks

diff --git a/dev-packages/browser-integration-tests/package.json b/dev-packages/browser-integration-tests/package.json
@@ -35,7 +35,7 @@
     "test:loader:replay_buffer": "PW_BUNDLE=loader_replay_buffer yarn test:loader",
     "test:loader:full": "PW_BUNDLE=loader_tracing_replay yarn test:loader",
     "test:loader:debug": "PW_BUNDLE=loader_debug yarn test:loader",
-    "test:ci": "yarn test:all --reporter='line'",
+    "test:ci": "yarn test:all",
     "test:update-snapshots": "yarn test:all --update-snapshots",
     "test:detect-flaky": "ts-node scripts/detectFlakyTests.ts"
   },

diff --git a/dev-packages/browser-integration-tests/playwright.config.ts b/dev-packages/browser-integration-tests/playwright.config.ts
@@ -30,6 +30,8 @@ const config: PlaywrightTestConfig = {
     },
   ],
 
+  reporter: process.env.CI ? [['line'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
+
   globalSetup: require.resolve('./playwright.setup.ts'),
   globalTeardown: require.resolve('./playwright.teardown.ts'),
 };

diff --git a/dev-packages/e2e-tests/test-applications/ember-classic/playwright.config.ts b/dev-packages/e2e-tests/test-applications/ember-classic/playwright.config.ts
@@ -35,7 +35,7 @@ const config: PlaywrightTestConfig = {
   forbidOnly: !!process.env.CI,
   retries: 0,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'list',
+  reporter: process.env.CI ? [['line'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */

diff --git a/dev-packages/e2e-tests/test-applications/ember-embroider/playwright.config.ts b/dev-packages/e2e-tests/test-applications/ember-embroider/playwright.config.ts
@@ -35,7 +35,7 @@ const config: PlaywrightTestConfig = {
   forbidOnly: !!process.env.CI,
   retries: 0,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'list',
+  reporter: process.env.CI ? [['line'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */

diff --git a/dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/playwright.config.mjs
@@ -23,7 +23,7 @@ const config = {
   /* Retry on CI only */
   retries: 0,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'list',
+  reporter: process.env.CI ? [['line'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */

diff --git a/dev-packages/e2e-tests/test-applications/react-send-to-sentry/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/react-send-to-sentry/playwright.config.mjs
@@ -23,7 +23,7 @@ const config = {
   /* Opt out of parallel tests on CI. */
   workers: 1,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'list',
+  reporter: process.env.CI ? [['line'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
   use: {
     /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */

diff --git a/dev-packages/test-utils/src/playwright-config.ts b/dev-packages/test-utils/src/playwright-config.ts
@@ -37,7 +37,7 @@ export function getPlaywrightConfig(
     /* In dev mode some apps are flaky, so we allow retry there... */
     retries: testEnv === 'development' ? 3 : 0,
     /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-    reporter: process.env.CI ? 'line' : 'list',
+    reporter: process.env.CI ? [['line'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
       /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */

diff --git a/packages/remix/package.json b/packages/remix/package.json
@@ -106,7 +106,7 @@
     "test:integration:prepare": "(cd test/integration && yarn install)",
     "test:integration:clean": "(cd test/integration && rimraf .cache node_modules build)",
     "test:integration:client": "yarn playwright install-deps && yarn playwright test test/integration/test/client/ --project='chromium'",
-    "test:integration:client:ci": "yarn test:integration:client --reporter='line'",
+    "test:integration:client:ci": "yarn test:integration:client",
     "test:integration:server": "export NODE_OPTIONS='--stack-trace-limit=25' && vitest run",
     "test:unit": "jest",
     "test:watch": "jest --watch",

diff --git a/packages/remix/playwright.config.ts b/packages/remix/playwright.config.ts
@@ -8,6 +8,7 @@ const config: PlaywrightTestConfig = {
   },
   // Run tests inside of a single file in parallel
   fullyParallel: true,
+  reporter: process.env.CI ? [['line'], ['junit', { outputFile: 'results.junit.xml' }]] : 'list',
   // Use 3 workers on CI, else use defaults (based on available CPU cores)
   // Note that 3 is a random number selected to work well with our CI setup
   workers: process.env.CI ? 3 : undefined,