WIP: Manually run E2E tests on CI?

.github/workflows/build.yml

@@ -43,6 +43,7 @@ env:
     ${{ github.workspace }}/packages/utils/esm
 
   BUILD_CACHE_KEY: ${{ github.event.inputs.commit || github.sha }}
+  BUILD_CACHE_TARBALL_KEY: tarball-${{ github.event.inputs.commit || github.sha }}
 
   # GH will use the first restore-key it finds that matches
   # So it will start by looking for one from the same branch, else take the newest one it can find elsewhere
@@ -729,20 +730,81 @@ jobs:
           cd packages/remix
           yarn test:integration:ci
 
+  job_e2e_prepare:
+    name: Prepare E2E tests
+    if:
+      (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
+      github.actor != 'dependabot[bot]'
+    needs: [job_get_metadata, job_build]
+    runs-on: ubuntu-20.04
+    timeout-minutes: 15
+    steps:
+      - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.HEAD_COMMIT }}
+      - name: Set up Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: 'package.json'
+      - name: Restore caches
+        uses: ./.github/actions/restore-cache
+        env:
+          DEPENDENCY_CACHE_KEY: ${{ needs.job_build.outputs.dependency_cache_key }}
+      - name: Check tarball cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ github.workspace }}/packages/*/*.tgz
+          key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
+      - name: Build tarballs
+        run: yarn build:tarball
+
   job_e2e_tests:
-    name: E2E (Shard ${{ matrix.shard }}) Tests
+    name: E2E ${{ matrix.label || matrix.test-application }} Test
     # We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks
     # Dependabot PRs sadly also don't have access to secrets, so we skip them as well
     if:
       (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
       github.actor != 'dependabot[bot]'
-    needs: [job_get_metadata, job_build]
+    needs: [job_get_metadata, job_build, job_e2e_prepare]
     runs-on: ubuntu-20.04
-    timeout-minutes: 30
+    timeout-minutes: 10
+    env:
+      E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
+      E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      # Needed because next/sveltekit expects a prefix
+      NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
+      E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
     strategy:
       fail-fast: false
       matrix:
-        shard: [1, 2, 3, 4]
+        test-application:
+          [
+            'node-express-app',
+            'create-react-app',
+            'create-next-app',
+            'create-remix-app',
+            'nextjs-app-dir',
+            'react-create-hash-router',
+            'standard-frontend-react',
+            'standard-frontend-react-tracing',
+            'sveltekit',
+          ]
+        build-command:
+          - false
+        label:
+          - false
+        # Add any variations of a test app here
+        # You should provide an alternate build-command as well as a matching label
+        include:
+          - test-application: 'create-react-app'
+            build-command: 'test:build-ts3.8'
+            label: 'create-react-app (TS 3.8)'
+          - test-application: 'standard-frontend-react'
+            build-command: 'test:build-ts3.8'
+            label: 'standard-frontend-react (TS 3.8)'
 
     steps:
       - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
@@ -761,22 +823,36 @@ jobs:
         env:
           DEPENDENCY_CACHE_KEY: ${{ needs.job_build.outputs.dependency_cache_key }}
 
+      - name: Restore tarball cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ github.workspace }}/packages/*/*.tgz
+          key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
+
       - name: Get node version
         id: versions
         run: |
           echo "echo node=$(jq -r '.volta.node' package.json)" >> $GITHUB_OUTPUT
-      - name: Run E2E tests
+
+      - name: Validate Verdaccio
+        run: yarn test:validate
+        working-directory: packages/e2e-tests
+
+      - name: Prepare Verdaccio
+        run: yarn test:prepare
+        working-directory: packages/e2e-tests
         env:
           E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
-          E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
-          E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
-          E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
-          E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
-          E2E_TEST_SHARD: ${{ matrix.shard }}
-          E2E_TEST_SHARD_AMOUNT: 4
-        run: |
-          cd packages/e2e-tests
-          yarn test:e2e
+
+      - name: Build E2E app
+        working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
+        timeout-minutes: 5
+        run: yarn ${{ matrix.build-command || 'test:build' }}
+
+      - name: Run E2E test
+        working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
+        timeout-minutes: 5
+        run: yarn test:assert
 
   job_required_tests:
     name: All required tests passed or skipped

.github/workflows/canary.yml

@@ -17,41 +17,108 @@ permissions:
   issues: write
 
 jobs:
-  job_canary_test:
-    name: Canary Tests
+  job_e2e_prepare:
+    name: Prepare E2E Canary tests
     runs-on: ubuntu-20.04
-    timeout-minutes: 60
+    timeout-minutes: 15
     steps:
-      - name: 'Check out current commit'
+      - name: Check out current commit
         uses: actions/checkout@v3
         with:
           ref: ${{ env.HEAD_COMMIT }}
-      - uses: pnpm/action-setup@v2
-        with:
-          version: 8.3.1
       - name: Set up Node
         uses: actions/setup-node@v3
         with:
           node-version-file: 'package.json'
       - name: Install dependencies
-        run: yarn install --ignore-engines --frozen-lockfile
+        run: yarn install
       - name: Build packages
         run: yarn build
+      - name: Check tarball cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ github.workspace }}/packages/*/*.tgz
+          key: tarballs-${{ env.HEAD_COMMIT }}
+      - name: Build tarballs
+        run: yarn build:tarball
+
+  job_e2e_tests:
+    name: E2E ${{ matrix.label }} Test
+    needs: [job_e2e_prepare]
+    runs-on: ubuntu-20.04
+    timeout-minutes: 10
+    env:
+      E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
+      E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      # Needed because next/sveltekit expects a prefix
+      NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
+      E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
+      E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - test-application: 'create-react-app'
+            build-command: 'test:build-canary'
+            label: 'create-react-app (canary)'
+          - test-application: 'nextjs-app-dir'
+            build-command: 'test:build-canary'
+            label: 'nextjs-app-dir (canary)'
+          - test-application: 'nextjs-app-dir'
+            build-command: 'test:build-latest'
+            label: 'nextjs-app-dir (latest)'
+          - test-application: 'react-create-hash-router'
+            build-command: 'test:build-canary'
+            label: 'react-create-hash-router (canary)'
+          - test-application: 'standard-frontend-react'
+            build-command: 'test:build-canary'
+            label: 'standard-frontend-react (canary)'
+
+    steps:
+      - name: Check out current commit
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.HEAD_COMMIT }}
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 8.3.1
+      - name: Set up Node
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: 'package.json'
+
+      - name: Restore tarball cache
+        uses: actions/cache/restore@v3
+        with:
+          path: ${{ github.workspace }}/packages/*/*.tgz
+          key: tarballs-${{ env.HEAD_COMMIT }}
+
       - name: Get node version
         id: versions
         run: |
           echo "echo node=$(jq -r '.volta.node' package.json)" >> $GITHUB_OUTPUT
-      - name: Run Canary Tests
+
+      - name: Validate Verdaccio
+        run: yarn test:validate
+        working-directory: packages/e2e-tests
+
+      - name: Prepare Verdaccio
+        run: yarn test:prepare
+        working-directory: packages/e2e-tests
         env:
           E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
-          E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
-          E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
-          E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
-          E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
-          CANARY_E2E_TEST: 'yes'
-        run: |
-          cd packages/e2e-tests
-          yarn test:e2e
+
+      - name: Build E2E app
+        working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
+        timeout-minutes: 5
+        run: yarn ${{ matrix.build-command }}
+
+      - name: Run E2E test
+        working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
+        timeout-minutes: 5
+        run: yarn test:assert
+
       - name: Create Issue
         if: failure() && github.event_name == 'schedule'
         uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd
@@ -101,4 +168,3 @@ jobs:
         with:
           filename: .github/CANARY_FAILURE_TEMPLATE.md
           update_existing: true
-          title: 'Ember Canary tests failed'

packages/e2e-tests/package.json

@@ -18,6 +18,8 @@
     "test:run": "ts-node run.ts",
     "test:validate-configuration": "ts-node validate-verdaccio-configuration.ts",
     "test:validate-test-app-setups": "ts-node validate-test-app-setups.ts",
+    "test:prepare": "ts-node prepare.ts",
+    "test:validate": "run-s test:validate-configuration test:validate-test-app-setups",
     "clean": "rimraf tmp test-applications/**/node_modules test-applications/**/dist"
   },
   "devDependencies": {

packages/e2e-tests/prepare.ts

@@ -0,0 +1,24 @@
+/* eslint-disable max-lines */
+/* eslint-disable no-console */
+import * as dotenv from 'dotenv';
+
+import { validate } from './lib/validate';
+import { registrySetup } from './registrySetup';
+
+async function run(): Promise<void> {
+  // Load environment variables from .env file locally
+  dotenv.config();
+
+  if (!validate()) {
+    process.exit(1);
+  }
+
+  try {
+    registrySetup();
+  } catch (error) {
+    console.error(error);
+    process.exit(1);
+  }
+}
+
+void run();

packages/e2e-tests/publish-packages.ts

@@ -5,9 +5,6 @@ import * as path from 'path';
 
 const repositoryRoot = path.resolve(__dirname, '../..');
 
-// Create tarballs
-childProcess.execSync('yarn build:tarball', { encoding: 'utf8', cwd: repositoryRoot, stdio: 'inherit' });
-
 // Get absolute paths of all the packages we want to publish to the fake registry
 const packageTarballPaths = glob.sync('packages/*/sentry-*.tgz', {
   cwd: repositoryRoot,

packages/e2e-tests/run.ts

@@ -16,11 +16,8 @@ async function run(): Promise<void> {
   }
 
   const envVarsToInject = {
-    REACT_APP_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
-    REMIX_APP_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
     NEXT_PUBLIC_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
     PUBLIC_E2E_TEST_DSN: process.env.E2E_TEST_DSN,
-    BASE_PORT: '27496', // just some random port
   };
 
   try {

packages/e2e-tests/test-applications/create-next-app/package.json

@@ -5,7 +5,9 @@
   "scripts": {
     "build": "next build",
     "test:prod": "TEST_ENV=prod playwright test",
-    "test:dev": "TEST_ENV=dev playwright test"
+    "test:dev": "TEST_ENV=dev playwright test",
+    "test:build": "pnpm install && npx playwright install && pnpm build",
+    "test:assert": "pnpm test:prod && pnpn test:dev"
   },
   "dependencies": {
     "@next/font": "13.0.7",

packages/e2e-tests/test-applications/create-next-app/playwright.config.ts

@@ -7,7 +7,7 @@ if (!testEnv) {
   throw new Error('No test env defined');
 }
 
-const port = Number(process.env.BASE_PORT) + Number(process.env.PORT_MODULO);
+const port = 9000;
 
 /**
  * See https://playwright.dev/docs/test-configuration.

packages/e2e-tests/test-applications/create-next-app/sentry.client.config.ts

@@ -6,7 +6,7 @@ import * as Sentry from '@sentry/nextjs';
 
 Sentry.init({
   environment: 'qa', // dynamic sampling bias to keep transactions
-  dsn: process.env.NEXT_PUBLIC_E2E_TEST_DSN,
+  dsn: process.env.E2E_TEST_DSN,
   // Adjust this value in production, or use tracesSampler for greater control
   tracesSampleRate: 1.0,
   // ...

packages/e2e-tests/test-applications/create-react-app/package.json

@@ -22,7 +22,11 @@
     "start": "react-scripts start",
     "build": "react-scripts build",
     "test": "react-scripts test",
-    "eject": "react-scripts eject"
+    "eject": "react-scripts eject",
+    "test:build": "pnpm install && pnpm build",
+    "test:build-ts3.8": "pnpm install && pnpm add [email protected] && pnpm build",
+    "test:build-canary": "pnpm install && pnpm add react@canary react-dom@canary && pnpm build",
+    "test:assert": "pnpm -v"
   },
   "eslintConfig": {
     "extends": [

packages/e2e-tests/test-applications/create-remix-app/app/entry.client.tsx

@@ -10,7 +10,7 @@ import { hydrateRoot } from 'react-dom/client';
 import * as Sentry from '@sentry/remix';
 
 Sentry.init({
-  dsn: process.env.REMIX_APP_E2E_TEST_DSN,
+  dsn: process.env.E2E_TEST_DSN,
   integrations: [
     new Sentry.BrowserTracing({
       routingInstrumentation: Sentry.remixRouterInstrumentation(useEffect, useLocation, useMatches),

packages/e2e-tests/test-applications/create-remix-app/app/entry.server.tsx

@@ -16,7 +16,7 @@ import * as Sentry from '@sentry/remix';
 const ABORT_DELAY = 5_000;
 
 Sentry.init({
-  dsn: process.env.REMIX_APP_E2E_TEST_DSN,
+  dsn: process.env.E2E_TEST_DSN,
   // Performance Monitoring
   tracesSampleRate: 1.0, // Capture 100% of the transactions, reduce in production!
 });

packages/e2e-tests/test-applications/create-remix-app/package.json

@@ -5,7 +5,9 @@
     "build": "remix build",
     "dev": "remix dev",
     "start": "remix-serve build",
-    "typecheck": "tsc"
+    "typecheck": "tsc",
+    "test:build": "pnpn install && pnpm build",
+    "test:assert": "pnpm typecheck"
   },
   "dependencies": {
     "@sentry/remix": "*",