Skip to content

Commit ef0ffd6

Browse files
mydeamydea
committed
test: Refactor E2E test running
1 parent 67c3b6a commit ef0ffd6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+348
-880
lines changed

‎.github/workflows/build.yml‎

Lines changed: 91 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ env:
4343
${{ github.workspace }}/packages/utils/esm
4444
4545
BUILD_CACHE_KEY: ${{ github.event.inputs.commit || github.sha }}
46+
BUILD_CACHE_TARBALL_KEY: tarball-${{ github.event.inputs.commit || github.sha }}
4647

4748
# GH will use the first restore-key it finds that matches
4849
# So it will start by looking for one from the same branch, else take the newest one it can find elsewhere
@@ -729,20 +730,82 @@ jobs:
729730
cd packages/remix
730731
yarn test:integration:ci
731732
733+
job_e2e_prepare:
734+
name: Prepare E2E tests
735+
if:
736+
(github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
737+
github.actor != 'dependabot[bot]'
738+
needs: [job_get_metadata, job_build]
739+
runs-on: ubuntu-20.04
740+
timeout-minutes: 15
741+
steps:
742+
- name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
743+
uses: actions/checkout@v3
744+
with:
745+
ref: ${{ env.HEAD_COMMIT }}
746+
- name: Set up Node
747+
uses: actions/setup-node@v3
748+
with:
749+
node-version-file: 'package.json'
750+
- name: Restore caches
751+
uses: ./.github/actions/restore-cache
752+
env:
753+
DEPENDENCY_CACHE_KEY: ${{ needs.job_build.outputs.dependency_cache_key }}
754+
- name: Check tarball cache
755+
uses: actions/cache@v3
756+
with:
757+
path: ${{ github.workspace }}/packages/*/*.tgz
758+
key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
759+
- name: Build tarballs
760+
run: yarn build:tarball
761+
732762
job_e2e_tests:
733-
name: E2E (Shard ${{ matrix.shard }}) Tests
763+
name: E2E ${{ matrix.label || matrix.test-application }} Test
734764
# We only run E2E tests for non-fork PRs because the E2E tests require secrets to work and they can't be accessed from forks
735765
# Dependabot PRs sadly also don't have access to secrets, so we skip them as well
736766
if:
737767
(github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) &&
738768
github.actor != 'dependabot[bot]'
739-
needs: [job_get_metadata, job_build]
769+
needs: [job_get_metadata, job_build, job_e2e_prepare]
740770
runs-on: ubuntu-20.04
741-
timeout-minutes: 30
771+
timeout-minutes: 10
772+
env:
773+
E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
774+
E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
775+
# Needed because some apps expect a certain prefix
776+
NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
777+
PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
778+
REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
779+
E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
780+
E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
742781
strategy:
743782
fail-fast: false
744783
matrix:
745-
shard: [1, 2, 3, 4]
784+
test-application:
785+
[
786+
'node-express-app',
787+
'create-react-app',
788+
'create-next-app',
789+
'create-remix-app',
790+
'nextjs-app-dir',
791+
'react-create-hash-router',
792+
'standard-frontend-react',
793+
'standard-frontend-react-tracing-import',
794+
'sveltekit',
795+
]
796+
build-command:
797+
- false
798+
label:
799+
- false
800+
# Add any variations of a test app here
801+
# You should provide an alternate build-command as well as a matching label
802+
include:
803+
- test-application: 'create-react-app'
804+
build-command: 'test:build-ts3.8'
805+
label: 'create-react-app (TS 3.8)'
806+
- test-application: 'standard-frontend-react'
807+
build-command: 'test:build-ts3.8'
808+
label: 'standard-frontend-react (TS 3.8)'
746809

747810
steps:
748811
- name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }})
@@ -761,22 +824,36 @@ jobs:
761824
env:
762825
DEPENDENCY_CACHE_KEY: ${{ needs.job_build.outputs.dependency_cache_key }}
763826

827+
- name: Restore tarball cache
828+
uses: actions/cache/restore@v3
829+
with:
830+
path: ${{ github.workspace }}/packages/*/*.tgz
831+
key: ${{ env.BUILD_CACHE_TARBALL_KEY }}
832+
764833
- name: Get node version
765834
id: versions
766835
run: |
767836
echo "echo node=$(jq -r '.volta.node' package.json)" >> $GITHUB_OUTPUT
768-
- name: Run E2E tests
837+
838+
- name: Validate Verdaccio
839+
run: yarn test:validate
840+
working-directory: packages/e2e-tests
841+
842+
- name: Prepare Verdaccio
843+
run: yarn test:prepare
844+
working-directory: packages/e2e-tests
769845
env:
770846
E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
771-
E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
772-
E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
773-
E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
774-
E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
775-
E2E_TEST_SHARD: ${{ matrix.shard }}
776-
E2E_TEST_SHARD_AMOUNT: 4
777-
run: |
778-
cd packages/e2e-tests
779-
yarn test:e2e
847+
848+
- name: Build E2E app
849+
working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
850+
timeout-minutes: 5
851+
run: yarn ${{ matrix.build-command || 'test:build' }}
852+
853+
- name: Run E2E test
854+
working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
855+
timeout-minutes: 5
856+
run: yarn test:assert
780857

781858
job_required_tests:
782859
name: All required tests passed or skipped

‎.github/workflows/canary.yml‎

Lines changed: 92 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,41 +17,116 @@ permissions:
1717
issues: write
1818

1919
jobs:
20-
job_canary_test:
21-
name: Canary Tests
20+
job_e2e_prepare:
21+
name: Prepare E2E Canary tests
2222
runs-on: ubuntu-20.04
23-
timeout-minutes: 60
23+
timeout-minutes: 30
2424
steps:
25-
- name: 'Check out current commit'
25+
- name: Check out current commit
2626
uses: actions/checkout@v3
2727
with:
2828
ref: ${{ env.HEAD_COMMIT }}
29-
- uses: pnpm/action-setup@v2
30-
with:
31-
version: 8.3.1
3229
- name: Set up Node
3330
uses: actions/setup-node@v3
3431
with:
3532
node-version-file: 'package.json'
33+
- name: Check canary cache
34+
uses: actions/cache@v3
35+
with:
36+
path: |
37+
${{ github.workspace }}/packages/*/*.tgz
38+
${{ github.workspace }}/node_modules
39+
${{ github.workspace }}/packages/*/node_modules
40+
key: canary-${{ env.HEAD_COMMIT }}
3641
- name: Install dependencies
37-
run: yarn install --ignore-engines --frozen-lockfile
42+
run: yarn install
3843
- name: Build packages
3944
run: yarn build
45+
46+
- name: Build tarballs
47+
run: yarn build:tarball
48+
49+
job_e2e_tests:
50+
name: E2E ${{ matrix.label }} Test
51+
needs: [job_e2e_prepare]
52+
runs-on: ubuntu-20.04
53+
timeout-minutes: 15
54+
env:
55+
E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
56+
E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
57+
# Needed because certain apps expect a certain prefix
58+
NEXT_PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
59+
PUBLIC_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
60+
REACT_APP_E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
61+
E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
62+
E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
63+
strategy:
64+
fail-fast: false
65+
matrix:
66+
include:
67+
- test-application: 'create-react-app'
68+
build-command: 'test:build-canary'
69+
label: 'create-react-app (canary)'
70+
- test-application: 'nextjs-app-dir'
71+
build-command: 'test:build-canary'
72+
label: 'nextjs-app-dir (canary)'
73+
- test-application: 'nextjs-app-dir'
74+
build-command: 'test:build-latest'
75+
label: 'nextjs-app-dir (latest)'
76+
- test-application: 'react-create-hash-router'
77+
build-command: 'test:build-canary'
78+
label: 'react-create-hash-router (canary)'
79+
- test-application: 'standard-frontend-react'
80+
build-command: 'test:build-canary'
81+
label: 'standard-frontend-react (canary)'
82+
83+
steps:
84+
- name: Check out current commit
85+
uses: actions/checkout@v3
86+
with:
87+
ref: ${{ env.HEAD_COMMIT }}
88+
- uses: pnpm/action-setup@v2
89+
with:
90+
version: 8.3.1
91+
- name: Set up Node
92+
uses: actions/setup-node@v3
93+
with:
94+
node-version-file: 'package.json'
95+
96+
- name: Restore canary cache
97+
uses: actions/cache/restore@v3
98+
with:
99+
path: |
100+
${{ github.workspace }}/packages/*/*.tgz
101+
${{ github.workspace }}/node_modules
102+
${{ github.workspace }}/packages/*/node_modules
103+
key: canary-${{ env.HEAD_COMMIT }}
104+
40105
- name: Get node version
41106
id: versions
42107
run: |
43108
echo "echo node=$(jq -r '.volta.node' package.json)" >> $GITHUB_OUTPUT
44-
- name: Run Canary Tests
109+
110+
- name: Validate Verdaccio
111+
run: yarn test:validate
112+
working-directory: packages/e2e-tests
113+
114+
- name: Prepare Verdaccio
115+
run: yarn test:prepare
116+
working-directory: packages/e2e-tests
45117
env:
46118
E2E_TEST_PUBLISH_SCRIPT_NODE_VERSION: ${{ steps.versions.outputs.node }}
47-
E2E_TEST_AUTH_TOKEN: ${{ secrets.E2E_TEST_AUTH_TOKEN }}
48-
E2E_TEST_DSN: ${{ secrets.E2E_TEST_DSN }}
49-
E2E_TEST_SENTRY_ORG_SLUG: 'sentry-javascript-sdks'
50-
E2E_TEST_SENTRY_TEST_PROJECT: 'sentry-javascript-e2e-tests'
51-
CANARY_E2E_TEST: 'yes'
52-
run: |
53-
cd packages/e2e-tests
54-
yarn test:e2e
119+
120+
- name: Build E2E app
121+
working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
122+
timeout-minutes: 5
123+
run: yarn ${{ matrix.build-command }}
124+
125+
- name: Run E2E test
126+
working-directory: packages/e2e-tests/test-applications/${{ matrix.test-application }}
127+
timeout-minutes: 5
128+
run: yarn test:assert
129+
55130
- name: Create Issue
56131
if: failure() && github.event_name == 'schedule'
57132
uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd
@@ -101,4 +176,3 @@ jobs:
101176
with:
102177
filename: .github/CANARY_FAILURE_TEMPLATE.md
103178
update_existing: true
104-
title: 'Ember Canary tests failed'

‎packages/e2e-tests/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
.env
22
tmp
3+
.tmp_build_output
4+
pnpm-lock.yaml

‎packages/e2e-tests/README.md‎

Lines changed: 13 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@ Prerequisites: Docker
1515
yarn test:e2e
1616
```
1717

18+
Or run only a single E2E test app:
19+
20+
```bash
21+
yarn test:run <app-name>
22+
```
23+
1824
## How they work
1925

2026
Before running any tests we launch a fake test registry (in our case [Verdaccio](https://verdaccio.org/docs/e2e/)), we
@@ -43,40 +49,14 @@ cat > test-applications/my-new-test-application/.npmrc << EOF
4349
@sentry:registry=http://localhost:4873
4450
@sentry-internal:registry=http://localhost:4873
4551
EOF
46-
47-
# Add a test recipe file to the test application
48-
touch test-applications/my-new-test-application/test-recipe.json
49-
```
50-
51-
To get you started with the recipe, you can copy the following into `test-recipe.json`:
52-
53-
```json
54-
{
55-
"$schema": "../../test-recipe-schema.json",
56-
"testApplicationName": "My New Test Application",
57-
"buildCommand": "pnpm install",
58-
"tests": [
59-
{
60-
"testName": "My new test",
61-
"testCommand": "pnpm test",
62-
"timeoutSeconds": 60
63-
}
64-
]
65-
}
6652
```
6753

68-
The `test-recipe.json` files follow a schema (`e2e-tests/test-recipe-schema.json`). Here is a basic explanation of the
69-
fields:
54+
Make sure to add a `test:build` and `test:assert` command to the new app's `package.json` file.
7055

71-
- The `buildCommand` command runs only once before any of the tests and is supposed to build the test application. If
72-
this command returns a non-zero exit code, it counts as a failed test and the test application's tests are not run. In
73-
the example above, we use the `--pure-lockfile` flag to install dependencies without modifiying the lockfile so that
74-
there aren't any changes in the git worktree after running the tests.
75-
- The `testCommand` command is supposed to run tests on the test application. If the configured command returns a
76-
non-zero exit code, it counts as a failed test.
77-
- A test timeout can be configured via `timeoutSeconds`, it defaults to `60`.
56+
Add the new test app to `test-application` matrix in `.github/workflows/build.yml` for the `job_e2e_tests` job. If you
57+
want to run a canary test, add it to the `canary.yml` workflow.
7858

79-
**An important thing to note:** In the context of the `buildCommand` the fake test registry is available at
59+
**An important thing to note:** In the context of the build/test commands the fake test registry is available at
8060
`http://localhost:4873`. It hosts all of our packages as if they were to be published with the state of the current
8161
branch. This means we can install the packages from this registry via the `.npmrc` configuration as seen above. If you
8262
add Sentry dependencies to your test application, you should set the dependency versions set to `latest || *` in order
@@ -89,7 +69,9 @@ for it to work with both regular and prerelease versions:
8969
"version": "1.0.0",
9070
"private": true,
9171
"scripts": {
92-
"test": "echo \"Hello world!\""
72+
"test": "echo \"Hello world!\"",
73+
"test:build": "pnpm install",
74+
"test:assert": "pnpm test"
9375
},
9476
"dependencies": {
9577
"@sentry/node": "latest || *"

0 commit comments

Comments
 (0)