From e97262edbbbb9203e7ec3cdb6541f5936688abaf Mon Sep 17 00:00:00 2001 From: Binoy Patel Date: Mon, 23 Oct 2023 12:33:02 -0400 Subject: [PATCH 1/3] chore(e2e): run e2e tests in production build From 30d54dfdc2375d14b37b6f2fc8253a1289180f10 Mon Sep 17 00:00:00 2001 From: Binoy Patel Date: Mon, 23 Oct 2023 14:44:28 -0400 Subject: [PATCH 2/3] chore(e2e): run e2e tests on PRs on ephemeral datasets - refactor: create a unified PR cleanup action - refactor: moved envVars script to utils and make it more resuable --- .github/workflows/e2e.yml | 30 +++++++++++++++++-- .../{docReportTeardown.yml => pr-cleanup.yml} | 13 ++++++-- package.json | 2 ++ scripts/doc-report/docClient.ts | 6 ++-- scripts/doc-report/docReport.ts | 6 ++-- scripts/doc-report/docReportCleanup.ts | 6 ++-- scripts/doc-report/docReportCreate.ts | 8 ++--- scripts/doc-report/envVars.ts | 13 -------- scripts/e2e/cleanup.ts | 25 ++++++++++++++++ scripts/e2e/e2eClient.ts | 18 +++++++++++ scripts/e2e/setup.ts | 19 ++++++++++++ scripts/utils/envVars.ts | 13 ++++++++ 12 files changed, 128 insertions(+), 31 deletions(-) rename .github/workflows/{docReportTeardown.yml => pr-cleanup.yml} (72%) delete mode 100644 scripts/doc-report/envVars.ts create mode 100644 scripts/e2e/cleanup.ts create mode 100644 scripts/e2e/e2eClient.ts create mode 100644 scripts/e2e/setup.ts create mode 100644 scripts/utils/envVars.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0a791a2cf15..f1f8db91fc8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -62,7 +62,8 @@ jobs: - name: Build CLI run: yarn build:cli # Needed for CLI tests - - name: Build E2E test studio + - name: Build E2E test studio on next + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }} env: # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN` @@ -70,9 +71,10 @@ jobs: SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }} SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }} SANITY_E2E_DATASET: ${{ secrets.SANITY_E2E_DATASET }} - run: yarn e2e:build + run: yarn e2e:setup && yarn e2e:build - - name: Run end-to-end tests + - name: Run E2E tests on next + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }} env: # Missing in docs but in use # here https://github.com/microsoft/playwright/blob/main/packages/playwright/src/reporters/blob.ts#L108 @@ -85,6 +87,28 @@ jobs: SANITY_E2E_DATASET: ${{ secrets.SANITY_E2E_DATASET }} run: yarn test:e2e --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + - name: Build E2E test studio on PR + if: ${{ github.event_name == 'pull_request' }} + env: + # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next + # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN` + # Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github + SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }} + SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }} + SANITY_E2E_DATASET: pr-${{ github.event.number }} + run: yarn e2e:setup && yarn e2e:build + + - name: Run E2E tests on PR + if: ${{ github.event_name == 'pull_request' }} + env: + # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next + # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN` + # Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github + SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }} + SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }} + SANITY_E2E_DATASET: pr-${{ github.event.number }} + run: yarn test:e2e --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + - uses: actions/upload-artifact@v3 if: always() with: diff --git a/.github/workflows/docReportTeardown.yml b/.github/workflows/pr-cleanup.yml similarity index 72% rename from .github/workflows/docReportTeardown.yml rename to .github/workflows/pr-cleanup.yml index 27cf821e5e0..150ead07c17 100644 --- a/.github/workflows/docReportTeardown.yml +++ b/.github/workflows/pr-cleanup.yml @@ -1,4 +1,4 @@ -name: 'Cleanup: Documentation Report' +name: 'Cleanup: PR Merge/Closed' on: # Build on closed or merged PRs @@ -6,7 +6,7 @@ on: types: [closed] jobs: - reportTeardown: + pr-cleanup: runs-on: ubuntu-latest env: TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} @@ -33,7 +33,14 @@ jobs: if: steps.cache-node-modules.outputs.cache-hit != 'true' run: yarn install --frozen-lockfile - - name: Remove datasets for closed PRs + - name: Remove E2E datasets for closed PRs + env: + SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }} + SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }} + SANITY_E2E_DATASET: pr-${{ github.event.number }} + run: yarn e2e:cleanup + + - name: Remove docs report datasets for closed PRs env: DOCS_REPORT_TOKEN: ${{ secrets.DOCS_REPORT_DATASET_TOKEN }} DOCS_REPORT_DATASET: pr-${{ github.event.number }} diff --git a/package.json b/package.json index 21bae3b54c0..6b78063613d 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,9 @@ "example:movies-studio": "yarn --cwd examples/movies-studio start", "e2e:dev": "yarn --cwd dev/studio-e2e-testing dev", "e2e:build": "yarn --cwd dev/studio-e2e-testing build", + "e2e:cleanup": "node -r dotenv-flow/config -r esbuild-register scripts/e2e/cleanup", "e2e:preview": "yarn e2e:build && yarn --cwd dev/studio-e2e-testing start", + "e2e:setup": "node -r dotenv-flow/config -r esbuild-register scripts/e2e/setup", "e2e:start": "yarn --cwd dev/studio-e2e-testing start", "etl": "node -r dotenv-flow/config -r esbuild-register scripts/etl", "init": "lerna clean --yes && run-s bootstrap build", diff --git a/scripts/doc-report/docClient.ts b/scripts/doc-report/docClient.ts index 265a9bac4ec..9f075642eb0 100644 --- a/scripts/doc-report/docClient.ts +++ b/scripts/doc-report/docClient.ts @@ -1,12 +1,14 @@ import {type SanityClient, createClient} from '@sanity/client' import {sanityIdify} from '../utils/sanityIdify' -import {readEnv} from './envVars' +import {readEnv} from '../utils/envVars' + +export type KnownEnvVar = 'DOCS_REPORT_DATASET' | 'DOCS_REPORT_TOKEN' export function createDocClient(dataset: string): SanityClient { return createClient({ projectId: 'c1zuxvqn', dataset: sanityIdify(dataset), - token: readEnv('DOCS_REPORT_TOKEN'), + token: readEnv('DOCS_REPORT_TOKEN'), apiVersion: '2023-02-03', useCdn: false, }) diff --git a/scripts/doc-report/docReport.ts b/scripts/doc-report/docReport.ts index 427963b5e0d..af8bd271c50 100644 --- a/scripts/doc-report/docReport.ts +++ b/scripts/doc-report/docReport.ts @@ -4,8 +4,8 @@ import {groupBy} from 'lodash' import {combineLatest, map} from 'rxjs' import prettier from 'prettier' import {startTimer} from '../utils/startTimer' -import {createDocClient} from './docClient' -import {readEnv} from './envVars' +import {readEnv} from '../utils/envVars' +import {KnownEnvVar, createDocClient} from './docClient' const QUERY = `*[_type=='exportSymbol'] { _id, @@ -35,7 +35,7 @@ interface TransformResult { branchNotDocumented: number } -const studioMetricsClient = createDocClient(readEnv('DOCS_REPORT_DATASET')) +const studioMetricsClient = createDocClient(readEnv('DOCS_REPORT_DATASET')) const studioMetricsClientProduction = createDocClient('next') function getDocumentationReport(symbols: ExportSymbol[]): Report[] { diff --git a/scripts/doc-report/docReportCleanup.ts b/scripts/doc-report/docReportCleanup.ts index 6d0871792de..f4ce890f623 100644 --- a/scripts/doc-report/docReportCleanup.ts +++ b/scripts/doc-report/docReportCleanup.ts @@ -1,9 +1,9 @@ import {sanityIdify} from '../utils/sanityIdify' import {startTimer} from '../utils/startTimer' -import {createDocClient} from './docClient' -import {readEnv} from './envVars' +import {readEnv} from '../utils/envVars' +import {KnownEnvVar, createDocClient} from './docClient' -const DATASET = readEnv('DOCS_REPORT_DATASET') +const DATASET = readEnv('DOCS_REPORT_DATASET') const studioMetricsClient = createDocClient(DATASET) const timer = startTimer(`Deleting dataset ${DATASET}`) diff --git a/scripts/doc-report/docReportCreate.ts b/scripts/doc-report/docReportCreate.ts index 813628f3daf..c3cc477d1c9 100644 --- a/scripts/doc-report/docReportCreate.ts +++ b/scripts/doc-report/docReportCreate.ts @@ -16,8 +16,8 @@ import readPackages from '../utils/readPackages' import type {PackageManifest} from '../types' import {sanityIdify} from '../utils/sanityIdify' import {startTimer} from '../utils/startTimer' -import {createDocClient} from './docClient' -import {readEnv} from './envVars' +import {readEnv} from '../utils/envVars' +import {KnownEnvVar, createDocClient} from './docClient' const ALLOWED_TAGS = ['public', 'alpha', 'beta', 'internal', 'experimental', 'deprecated'] interface Package { @@ -194,9 +194,9 @@ function getPackageMutations(pkg: Package): Mutation[] { }) } -const dataset = sanityIdify(readEnv('DOCS_REPORT_DATASET')) +const dataset = sanityIdify(readEnv('DOCS_REPORT_DATASET')) -const studioMetricsClient = createDocClient(readEnv('DOCS_REPORT_DATASET')) +const studioMetricsClient = createDocClient(readEnv('DOCS_REPORT_DATASET')) studioMetricsClient.datasets.list().then(async (datasets) => { // If the dataset doesn't exist, create it diff --git a/scripts/doc-report/envVars.ts b/scripts/doc-report/envVars.ts deleted file mode 100644 index 8565570e196..00000000000 --- a/scripts/doc-report/envVars.ts +++ /dev/null @@ -1,13 +0,0 @@ -type KnownEnvVar = 'DOCS_REPORT_DATASET' | 'DOCS_REPORT_TOKEN' - -export function readEnv(name: KnownEnvVar): string { - const val = findEnv(name) - if (val === undefined) { - throw new Error(`Missing required environment variable "${name}"`) - } - return val -} - -export function findEnv(name: KnownEnvVar): string | undefined { - return process.env[name] -} diff --git a/scripts/e2e/cleanup.ts b/scripts/e2e/cleanup.ts new file mode 100644 index 00000000000..421880aeb72 --- /dev/null +++ b/scripts/e2e/cleanup.ts @@ -0,0 +1,25 @@ +import {sanityIdify} from '../utils/sanityIdify' +import {startTimer} from '../utils/startTimer' +import {readEnv} from '../utils/envVars' +import {KnownEnvVar, createE2EClient} from './e2eClient' + +const DATASET = readEnv('SANITY_E2E_DATASET') +const studioE2EClient = createE2EClient(DATASET) + +const timer = startTimer(`Deleting dataset ${DATASET}`) + +studioE2EClient.datasets + .delete(sanityIdify(DATASET)) + .then((res) => { + if (res.deleted) { + console.log('Deleted dataset') + } else { + console.log('Dataset was not deleted') + } + }) + .catch((err) => { + throw new Error(`Something went wrong! ${err?.response?.body?.message}`) + }) + .finally(() => { + timer.end() + }) diff --git a/scripts/e2e/e2eClient.ts b/scripts/e2e/e2eClient.ts new file mode 100644 index 00000000000..367f16a349f --- /dev/null +++ b/scripts/e2e/e2eClient.ts @@ -0,0 +1,18 @@ +import {type SanityClient, createClient} from '@sanity/client' +import {sanityIdify} from '../utils/sanityIdify' +import {readEnv} from '../utils/envVars' + +export type KnownEnvVar = + | 'SANITY_E2E_DATASET' + | 'SANITY_E2E_PROJECT_ID' + | 'SANITY_E2E_SESSION_TOKEN' + +export function createE2EClient(dataset: string): SanityClient { + return createClient({ + projectId: readEnv('SANITY_E2E_PROJECT_ID'), + dataset: sanityIdify(dataset), + token: readEnv('SANITY_E2E_SESSION_TOKEN'), + apiVersion: '2023-02-03', + useCdn: false, + }) +} diff --git a/scripts/e2e/setup.ts b/scripts/e2e/setup.ts new file mode 100644 index 00000000000..2c42a5f030c --- /dev/null +++ b/scripts/e2e/setup.ts @@ -0,0 +1,19 @@ +import {readEnv} from '../utils/envVars' +import {sanityIdify} from '../utils/sanityIdify' +import {startTimer} from '../utils/startTimer' +import {KnownEnvVar, createE2EClient} from './e2eClient' + +const dataset = sanityIdify(readEnv('SANITY_E2E_DATASET')) + +const studioE2EClient = createE2EClient(readEnv('SANITY_E2E_DATASET')) + +studioE2EClient.datasets.list().then(async (datasets) => { + // If the dataset doesn't exist, create it + if (!datasets.find((ds) => ds.name === dataset)) { + const timer = startTimer(`Creating dataset ${dataset}`) + await studioE2EClient.datasets.create(dataset, { + aclMode: 'public', + }) + timer.end() + } +}) diff --git a/scripts/utils/envVars.ts b/scripts/utils/envVars.ts new file mode 100644 index 00000000000..5b6076589b2 --- /dev/null +++ b/scripts/utils/envVars.ts @@ -0,0 +1,13 @@ +export function readEnv(name: KnownEnvVar): string { + const val = findEnv(name) + if (val === undefined) { + throw new Error(`Missing required environment variable "${name}"`) + } + return val +} + +export function findEnv( + name: KnownEnvVar, +): string | undefined { + return process.env[name] +} From e41060f806e3d997458da5b53bf5d309261281f2 Mon Sep 17 00:00:00 2001 From: Binoy Patel Date: Tue, 31 Oct 2023 10:16:19 -0400 Subject: [PATCH 3/3] chore(e2e): add missing env var to pr e2e tests --- .github/workflows/e2e.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f1f8db91fc8..b14ca062f66 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -101,6 +101,9 @@ jobs: - name: Run E2E tests on PR if: ${{ github.event_name == 'pull_request' }} env: + # Missing in docs but in use + # here https://github.com/microsoft/playwright/blob/main/packages/playwright/src/reporters/blob.ts#L108 + PWTEST_BLOB_REPORT_NAME: ${{ matrix.project }} # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN` # Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github