chore(e2e): run e2e tests on PRs on ephemeral datasets

.github/workflows/e2e.yml

@@ -62,17 +62,19 @@ jobs:
       - name: Build CLI
         run: yarn build:cli # Needed for CLI tests
 
-      - name: Build E2E test studio
+      - name: Build E2E test studio on next
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }}
         env:
           # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next
           # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN`
           # Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github
           SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
           SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }}
           SANITY_E2E_DATASET: ${{ secrets.SANITY_E2E_DATASET }}
-        run: yarn e2e:build
+        run: yarn e2e:setup && yarn e2e:build
 
-      - name: Run end-to-end tests
+      - name: Run E2E tests on next
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }}
         env:
           # Missing in docs but in use
           # here https://github.com/microsoft/playwright/blob/main/packages/playwright/src/reporters/blob.ts#L108
@@ -85,6 +87,31 @@ jobs:
           SANITY_E2E_DATASET: ${{ secrets.SANITY_E2E_DATASET }}
         run: yarn test:e2e --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
+      - name: Build E2E test studio on PR
+        if: ${{ github.event_name == 'pull_request' }}
+        env:
+          # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next
+          # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN`
+          # Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github
+          SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
+          SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }}
+          SANITY_E2E_DATASET: pr-${{ github.event.number }}
+        run: yarn e2e:setup && yarn e2e:build
+
+      - name: Run E2E tests on PR
+        if: ${{ github.event_name == 'pull_request' }}
+        env:
+          # Missing in docs but in use
+          # here https://github.com/microsoft/playwright/blob/main/packages/playwright/src/reporters/blob.ts#L108
+          PWTEST_BLOB_REPORT_NAME: ${{ matrix.project }}
+          # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next
+          # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN`
+          # Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github
+          SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
+          SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }}
+          SANITY_E2E_DATASET: pr-${{ github.event.number }}
+        run: yarn test:e2e --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
       - uses: actions/upload-artifact@v3
         if: always()
         with:

.github/workflows/docReportTeardown.yml → .github/workflows/pr-cleanup.yml

@@ -1,12 +1,12 @@
-name: 'Cleanup: Documentation Report'
+name: 'Cleanup: PR Merge/Closed'
 
 on:
   # Build on closed or merged PRs
   pull_request:
     types: [closed]
 
 jobs:
-  reportTeardown:
+  pr-cleanup:
     runs-on: ubuntu-latest
     env:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
@@ -33,7 +33,14 @@ jobs:
         if: steps.cache-node-modules.outputs.cache-hit != 'true'
         run: yarn install --frozen-lockfile
 
-      - name: Remove datasets for closed PRs
+      - name: Remove E2E datasets for closed PRs
+        env:
+          SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
+          SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }}
+          SANITY_E2E_DATASET: pr-${{ github.event.number }}
+        run: yarn e2e:cleanup
+
+      - name: Remove docs report datasets for closed PRs
         env:
           DOCS_REPORT_TOKEN: ${{ secrets.DOCS_REPORT_DATASET_TOKEN }}
           DOCS_REPORT_DATASET: pr-${{ github.event.number }}

package.json

@@ -42,7 +42,9 @@
     "example:movies-studio": "yarn --cwd examples/movies-studio start",
     "e2e:dev": "yarn --cwd dev/studio-e2e-testing dev",
     "e2e:build": "yarn --cwd dev/studio-e2e-testing build",
+    "e2e:cleanup": "node -r dotenv-flow/config -r esbuild-register scripts/e2e/cleanup",
     "e2e:preview": "yarn e2e:build && yarn --cwd dev/studio-e2e-testing start",
+    "e2e:setup": "node -r dotenv-flow/config -r esbuild-register scripts/e2e/setup",
     "e2e:start": "yarn --cwd dev/studio-e2e-testing start",
     "etl": "node -r dotenv-flow/config -r esbuild-register scripts/etl",
     "init": "lerna clean --yes && run-s bootstrap build",

scripts/doc-report/docClient.ts

@@ -1,12 +1,14 @@
 import {type SanityClient, createClient} from '@sanity/client'
 import {sanityIdify} from '../utils/sanityIdify'
-import {readEnv} from './envVars'
+import {readEnv} from '../utils/envVars'
+
+export type KnownEnvVar = 'DOCS_REPORT_DATASET' | 'DOCS_REPORT_TOKEN'
 
 export function createDocClient(dataset: string): SanityClient {
   return createClient({
     projectId: 'c1zuxvqn',
     dataset: sanityIdify(dataset),
-    token: readEnv('DOCS_REPORT_TOKEN'),
+    token: readEnv<KnownEnvVar>('DOCS_REPORT_TOKEN'),
     apiVersion: '2023-02-03',
     useCdn: false,
   })

scripts/doc-report/docReport.ts

@@ -4,8 +4,8 @@ import {groupBy} from 'lodash'
 import {combineLatest, map} from 'rxjs'
 import prettier from 'prettier'
 import {startTimer} from '../utils/startTimer'
-import {createDocClient} from './docClient'
-import {readEnv} from './envVars'
+import {readEnv} from '../utils/envVars'
+import {KnownEnvVar, createDocClient} from './docClient'
 
 const QUERY = `*[_type=='exportSymbol'] {
   _id,
@@ -35,7 +35,7 @@ interface TransformResult {
   branchNotDocumented: number
 }
 
-const studioMetricsClient = createDocClient(readEnv('DOCS_REPORT_DATASET'))
+const studioMetricsClient = createDocClient(readEnv<KnownEnvVar>('DOCS_REPORT_DATASET'))
 const studioMetricsClientProduction = createDocClient('next')
 
 function getDocumentationReport(symbols: ExportSymbol[]): Report[] {

scripts/doc-report/docReportCleanup.ts

@@ -1,9 +1,9 @@
 import {sanityIdify} from '../utils/sanityIdify'
 import {startTimer} from '../utils/startTimer'
-import {createDocClient} from './docClient'
-import {readEnv} from './envVars'
+import {readEnv} from '../utils/envVars'
+import {KnownEnvVar, createDocClient} from './docClient'
 
-const DATASET = readEnv('DOCS_REPORT_DATASET')
+const DATASET = readEnv<KnownEnvVar>('DOCS_REPORT_DATASET')
 const studioMetricsClient = createDocClient(DATASET)
 
 const timer = startTimer(`Deleting dataset ${DATASET}`)

scripts/doc-report/docReportCreate.ts

@@ -16,8 +16,8 @@ import readPackages from '../utils/readPackages'
 import type {PackageManifest} from '../types'
 import {sanityIdify} from '../utils/sanityIdify'
 import {startTimer} from '../utils/startTimer'
-import {createDocClient} from './docClient'
-import {readEnv} from './envVars'
+import {readEnv} from '../utils/envVars'
+import {KnownEnvVar, createDocClient} from './docClient'
 
 const ALLOWED_TAGS = ['public', 'alpha', 'beta', 'internal', 'experimental', 'deprecated']
 interface Package {
@@ -194,9 +194,9 @@ function getPackageMutations(pkg: Package): Mutation[] {
   })
 }
 
-const dataset = sanityIdify(readEnv('DOCS_REPORT_DATASET'))
+const dataset = sanityIdify(readEnv<KnownEnvVar>('DOCS_REPORT_DATASET'))
 
-const studioMetricsClient = createDocClient(readEnv('DOCS_REPORT_DATASET'))
+const studioMetricsClient = createDocClient(readEnv<KnownEnvVar>('DOCS_REPORT_DATASET'))
 
 studioMetricsClient.datasets.list().then(async (datasets) => {
   // If the dataset doesn't exist, create it

scripts/e2e/cleanup.ts

@@ -0,0 +1,25 @@
+import {sanityIdify} from '../utils/sanityIdify'
+import {startTimer} from '../utils/startTimer'
+import {readEnv} from '../utils/envVars'
+import {KnownEnvVar, createE2EClient} from './e2eClient'
+
+const DATASET = readEnv<KnownEnvVar>('SANITY_E2E_DATASET')
+const studioE2EClient = createE2EClient(DATASET)
+
+const timer = startTimer(`Deleting dataset ${DATASET}`)
+
+studioE2EClient.datasets
+  .delete(sanityIdify(DATASET))
+  .then((res) => {
+    if (res.deleted) {
+      console.log('Deleted dataset')
+    } else {
+      console.log('Dataset was not deleted')
+    }
+  })
+  .catch((err) => {
+    throw new Error(`Something went wrong! ${err?.response?.body?.message}`)
+  })
+  .finally(() => {
+    timer.end()
+  })

scripts/e2e/e2eClient.ts

@@ -0,0 +1,18 @@
+import {type SanityClient, createClient} from '@sanity/client'
+import {sanityIdify} from '../utils/sanityIdify'
+import {readEnv} from '../utils/envVars'
+
+export type KnownEnvVar =
+  | 'SANITY_E2E_DATASET'
+  | 'SANITY_E2E_PROJECT_ID'
+  | 'SANITY_E2E_SESSION_TOKEN'
+
+export function createE2EClient(dataset: string): SanityClient {
+  return createClient({
+    projectId: readEnv<KnownEnvVar>('SANITY_E2E_PROJECT_ID'),
+    dataset: sanityIdify(dataset),
+    token: readEnv<KnownEnvVar>('SANITY_E2E_SESSION_TOKEN'),
+    apiVersion: '2023-02-03',
+    useCdn: false,
+  })
+}

scripts/e2e/setup.ts

@@ -0,0 +1,19 @@
+import {readEnv} from '../utils/envVars'
+import {sanityIdify} from '../utils/sanityIdify'
+import {startTimer} from '../utils/startTimer'
+import {KnownEnvVar, createE2EClient} from './e2eClient'
+
+const dataset = sanityIdify(readEnv<KnownEnvVar>('SANITY_E2E_DATASET'))
+
+const studioE2EClient = createE2EClient(readEnv<KnownEnvVar>('SANITY_E2E_DATASET'))
+
+studioE2EClient.datasets.list().then(async (datasets) => {
+  // If the dataset doesn't exist, create it
+  if (!datasets.find((ds) => ds.name === dataset)) {
+    const timer = startTimer(`Creating dataset ${dataset}`)
+    await studioE2EClient.datasets.create(dataset, {
+      aclMode: 'public',
+    })
+    timer.end()
+  }
+})

scripts/utils/envVars.ts

@@ -0,0 +1,13 @@
+export function readEnv<KnownEnvVar extends keyof NodeJS.ProcessEnv>(name: KnownEnvVar): string {
+  const val = findEnv<KnownEnvVar>(name)
+  if (val === undefined) {
+    throw new Error(`Missing required environment variable "${name}"`)
+  }
+  return val
+}
+
+export function findEnv<KnownEnvVar extends keyof NodeJS.ProcessEnv>(
+  name: KnownEnvVar,
+): string | undefined {
+  return process.env[name]
+}