Skip to content

Commit

Permalink
chore(e2e): run e2e tests on PRs on ephemeral datasets (#5046)
Browse files Browse the repository at this point in the history
* chore(e2e): run e2e tests in production build

* chore(e2e): run e2e tests on PRs on ephemeral datasets

- refactor: create a unified PR cleanup action
- refactor: moved envVars script to utils and make it more resuable

* chore(e2e): add missing env var to pr e2e tests
  • Loading branch information
binoy14 authored Oct 31, 2023
1 parent 275293c commit 8486c0a
Show file tree
Hide file tree
Showing 12 changed files with 131 additions and 31 deletions.
33 changes: 30 additions & 3 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,17 +62,19 @@ jobs:
- name: Build CLI
run: yarn build:cli # Needed for CLI tests

- name: Build E2E test studio
- name: Build E2E test studio on next
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }}
env:
# Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next
# Change the below to `secrets.SANITY_E2E_SESSION_TOKEN`
# Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github
SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }}
SANITY_E2E_DATASET: ${{ secrets.SANITY_E2E_DATASET }}
run: yarn e2e:build
run: yarn e2e:setup && yarn e2e:build

- name: Run end-to-end tests
- name: Run E2E tests on next
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }}
env:
# Missing in docs but in use
# here https://github.com/microsoft/playwright/blob/main/packages/playwright/src/reporters/blob.ts#L108
Expand All @@ -85,6 +87,31 @@ jobs:
SANITY_E2E_DATASET: ${{ secrets.SANITY_E2E_DATASET }}
run: yarn test:e2e --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}

- name: Build E2E test studio on PR
if: ${{ github.event_name == 'pull_request' }}
env:
# Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next
# Change the below to `secrets.SANITY_E2E_SESSION_TOKEN`
# Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github
SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }}
SANITY_E2E_DATASET: pr-${{ github.event.number }}
run: yarn e2e:setup && yarn e2e:build

- name: Run E2E tests on PR
if: ${{ github.event_name == 'pull_request' }}
env:
# Missing in docs but in use
# here https://github.com/microsoft/playwright/blob/main/packages/playwright/src/reporters/blob.ts#L108
PWTEST_BLOB_REPORT_NAME: ${{ matrix.project }}
# Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next
# Change the below to `secrets.SANITY_E2E_SESSION_TOKEN`
# Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github
SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }}
SANITY_E2E_DATASET: pr-${{ github.event.number }}
run: yarn test:e2e --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}

- uses: actions/upload-artifact@v3
if: always()
with:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
name: 'Cleanup: Documentation Report'
name: 'Cleanup: PR Merge/Closed'

on:
# Build on closed or merged PRs
pull_request:
types: [closed]

jobs:
reportTeardown:
pr-cleanup:
runs-on: ubuntu-latest
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
Expand All @@ -33,7 +33,14 @@ jobs:
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: yarn install --frozen-lockfile

- name: Remove datasets for closed PRs
- name: Remove E2E datasets for closed PRs
env:
SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
SANITY_E2E_PROJECT_ID: ${{ secrets.SANITY_E2E_PROJECT_ID }}
SANITY_E2E_DATASET: pr-${{ github.event.number }}
run: yarn e2e:cleanup

- name: Remove docs report datasets for closed PRs
env:
DOCS_REPORT_TOKEN: ${{ secrets.DOCS_REPORT_DATASET_TOKEN }}
DOCS_REPORT_DATASET: pr-${{ github.event.number }}
Expand Down
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@
"example:movies-studio": "yarn --cwd examples/movies-studio start",
"e2e:dev": "yarn --cwd dev/studio-e2e-testing dev",
"e2e:build": "yarn --cwd dev/studio-e2e-testing build",
"e2e:cleanup": "node -r dotenv-flow/config -r esbuild-register scripts/e2e/cleanup",
"e2e:preview": "yarn e2e:build && yarn --cwd dev/studio-e2e-testing start",
"e2e:setup": "node -r dotenv-flow/config -r esbuild-register scripts/e2e/setup",
"e2e:start": "yarn --cwd dev/studio-e2e-testing start",
"etl": "node -r dotenv-flow/config -r esbuild-register scripts/etl",
"init": "lerna clean --yes && run-s bootstrap build",
Expand Down
6 changes: 4 additions & 2 deletions scripts/doc-report/docClient.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
import {type SanityClient, createClient} from '@sanity/client'
import {sanityIdify} from '../utils/sanityIdify'
import {readEnv} from './envVars'
import {readEnv} from '../utils/envVars'

export type KnownEnvVar = 'DOCS_REPORT_DATASET' | 'DOCS_REPORT_TOKEN'

export function createDocClient(dataset: string): SanityClient {
return createClient({
projectId: 'c1zuxvqn',
dataset: sanityIdify(dataset),
token: readEnv('DOCS_REPORT_TOKEN'),
token: readEnv<KnownEnvVar>('DOCS_REPORT_TOKEN'),
apiVersion: '2023-02-03',
useCdn: false,
})
Expand Down
6 changes: 3 additions & 3 deletions scripts/doc-report/docReport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import {groupBy} from 'lodash'
import {combineLatest, map} from 'rxjs'
import prettier from 'prettier'
import {startTimer} from '../utils/startTimer'
import {createDocClient} from './docClient'
import {readEnv} from './envVars'
import {readEnv} from '../utils/envVars'
import {KnownEnvVar, createDocClient} from './docClient'

const QUERY = `*[_type=='exportSymbol'] {
_id,
Expand Down Expand Up @@ -35,7 +35,7 @@ interface TransformResult {
branchNotDocumented: number
}

const studioMetricsClient = createDocClient(readEnv('DOCS_REPORT_DATASET'))
const studioMetricsClient = createDocClient(readEnv<KnownEnvVar>('DOCS_REPORT_DATASET'))
const studioMetricsClientProduction = createDocClient('next')

function getDocumentationReport(symbols: ExportSymbol[]): Report[] {
Expand Down
6 changes: 3 additions & 3 deletions scripts/doc-report/docReportCleanup.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import {sanityIdify} from '../utils/sanityIdify'
import {startTimer} from '../utils/startTimer'
import {createDocClient} from './docClient'
import {readEnv} from './envVars'
import {readEnv} from '../utils/envVars'
import {KnownEnvVar, createDocClient} from './docClient'

const DATASET = readEnv('DOCS_REPORT_DATASET')
const DATASET = readEnv<KnownEnvVar>('DOCS_REPORT_DATASET')
const studioMetricsClient = createDocClient(DATASET)

const timer = startTimer(`Deleting dataset ${DATASET}`)
Expand Down
8 changes: 4 additions & 4 deletions scripts/doc-report/docReportCreate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ import readPackages from '../utils/readPackages'
import type {PackageManifest} from '../types'
import {sanityIdify} from '../utils/sanityIdify'
import {startTimer} from '../utils/startTimer'
import {createDocClient} from './docClient'
import {readEnv} from './envVars'
import {readEnv} from '../utils/envVars'
import {KnownEnvVar, createDocClient} from './docClient'

const ALLOWED_TAGS = ['public', 'alpha', 'beta', 'internal', 'experimental', 'deprecated']
interface Package {
Expand Down Expand Up @@ -194,9 +194,9 @@ function getPackageMutations(pkg: Package): Mutation[] {
})
}

const dataset = sanityIdify(readEnv('DOCS_REPORT_DATASET'))
const dataset = sanityIdify(readEnv<KnownEnvVar>('DOCS_REPORT_DATASET'))

const studioMetricsClient = createDocClient(readEnv('DOCS_REPORT_DATASET'))
const studioMetricsClient = createDocClient(readEnv<KnownEnvVar>('DOCS_REPORT_DATASET'))

studioMetricsClient.datasets.list().then(async (datasets) => {
// If the dataset doesn't exist, create it
Expand Down
13 changes: 0 additions & 13 deletions scripts/doc-report/envVars.ts

This file was deleted.

25 changes: 25 additions & 0 deletions scripts/e2e/cleanup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import {sanityIdify} from '../utils/sanityIdify'
import {startTimer} from '../utils/startTimer'
import {readEnv} from '../utils/envVars'
import {KnownEnvVar, createE2EClient} from './e2eClient'

const DATASET = readEnv<KnownEnvVar>('SANITY_E2E_DATASET')
const studioE2EClient = createE2EClient(DATASET)

const timer = startTimer(`Deleting dataset ${DATASET}`)

studioE2EClient.datasets
.delete(sanityIdify(DATASET))
.then((res) => {
if (res.deleted) {
console.log('Deleted dataset')
} else {
console.log('Dataset was not deleted')
}
})
.catch((err) => {
throw new Error(`Something went wrong! ${err?.response?.body?.message}`)
})
.finally(() => {
timer.end()
})
18 changes: 18 additions & 0 deletions scripts/e2e/e2eClient.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import {type SanityClient, createClient} from '@sanity/client'
import {sanityIdify} from '../utils/sanityIdify'
import {readEnv} from '../utils/envVars'

export type KnownEnvVar =
| 'SANITY_E2E_DATASET'
| 'SANITY_E2E_PROJECT_ID'
| 'SANITY_E2E_SESSION_TOKEN'

export function createE2EClient(dataset: string): SanityClient {
return createClient({
projectId: readEnv<KnownEnvVar>('SANITY_E2E_PROJECT_ID'),
dataset: sanityIdify(dataset),
token: readEnv<KnownEnvVar>('SANITY_E2E_SESSION_TOKEN'),
apiVersion: '2023-02-03',
useCdn: false,
})
}
19 changes: 19 additions & 0 deletions scripts/e2e/setup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import {readEnv} from '../utils/envVars'
import {sanityIdify} from '../utils/sanityIdify'
import {startTimer} from '../utils/startTimer'
import {KnownEnvVar, createE2EClient} from './e2eClient'

const dataset = sanityIdify(readEnv<KnownEnvVar>('SANITY_E2E_DATASET'))

const studioE2EClient = createE2EClient(readEnv<KnownEnvVar>('SANITY_E2E_DATASET'))

studioE2EClient.datasets.list().then(async (datasets) => {
// If the dataset doesn't exist, create it
if (!datasets.find((ds) => ds.name === dataset)) {
const timer = startTimer(`Creating dataset ${dataset}`)
await studioE2EClient.datasets.create(dataset, {
aclMode: 'public',
})
timer.end()
}
})
13 changes: 13 additions & 0 deletions scripts/utils/envVars.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
export function readEnv<KnownEnvVar extends keyof NodeJS.ProcessEnv>(name: KnownEnvVar): string {
const val = findEnv<KnownEnvVar>(name)
if (val === undefined) {
throw new Error(`Missing required environment variable "${name}"`)
}
return val
}

export function findEnv<KnownEnvVar extends keyof NodeJS.ProcessEnv>(
name: KnownEnvVar,
): string | undefined {
return process.env[name]
}

2 comments on commit 8486c0a

@vercel
Copy link

@vercel vercel bot commented on 8486c0a Oct 31, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

performance-studio – ./

performance-studio.sanity.build
performance-studio-git-next.sanity.build

@vercel
Copy link

@vercel vercel bot commented on 8486c0a Oct 31, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

test-studio – ./

test-studio-git-next.sanity.build
test-studio.sanity.build

Please sign in to comment.