From afc762fce71b073dbb6da1bbb64e67d41a9044c3 Mon Sep 17 00:00:00 2001 From: pieh Date: Wed, 23 Oct 2024 19:20:29 +0200 Subject: [PATCH 1/6] ci: decrease number of shards when running Vercel tests --- .github/workflows/test-e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 33f9e0266..6d6ab2933 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -56,12 +56,12 @@ jobs: run: | if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then VERSION_SELECTORS=[${{ github.event.inputs.versions }}] - echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]" >> $GITHUB_OUTPUT - echo "total=12" >> $GITHUB_OUTPUT + echo "group=[1, 2, 3, 4]" >> $GITHUB_OUTPUT + echo "total=4" >> $GITHUB_OUTPUT elif [ "${{ github.event_name }}" == "pull_request" ]; then VERSION_SELECTORS=[\"latest\"] - echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]" >> $GITHUB_OUTPUT - echo "total=12" >> $GITHUB_OUTPUT + echo "group=[1, 2, 3, 4]" >> $GITHUB_OUTPUT + echo "total=4" >> $GITHUB_OUTPUT else VERSION_SELECTORS=[\"latest\",\"canary\",\"14.2.15\",\"13.5.1\"] echo "group=[1, 2, 3, 4]" >> $GITHUB_OUTPUT From 960ef04973ad5211ab8439c852178ceedea7cc41 Mon Sep 17 00:00:00 2001 From: pieh Date: Thu, 24 Oct 2024 10:07:53 +0200 Subject: [PATCH 2/6] test: use PWD instead of hardcoded repo directory --- run-local-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-local-test.sh b/run-local-test.sh index f3b8251f1..5c1951975 100755 --- a/run-local-test.sh +++ b/run-local-test.sh @@ -15,7 +15,7 @@ export NEXT_TEST_MODE=deploy export RUNTIME_DIR=$(pwd) cp tests/netlify-deploy.ts ../next.js/test/lib/next-modes/netlify-deploy.ts cd ../next.js/ -git apply ../opennextjs-netlify/tests/e2e-utils.patch || git apply ../opennextjs-netlify/tests/e2e-utils-v2.patch +git apply $RUNTIME_DIR/tests/e2e-utils.patch || git apply $RUNTIME_DIR/tests/e2e-utils-v2.patch node run-tests.js --type e2e --debug --test-pattern $1 git checkout -- test/lib/e2e-utils.ts From 645f2203c40b3b62947bc03d3ca1906762ae92d4 Mon Sep 17 00:00:00 2001 From: pieh Date: Thu, 24 Oct 2024 10:08:34 +0200 Subject: [PATCH 3/6] test: collect build logs in Vercel e2e tests (like we do in our own e2e tests) --- tests/netlify-deploy.ts | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/tests/netlify-deploy.ts b/tests/netlify-deploy.ts index f68e52583..df878e222 100644 --- a/tests/netlify-deploy.ts +++ b/tests/netlify-deploy.ts @@ -6,15 +6,6 @@ import { tmpdir } from 'node:os' import path from 'path' import { NextInstance } from './base' -type NetlifyDeployResponse = { - name: string - site_id: string - site_name: string - deploy_id: string - deploy_url: string - logs: string -} - async function packNextRuntimeImpl() { const runtimePackDir = await fs.mkdtemp(path.join(tmpdir(), 'opennextjs-netlify-pack')) @@ -133,7 +124,7 @@ export class NextDeployInstance extends NextInstance { const deployRes = await execa( 'npx', - ['netlify', 'deploy', '--build', '--json', '--message', deployTitle ?? ''], + ['netlify', 'deploy', '--build', '--message', deployTitle ?? ''], { cwd: this.testDir, reject: false, @@ -142,17 +133,29 @@ export class NextDeployInstance extends NextInstance { if (deployRes.exitCode !== 0) { throw new Error( - `Failed to deploy project ${deployRes.stdout} ${deployRes.stderr} (${deployRes.exitCode})`, + `Failed to deploy project (${deployRes.exitCode}) ${deployRes.stdout} ${deployRes.stderr} `, ) } try { - const data: NetlifyDeployResponse = JSON.parse(deployRes.stdout) - this._url = data.deploy_url + const [url] = new RegExp(/https:.+\.netlify\.app/gm).exec(deployRes.stdout) || [] + if (!url) { + throw new Error('Could not extract the URL from the build logs') + } + const [deployID] = new URL(url).host.split('--') + this._url = url this._parsedUrl = new URL(this._url) - this._deployId = data.deploy_id - require('console').log(`Deployment URL: ${data.deploy_url}`) - require('console').log(`Logs: ${data.logs}`) + this._deployId = deployID + this._cliOutput = deployRes.stdout + deployRes.stdout + require('console').log(`Deployment URL: ${this._url}`) + + const [buildLogsUrl] = + new RegExp(/https:\/\/app\.netlify\.com\/sites\/.+\/deploys\/[0-9a-f]+/gm).exec( + deployRes.stdout, + ) || [] + if (buildLogsUrl) { + require('console').log(`Logs: ${buildLogsUrl}`) + } } catch (err) { console.error(err) throw new Error(`Failed to parse deploy output: ${deployRes.stdout}`) From 21bb07130570ff97ac24c47c631204cc50601436 Mon Sep 17 00:00:00 2001 From: pieh Date: Thu, 24 Oct 2024 10:09:03 +0200 Subject: [PATCH 4/6] test: skip esm-externals-false tests (they are just checking CLI output) --- tests/test-config.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test-config.json b/tests/test-config.json index 13cf56a21..b8fd95db1 100644 --- a/tests/test-config.json +++ b/tests/test-config.json @@ -124,7 +124,9 @@ { "file": "test/e2e/app-dir/app-static/app-static.test.ts", "reason": "Uses CLI output", - "tests": ["app-dir static/dynamic handling should warn for too many cache tags"] + "tests": [ + "app-dir static/dynamic handling should warn for too many cache tags" + ] }, { "file": "test/e2e/app-dir/parallel-routes-and-interception/parallel-routes-and-interception.test.ts", @@ -358,6 +360,10 @@ { "file": "test/e2e/app-dir/dynamic-io-request-apis/dynamic-io-request-apis.test", "reason": "Uses CLI output" + }, + { + "file": "test/e2e/next-config-warnings/esm-externals-false/esm-externals-false.test.ts", + "reason": "Uses CLI output" } ], "failures": [ From 1f5c7f7a1c2a445f914784f02f3647e21b1a4a2e Mon Sep 17 00:00:00 2001 From: pieh Date: Thu, 24 Oct 2024 12:01:36 +0200 Subject: [PATCH 5/6] ci: skip reporting about tests that use canary only features when testing stable --- tools/deno/junit2json.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/deno/junit2json.ts b/tools/deno/junit2json.ts index d1c133ebf..d38847056 100644 --- a/tools/deno/junit2json.ts +++ b/tools/deno/junit2json.ts @@ -117,6 +117,13 @@ function junitToJson(xmlData: { testsuites: JUnitTestSuites }): Array if (skippedTestsForFile?.some(({ name }) => name === testCase['@name'])) { continue } + + // skip reporting on tests that even fail to deploy because they rely on experiments not available + // in currently tested version + if (testCase.failure?.includes('CanaryOnlyError')) { + continue + } + const status = testCase.failure ? 'failed' : 'passed' const test: TestCase = { name: testCase['@name'], From 7faa8763172a6aad97225872879e6b4d9112f4fb Mon Sep 17 00:00:00 2001 From: pieh Date: Thu, 24 Oct 2024 15:30:00 +0200 Subject: [PATCH 6/6] test: fix typo when setting _cliOutput --- tests/netlify-deploy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/netlify-deploy.ts b/tests/netlify-deploy.ts index df878e222..cbc6ab498 100644 --- a/tests/netlify-deploy.ts +++ b/tests/netlify-deploy.ts @@ -146,7 +146,7 @@ export class NextDeployInstance extends NextInstance { this._url = url this._parsedUrl = new URL(this._url) this._deployId = deployID - this._cliOutput = deployRes.stdout + deployRes.stdout + this._cliOutput = deployRes.stdout + deployRes.stderr require('console').log(`Deployment URL: ${this._url}`) const [buildLogsUrl] =