test: skip reporting CanaryOnly failures for stable version tests (#2698)

* ci: decrease number of shards when running Vercel tests

* test: use PWD instead of hardcoded repo directory

* test: collect build logs in Vercel e2e tests (like we do in our own e2e tests)

* test: skip esm-externals-false tests (they are just checking CLI output)

* ci: skip reporting about tests that use canary only features when testing stable

* test: fix typo when setting _cliOutput

Author: pieh

.github/workflows/test-e2e.yml

@@ -56,12 +56,12 @@ jobs:
         run: |
           if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
             VERSION_SELECTORS=[${{ github.event.inputs.versions }}]
-            echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]" >> $GITHUB_OUTPUT
-            echo "total=12" >> $GITHUB_OUTPUT
+            echo "group=[1, 2, 3, 4]" >> $GITHUB_OUTPUT
+            echo "total=4" >> $GITHUB_OUTPUT
           elif [ "${{ github.event_name }}" == "pull_request" ]; then
             VERSION_SELECTORS=[\"latest\"]
-            echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]" >> $GITHUB_OUTPUT
-            echo "total=12" >> $GITHUB_OUTPUT
+            echo "group=[1, 2, 3, 4]" >> $GITHUB_OUTPUT
+            echo "total=4" >> $GITHUB_OUTPUT
           else
             VERSION_SELECTORS=[\"latest\",\"canary\",\"14.2.15\",\"13.5.1\"]
             echo "group=[1, 2, 3, 4]" >> $GITHUB_OUTPUT

run-local-test.sh

@@ -15,7 +15,7 @@ export NEXT_TEST_MODE=deploy
 export RUNTIME_DIR=$(pwd)
 cp tests/netlify-deploy.ts ../next.js/test/lib/next-modes/netlify-deploy.ts
 cd ../next.js/
-git apply ../opennextjs-netlify/tests/e2e-utils.patch || git apply ../opennextjs-netlify/tests/e2e-utils-v2.patch
+git apply $RUNTIME_DIR/tests/e2e-utils.patch || git apply $RUNTIME_DIR/tests/e2e-utils-v2.patch
 node run-tests.js --type e2e --debug --test-pattern $1
 git checkout -- test/lib/e2e-utils.ts
 

tests/netlify-deploy.ts

@@ -6,15 +6,6 @@ import { tmpdir } from 'node:os'
 import path from 'path'
 import { NextInstance } from './base'
 
-type NetlifyDeployResponse = {
-  name: string
-  site_id: string
-  site_name: string
-  deploy_id: string
-  deploy_url: string
-  logs: string
-}
-
 async function packNextRuntimeImpl() {
   const runtimePackDir = await fs.mkdtemp(path.join(tmpdir(), 'opennextjs-netlify-pack'))
 
@@ -133,7 +124,7 @@ export class NextDeployInstance extends NextInstance {
 
     const deployRes = await execa(
       'npx',
-      ['netlify', 'deploy', '--build', '--json', '--message', deployTitle ?? ''],
+      ['netlify', 'deploy', '--build', '--message', deployTitle ?? ''],
       {
         cwd: this.testDir,
         reject: false,
@@ -142,17 +133,29 @@ export class NextDeployInstance extends NextInstance {
 
     if (deployRes.exitCode !== 0) {
       throw new Error(
-        `Failed to deploy project ${deployRes.stdout} ${deployRes.stderr} (${deployRes.exitCode})`,
+        `Failed to deploy project (${deployRes.exitCode}) ${deployRes.stdout} ${deployRes.stderr} `,
       )
     }
 
     try {
-      const data: NetlifyDeployResponse = JSON.parse(deployRes.stdout)
-      this._url = data.deploy_url
+      const [url] = new RegExp(/https:.+\.netlify\.app/gm).exec(deployRes.stdout) || []
+      if (!url) {
+        throw new Error('Could not extract the URL from the build logs')
+      }
+      const [deployID] = new URL(url).host.split('--')
+      this._url = url
       this._parsedUrl = new URL(this._url)
-      this._deployId = data.deploy_id
-      require('console').log(`Deployment URL: ${data.deploy_url}`)
-      require('console').log(`Logs: ${data.logs}`)
+      this._deployId = deployID
+      this._cliOutput = deployRes.stdout + deployRes.stderr
+      require('console').log(`Deployment URL: ${this._url}`)
+
+      const [buildLogsUrl] =
+        new RegExp(/https:\/\/app\.netlify\.com\/sites\/.+\/deploys\/[0-9a-f]+/gm).exec(
+          deployRes.stdout,
+        ) || []
+      if (buildLogsUrl) {
+        require('console').log(`Logs: ${buildLogsUrl}`)
+      }
     } catch (err) {
       console.error(err)
       throw new Error(`Failed to parse deploy output: ${deployRes.stdout}`)

tests/test-config.json

@@ -124,7 +124,9 @@
     {
       "file": "test/e2e/app-dir/app-static/app-static.test.ts",
       "reason": "Uses CLI output",
-      "tests": ["app-dir static/dynamic handling should warn for too many cache tags"]
+      "tests": [
+        "app-dir static/dynamic handling should warn for too many cache tags"
+      ]
     },
     {
       "file": "test/e2e/app-dir/parallel-routes-and-interception/parallel-routes-and-interception.test.ts",
@@ -358,6 +360,10 @@
     {
       "file": "test/e2e/app-dir/dynamic-io-request-apis/dynamic-io-request-apis.test",
       "reason": "Uses CLI output"
+    },
+    {
+      "file": "test/e2e/next-config-warnings/esm-externals-false/esm-externals-false.test.ts",
+      "reason": "Uses CLI output"
     }
   ],
   "failures": [

tools/deno/junit2json.ts

@@ -117,6 +117,13 @@ function junitToJson(xmlData: { testsuites: JUnitTestSuites }): Array<TestSuite>
       if (skippedTestsForFile?.some(({ name }) => name === testCase['@name'])) {
         continue
       }
+
+      // skip reporting on tests that even fail to deploy because they rely on experiments not available
+      // in currently tested version
+      if (testCase.failure?.includes('CanaryOnlyError')) {
+        continue
+      }
+
       const status = testCase.failure ? 'failed' : 'passed'
       const test: TestCase = {
         name: testCase['@name'],