Granular c3 e2e caching

.github/actions/run-c3-e2e/action.yml

@@ -49,7 +49,7 @@ runs:
     - name: E2E Tests
       id: run-e2e
       shell: bash
-      run: pnpm run test:e2e --filter create-cloudflare
+      run: pnpm run test:e2e:create-cloudflare
       env:
         CLOUDFLARE_API_TOKEN: ${{ inputs.apiToken }}
         CLOUDFLARE_ACCOUNT_ID: ${{ inputs.accountId }}

package.json

@@ -24,6 +24,7 @@
 		"test": "vitest run --no-file-parallelism && dotenv -- turbo test --filter=wrangler --filter=miniflare --filter=kv-asset-handler --filter=@cloudflare/vitest-pool-workers --filter=@cloudflare/vitest-pool-workers-examples",
 		"test:ci": "dotenv -- turbo test:ci",
 		"test:e2e": "dotenv -- turbo test:e2e",
+		"test:e2e:create-cloudflare": "node -r esbuild-register tools/e2e/runIndividualE2ETests.ts",
 		"test:e2e:wrangler": "node -r esbuild-register tools/e2e/runIndividualE2EFiles.ts",
 		"test:watch": "turbo test:watch",
 		"type:tests": "dotenv -- turbo type:tests"

packages/create-cloudflare/e2e-tests/definitions.ts

@@ -0,0 +1,20 @@
+export const frameworks = [
+	"astro",
+	"docusaurus",
+	"analog",
+	"angular",
+	"gatsby",
+	"hono",
+	"qwik",
+	"remix",
+	"next",
+	"nuxt",
+	"react",
+	"solid",
+	"svelte",
+	"vue",
+] as const;
+type Framework = (typeof frameworks)[number];
+export const frameworkMap = Object.fromEntries(
+	frameworks.map((f) => [f, f]),
+) as Record<Framework, string>;

packages/create-cloudflare/e2e-tests/frameworks.test.ts

@@ -17,6 +17,7 @@
 } from "vitest";
 import { deleteProject, deleteWorker } from "../scripts/common";
 import { getFrameworkMap } from "../src/templates";
+import { frameworkMap } from "./definitions";
 import { getFrameworkToTest } from "./frameworkToTest";
 import {
 	createTestLogStream,
@@ -68,8 +69,8 @@
 const { name: pm, npx } = detectPackageManager();
 
 // These are ordered based on speed and reliability for ease of debugging
-const frameworkTests: Record<string, FrameworkTestConfig> = {
-	astro: {
+export const frameworkTests: Record<string, FrameworkTestConfig> = {
+	[frameworkMap.astro]: {
 		testCommitMessage: true,
 		quarantine: true,
 		unsupportedOSs: ["win32"],
@@ -97,7 +98,7 @@
 			"strict",
 		],
 	},
-	docusaurus: {
+	[frameworkMap.docusaurus]: {
 		unsupportedPms: ["bun"],
 		testCommitMessage: true,
 		unsupportedOSs: ["win32"],
@@ -114,7 +115,7 @@
 			},
 		],
 	},
-	analog: {
+	[frameworkMap.analog]: {
 		testCommitMessage: true,
 		timeout: LONG_TIMEOUT,
 		unsupportedOSs: ["win32"],
@@ -142,7 +143,7 @@
 		flags: ["--skipTailwind"],
 		quarantine: true,
 	},
-	angular: {
+	[frameworkMap.angular]: {
 		testCommitMessage: true,
 		timeout: LONG_TIMEOUT,
 		unsupportedOSs: ["win32"],
@@ -152,7 +153,7 @@
 		},
 		flags: ["--style", "sass"],
 	},
-	gatsby: {
+	[frameworkMap.gatsby]: {
 		unsupportedPms: ["bun", "pnpm"],
 		promptHandlers: [
 			{
@@ -167,7 +168,7 @@
 			expectedText: "Gatsby!",
 		},
 	},
-	hono: {
+	[frameworkMap.hono]: {
 		testCommitMessage: false,
 		unsupportedOSs: ["win32"],
 		verifyDeploy: {
@@ -181,7 +182,7 @@
 			},
 		],
 	},
-	qwik: {
+	[frameworkMap.qwik]: {
 		promptHandlers: [
 			{
 				matcher: /Yes looks good, finish update/,
@@ -210,7 +211,7 @@
 			expectedText: "C3_TEST",
 		},
 	},
-	remix: {
+	[frameworkMap.remix]: {
 		testCommitMessage: true,
 		timeout: LONG_TIMEOUT,
 		unsupportedPms: ["yarn"],
@@ -235,7 +236,7 @@
 		},
 		flags: ["--typescript", "--no-install", "--no-git-init"],
 	},
-	next: {
+	[frameworkMap.next]: {
 		promptHandlers: [
 			{
 				matcher: /Do you want to use the next-on-pages eslint-plugin\?/,
@@ -263,7 +264,7 @@
 			"@/*",
 		],
 	},
-	nuxt: {
+	[frameworkMap.nuxt]: {
 		testCommitMessage: true,
 		timeout: LONG_TIMEOUT,
 		unsupportedOSs: ["win32"],
@@ -286,7 +287,7 @@
 			expectedText: "C3_TEST",
 		},
 	},
-	react: {
+	[frameworkMap.react]: {
 		promptHandlers: [
 			{
 				matcher: /Select a variant:/,
@@ -302,7 +303,7 @@
 			expectedText: "Vite + React",
 		},
 	},
-	solid: {
+	[frameworkMap.solid]: {
 		promptHandlers: [
 			{
 				matcher: /Which template would you like to use/,
@@ -322,7 +323,7 @@
 			expectedText: "Hello world",
 		},
 	},
-	svelte: {
+	[frameworkMap.svelte]: {
 		promptHandlers: [
 			{
 				matcher: /Which Svelte app template/,
@@ -355,7 +356,7 @@
 			expectedText: "C3_TEST",
 		},
 	},
-	vue: {
+	[frameworkMap.vue]: {
 		testCommitMessage: true,
 		unsupportedOSs: ["win32"],
 		verifyDeploy: {
@@ -368,7 +369,7 @@
 };
 
 describe.concurrent(`E2E: Web frameworks`, () => {
 	let frameworkMap: TemplateMap;
 	let logStream: WriteStream;
 
 	beforeAll(async (ctx) => {
@@ -576,7 +577,7 @@
 		return;
 	}
 
 	const frameworkMap = getFrameworkMap({ experimental: false });
 	const template = frameworkMap[framework];
 
 	// Run the devserver on a random port to avoid colliding with other tests

tools/e2e/runIndividualE2ETests.ts

@@ -0,0 +1,38 @@
+/**
+ * Turbo only supports caching on the individual task level, but for C3's
+ * e2e tests we want to support caching on a more granular basis—at the file level & test level.
+ *
+ * As such, we run the `test:e2e` turbo task multiple times—once per e2e test file, and once per framework test
+ * with different arguments, ensuring that each file's tests can be cached individually.
+ *
+ * The intended flow here is that CI will run this file, which will trigger turbo to run
+ * an individual task for each C3 e2e test file & test, using `execSync`.
+ */
+import { execSync } from "child_process";
+import { readdirSync } from "fs";
+import { frameworks } from "../../packages/create-cloudflare/e2e-tests/definitions";
+
+// Get a list of e2e test files, each of which should have an associated script
+const e2eTests = readdirSync("packages/create-cloudflare/e2e-tests");
+
+const tasks = new Set<string>();
+
+for (const file of e2eTests) {
+	if (file === "frameworks.test.ts") {
+		for (const framework of frameworks) {
+			tasks.add(
+				`pnpm test:e2e --log-order=stream --summarize --filter create-cloudflare --concurrency 1 -- run ./e2e-tests/${file} -t ${framework}`
+			);
+		}
+	} else if (file.endsWith(".test.ts")) {
+		tasks.add(
+			`pnpm test:e2e --log-order=stream --summarize --filter create-cloudflare --concurrency 1 -- run ./e2e-tests/${file}`
+		);
+	}
+}
+
+for (const task of tasks.values()) {
+	execSync(task, {
+		stdio: "inherit",
+	});
+}