feat: support analytics engine in local/remote dev (#6666)

This adds "support" for analytics engine datasets for `wrangler dev`. Specifically, it simply mocks the AE bindings so that they exist while developing (and don't throw when accessed). This does NOT add support in Pages, though we very well could do so in a similar way in a followup.
cloudflare · Sep 10, 2024 · 4107f57 · 4107f57
1 parent 67711c2
commit 4107f57
Show file tree

Hide file tree

Showing 15 changed files with 185 additions and 2 deletions.
diff --git a/.changeset/long-rules-wait.md b/.changeset/long-rules-wait.md
@@ -0,0 +1,9 @@
+---
+"wrangler": minor
+---
+
+feat: support analytics engine in local/remote dev
+
+This adds "support" for analytics engine datasets for `wrangler dev`. Specifically, it simply mocks the AE bindings so that they exist while developing (and don't throw when accessed).
+
+This does NOT add support in Pages, though we very well could do so in a similar way in a followup.
diff --git a/packages/wrangler/e2e/dev.test.ts b/packages/wrangler/e2e/dev.test.ts
@@ -602,6 +602,104 @@ describe("writes debug logs to hidden file", () => {
 	});
 });
 
+describe("analytics engine", () => {
+	describe.each([
+		{ cmd: "wrangler dev" },
+		{ cmd: "wrangler dev --x-dev-env" },
+		{ cmd: "wrangler dev --remote" },
+		{ cmd: "wrangler dev --remote --x-dev-env" },
+	])("mock analytics engine datasets: $cmd", ({ cmd }) => {
+		describe("module worker", () => {
+			it("analytics engine datasets are mocked in dev", async () => {
+				const helper = new WranglerE2ETestHelper();
+				await helper.seed({
+					"wrangler.toml": dedent`
+				name = "${workerName}"
+				main = "src/index.ts"
+				compatibility_date = "2024-08-08"
+
+				[[analytics_engine_datasets]]
+				binding = "ANALYTICS_BINDING"
+				dataset = "ANALYTICS_DATASET"
+			`,
+					"src/index.ts": dedent`
+				export default {
+					fetch(request, env) {
+						// let's make an analytics call
+						env.ANALYTICS_BINDING.writeDataPoint({
+							'blobs': ["Seattle", "USA", "pro_sensor_9000"], // City, State
+							'doubles': [25, 0.5],
+							'indexes': ["a3cd45"]
+						});
+						// and return a response
+						return new Response("successfully wrote datapoint from module worker");
+					}
+				}`,
+					"package.json": dedent`
+				{
+					"name": "worker",
+					"version": "0.0.0",
+					"private": true
+				}
+				`,
+				});
+				const worker = helper.runLongLived(cmd);
+
+				const { url } = await worker.waitForReady();
+
+				const text = await fetchText(url);
+				expect(text).toContain(
+					`successfully wrote datapoint from module worker`
+				);
+			});
+		});
+
+		describe("service worker", async () => {
+			it("analytics engine datasets are mocked in dev", async () => {
+				const helper = new WranglerE2ETestHelper();
+				await helper.seed({
+					"wrangler.toml": dedent`
+				name = "${workerName}"
+				main = "src/index.ts"
+				compatibility_date = "2024-08-08"
+
+				[[analytics_engine_datasets]]
+				binding = "ANALYTICS_BINDING"
+				dataset = "ANALYTICS_DATASET"
+			`,
+					"src/index.ts": dedent`
+							addEventListener("fetch", (event) => {
+								// let's make an analytics call
+								ANALYTICS_BINDING.writeDataPoint({
+									blobs: ["Seattle", "USA", "pro_sensor_9000"], // City, State
+									doubles: [25, 0.5],
+									indexes: ["a3cd45"],
+								});
+								// and return a response
+								event.respondWith(new Response("successfully wrote datapoint from service worker"));
+							});
+				`,
+					"package.json": dedent`
+				{
+					"name": "worker",
+					"version": "0.0.0",
+					"private": true
+				}
+				`,
+				});
+				const worker = helper.runLongLived(cmd);
+
+				const { url } = await worker.waitForReady();
+
+				const text = await fetchText(url);
+				expect(text).toContain(
+					`successfully wrote datapoint from service worker`
+				);
+			});
+		});
+	});
+});
+
 describe("zone selection", () => {
 	it("defaults to a workers.dev preview", async () => {
 		const helper = new WranglerE2ETestHelper();

diff --git a/packages/wrangler/src/__tests__/navigator-user-agent.test.ts b/packages/wrangler/src/__tests__/navigator-user-agent.test.ts
@@ -114,6 +114,7 @@ describe("defineNavigatorUserAgent is respected", () => {
 				additionalModules: [],
 				moduleCollector: noopModuleCollector,
 				serveLegacyAssetsFromWorker: false,
+				mockAnalyticsEngineDatasets: [],
 				doBindings: [],
 				define: {},
 				alias: {},
@@ -176,6 +177,7 @@ describe("defineNavigatorUserAgent is respected", () => {
 				doBindings: [],
 				define: {},
 				alias: {},
+				mockAnalyticsEngineDatasets: [],
 				checkFetch: false,
 				targetConsumer: "deploy",
 				local: true,

diff --git a/packages/wrangler/src/api/startDevWorker/BundlerController.ts b/packages/wrangler/src/api/startDevWorker/BundlerController.ts
@@ -115,6 +115,8 @@ export class BundlerController extends Controller<BundlerControllerEventMap> {
 						nodejsCompatMode: config.build.nodejsCompatMode,
 						define: config.build.define,
 						checkFetch: true,
+						mockAnalyticsEngineDatasets:
+							bindings.analytics_engine_datasets ?? [],
 						alias: config.build.alias,
 						legacyAssets: config.legacy?.legacyAssets,
 						// enable the cache when publishing
@@ -227,6 +229,7 @@ export class BundlerController extends Controller<BundlerControllerEventMap> {
 				noBundle: !config.build?.bundle,
 				findAdditionalModules: config.build?.findAdditionalModules,
 				durableObjects: bindings?.durable_objects ?? { bindings: [] },
+				mockAnalyticsEngineDatasets: bindings.analytics_engine_datasets ?? [],
 				local: !config.dev?.remote,
 				// startDevWorker only applies to "dev"
 				targetConsumer: "dev",

diff --git a/packages/wrangler/src/deploy/deploy.ts b/packages/wrangler/src/deploy/deploy.ts
@@ -551,6 +551,8 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
 						checkFetch: false,
 						alias: config.alias,
 						legacyAssets: config.legacy_assets,
+						// We do not mock AE datasets when deploying
+						mockAnalyticsEngineDatasets: [],
 						// enable the cache when publishing
 						bypassAssetCache: false,
 						// We want to know if the build is for development or publishing

diff --git a/packages/wrangler/src/deployment-bundle/bundle.ts b/packages/wrangler/src/deployment-bundle/bundle.ts
@@ -86,6 +86,7 @@ export type BundleOptions = {
 	define: Config["define"];
 	alias: Config["alias"];
 	checkFetch: boolean;
+	mockAnalyticsEngineDatasets: Config["analytics_engine_datasets"];
 	targetConsumer: "dev" | "deploy";
 	testScheduled?: boolean;
 	inject?: string[];
@@ -122,6 +123,7 @@ export async function bundleWorker(
 		alias,
 		define,
 		checkFetch,
+		mockAnalyticsEngineDatasets,
 		legacyAssets,
 		bypassAssetCache,
 		targetConsumer,
@@ -148,6 +150,21 @@ export async function bundleWorker(
 	// At this point, we take the opportunity to "wrap" the worker with middleware.
 	const middlewareToLoad: MiddlewareLoader[] = [];
 
+	if (
+		targetConsumer === "dev" &&
+		mockAnalyticsEngineDatasets &&
+		mockAnalyticsEngineDatasets.length > 0
+	) {
+		middlewareToLoad.push({
+			name: "mock-analytics-engine",
+			path: "templates/middleware/middleware-mock-analytics-engine.ts",
+			config: {
+				bindings: mockAnalyticsEngineDatasets.map(({ binding }) => binding),
+			},
+			supports: ["modules", "service-worker"],
+		});
+	}
+
 	if (
 		targetConsumer === "dev" &&
 		!process.env.WRANGLER_DISABLE_REQUEST_BODY_DRAINING

diff --git a/packages/wrangler/src/dev/dev.tsx b/packages/wrangler/src/dev/dev.tsx
@@ -606,6 +606,7 @@ function DevSession(props: DevSessionProps) {
 		alias: props.alias,
 		noBundle: props.noBundle,
 		findAdditionalModules: props.findAdditionalModules,
+		mockAnalyticsEngineDatasets: props.bindings.analytics_engine_datasets ?? [],
 		legacyAssets: props.legacyAssetsConfig,
 		durableObjects: props.bindings.durable_objects || { bindings: [] },
 		local: props.local,

diff --git a/packages/wrangler/src/dev/start-server.ts b/packages/wrangler/src/dev/start-server.ts
@@ -243,6 +243,7 @@ export async function startDevServer(
 		testScheduled: props.testScheduled,
 		local: props.local,
 		doBindings: props.bindings.durable_objects?.bindings ?? [],
+		mockAnalyticsEngineDatasets: props.bindings.analytics_engine_datasets ?? [],
 		projectRoot: props.projectRoot,
 		defineNavigatorUserAgent: isNavigatorDefined(
 			props.compatibilityDate,
@@ -416,6 +417,7 @@ async function runEsbuild({
 	testScheduled,
 	local,
 	doBindings,
+	mockAnalyticsEngineDatasets,
 	projectRoot,
 	defineNavigatorUserAgent,
 }: {
@@ -438,6 +440,7 @@ async function runEsbuild({
 	testScheduled?: boolean;
 	local: boolean;
 	doBindings: DurableObjectBindings;
+	mockAnalyticsEngineDatasets: Config["analytics_engine_datasets"];
 	projectRoot: string | undefined;
 	defineNavigatorUserAgent: boolean;
 }): Promise<EsbuildBundle> {
@@ -475,6 +478,7 @@ async function runEsbuild({
 					nodejsCompatMode,
 					define,
 					checkFetch: true,
+					mockAnalyticsEngineDatasets,
 					alias,
 					legacyAssets,
 					// disable the cache in dev

diff --git a/packages/wrangler/src/dev/use-esbuild.ts b/packages/wrangler/src/dev/use-esbuild.ts
@@ -51,6 +51,7 @@ export type EsbuildBundleProps = {
 	noBundle: boolean;
 	findAdditionalModules: boolean | undefined;
 	durableObjects: Config["durable_objects"];
+	mockAnalyticsEngineDatasets: Config["analytics_engine_datasets"];
 	local: boolean;
 	targetConsumer: "dev" | "deploy";
 	testScheduled: boolean;
@@ -78,6 +79,7 @@ export function runBuild(
 		alias,
 		noBundle,
 		findAdditionalModules,
+		mockAnalyticsEngineDatasets,
 		durableObjects,
 		local,
 		targetConsumer,
@@ -103,6 +105,7 @@ export function runBuild(
 		noBundle: boolean;
 		findAdditionalModules: boolean | undefined;
 		durableObjects: Config["durable_objects"];
+		mockAnalyticsEngineDatasets: Config["analytics_engine_datasets"];
 		local: boolean;
 		targetConsumer: "dev" | "deploy";
 		testScheduled: boolean;
@@ -183,6 +186,7 @@ export function runBuild(
 						alias,
 						define,
 						checkFetch: true,
+						mockAnalyticsEngineDatasets,
 						legacyAssets,
 						// disable the cache in dev
 						bypassAssetCache: true,
@@ -264,6 +268,7 @@ export function useEsbuild({
 	define,
 	noBundle,
 	findAdditionalModules,
+	mockAnalyticsEngineDatasets,
 	durableObjects,
 	local,
 	targetConsumer,
@@ -295,6 +300,7 @@ export function useEsbuild({
 				noBundle,
 				findAdditionalModules,
 				durableObjects,
+				mockAnalyticsEngineDatasets,
 				local,
 				targetConsumer,
 				testScheduled,
@@ -327,6 +333,7 @@ export function useEsbuild({
 		alias,
 		define,
 		legacyAssets,
+		mockAnalyticsEngineDatasets,
 		durableObjects,
 		local,
 		targetConsumer,

diff --git a/packages/wrangler/src/pages/functions/buildPlugin.ts b/packages/wrangler/src/pages/functions/buildPlugin.ts
@@ -106,6 +106,8 @@ export function buildPluginFromFunctions({
 		],
 		serveLegacyAssetsFromWorker: false,
 		checkFetch: local,
+		// TODO: mock AE datasets in Pages functions for dev
+		mockAnalyticsEngineDatasets: [],
 		targetConsumer: local ? "dev" : "deploy",
 		forPages: true,
 		local,

diff --git a/packages/wrangler/src/pages/functions/buildWorker.ts b/packages/wrangler/src/pages/functions/buildWorker.ts
@@ -72,6 +72,8 @@ export function buildWorkerFromFunctions({
 		sourcemap,
 		watch,
 		nodejsCompatMode,
+		// TODO: mock AE datasets in Pages functions for dev
+		mockAnalyticsEngineDatasets: [],
 		define: {
 			__FALLBACK_SERVICE__: JSON.stringify(fallbackService),
 		},
@@ -153,6 +155,8 @@ export function buildRawWorker({
 		sourcemap,
 		watch,
 		nodejsCompatMode,
+		// TODO: mock AE datasets in Pages functions for dev
+		mockAnalyticsEngineDatasets: [],
 		define: {},
 		alias: {},
 		doBindings: [], // Pages functions don't support internal Durable Objects

diff --git a/packages/wrangler/src/versions/upload.ts b/packages/wrangler/src/versions/upload.ts
@@ -311,6 +311,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
 						alias: { ...config.alias, ...props.alias },
 						checkFetch: false,
 						legacyAssets: config.legacy_assets,
+						mockAnalyticsEngineDatasets: [],
 						// enable the cache when publishing
 						bypassAssetCache: false,
 						// We want to know if the build is for development or publishing

diff --git a/packages/wrangler/templates/middleware/middleware-mock-analytics-engine.d.ts b/packages/wrangler/templates/middleware/middleware-mock-analytics-engine.d.ts
@@ -0,0 +1,3 @@
+declare module "config:middleware/mock-analytics-engine" {
+	export const bindings: string[];
+}
diff --git a/packages/wrangler/templates/middleware/middleware-mock-analytics-engine.ts b/packages/wrangler/templates/middleware/middleware-mock-analytics-engine.ts
@@ -0,0 +1,30 @@
+/// <reference path="middleware-mock-analytics-engine.d.ts"/>
+
+import { bindings } from "config:middleware/mock-analytics-engine";
+import type { Middleware } from "./common";
+
+const bindingsEnv = Object.fromEntries(
+	bindings.map((binding) => [
+		binding,
+		{
+			writeDataPoint() {
+				// no op in dev
+			},
+		},
+	])
+) satisfies Record<string, AnalyticsEngineDataset>;
+
+const analyticsEngine: Middleware = async (
+	request,
+	env,
+	_ctx,
+	middlewareCtx
+) => {
+	// we're going to directly modify env so it maintains referential equality
+	for (const binding of bindings) {
+		env[binding] ??= bindingsEnv[binding];
+	}
+	return await middlewareCtx.next(request, env);
+};
+
+export default analyticsEngine;
diff --git a/packages/wrangler/templates/middleware/middleware-serve-static-assets.ts b/packages/wrangler/templates/middleware/middleware-serve-static-assets.ts
@@ -14,7 +14,7 @@ import type * as kvAssetHandler from "@cloudflare/kv-asset-handler";
 
 const ASSET_MANIFEST = JSON.parse(manifest);
 
-const staticAssets: Middleware = async (request, env, _ctx, middlewareCtx) => {
+const staticAssets: Middleware = async (request, env, ctx, middlewareCtx) => {
 	let options: Partial<Options> = {
 		ASSET_MANIFEST,
 		ASSET_NAMESPACE: env.__STATIC_CONTENT,
@@ -27,7 +27,7 @@ const staticAssets: Middleware = async (request, env, _ctx, middlewareCtx) => {
 			{
 				request,
 				waitUntil(promise) {
-					return _ctx.waitUntil(promise);
+					return ctx.waitUntil(promise);
 				},
 			},
 			options