fix: reduce the number of parallel file reads on Windows to avoid EMF…

…ILE type errors (#6009)

* use createBatches to batch assets for Pages assets upload

* pages assets are already batched
don't batch a batch

* reduce pages assets MAX_BUCKET_FILE_COUNT on windows

* Add changeset

* Add comment

---------

Co-authored-by: Peter Bacon Darwin <[email protected]>

.changeset/twenty-jeans-matter.md

@@ -0,0 +1,7 @@
+---
+"wrangler": patch
+---
+
+fix: reduce the number of parallel file reads on Windows to avoid EMFILE type errors
+
+Fixes #1586

packages/wrangler/src/__tests__/utils/create-batches.test.ts

@@ -0,0 +1,30 @@
+import { createBatches } from "../../utils/create-batches";
+
+// split a template string into an array of chars (convenience util to make writing inputs/outputs easier)
+const s = (input: TemplateStringsArray) => input[0].split("");
+
+describe("createBatches", () => {
+	const data = [
+		{
+			input: s`abcde`,
+			batchSize: 2,
+			output: [s`ab`, s`cd`, s`e`],
+		},
+		{
+			input: s`abcdefgh`,
+			batchSize: 3,
+			output: [s`abc`, s`def`, s`gh`],
+		},
+		{
+			input: s`abcdefghijklmnopqrstuvwxyz`,
+			batchSize: 6,
+			output: [s`abcdef`, s`ghijkl`, s`mnopqr`, s`stuvwx`, s`yz`],
+		},
+	];
+
+	for (const { input, batchSize, output } of data) {
+		test(`${input} in batches of ${batchSize}`, () => {
+			expect([...createBatches(input, batchSize)]).toEqual(output);
+		});
+	}
+});

packages/wrangler/src/init.ts

@@ -17,6 +17,7 @@ import { getPackageManager } from "./package-manager";
 import { parsePackageJSON, parseTOML, readFileSync } from "./parse";
 import { getBasePath } from "./paths";
 import { requireAuth } from "./user";
+import { createBatches } from "./utils/create-batches";
 import * as shellquote from "./utils/shell-quote";
 import { CommandLineArgsError, printWranglerBanner } from "./index";
 import type { RawConfig } from "./config";
@@ -1153,12 +1154,6 @@ export function mapBindings(bindings: WorkerMetadataBinding[]): RawConfig {
 	);
 }
 
-function* createBatches<T>(array: T[], size: number): IterableIterator<T[]> {
-	for (let i = 0; i < array.length; i += size) {
-		yield array.slice(i, i + size);
-	}
-}
-
 /** Assert that there is no type argument passed. */
 function assertNoTypeArg(args: InitArgs) {
 	if (args.type) {

packages/wrangler/src/pages/constants.ts

@@ -1,10 +1,14 @@
 import { version as wranglerVersion } from "../../package.json";
 
+const isWindows = process.platform === "win32";
+
 export const MAX_ASSET_COUNT = 20_000;
 export const MAX_ASSET_SIZE = 25 * 1024 * 1024;
 export const PAGES_CONFIG_CACHE_FILENAME = "pages.json";
 export const MAX_BUCKET_SIZE = 40 * 1024 * 1024;
-export const MAX_BUCKET_FILE_COUNT = 2000;
+// Reduce the maximum number of files in a bucket on Windows
+// This helps to avoid EMFILE errors when reading them into memory.
+export const MAX_BUCKET_FILE_COUNT = isWindows ? 1000 : 2000;
 export const BULK_UPLOAD_CONCURRENCY = 3;
 export const MAX_UPLOAD_ATTEMPTS = 5;
 export const MAX_UPLOAD_GATEWAY_ERRORS = 5;

packages/wrangler/src/utils/create-batches.ts

@@ -0,0 +1,8 @@
+export function* createBatches<T>(
+	array: T[],
+	size: number
+): IterableIterator<T[]> {
+	for (let i = 0; i < array.length; i += size) {
+		yield array.slice(i, i + size);
+	}
+}