Attempt to fix types-registry with retry / nAtATime (#1057)

.changeset/fair-pigs-jog.md

@@ -0,0 +1,5 @@
+---
+"@definitelytyped/utils": patch
+---
+
+Add retry function

packages/publisher/src/publish-registry.ts

@@ -29,6 +29,8 @@ import {
   NpmPublishClient,
   cacheDir,
   isObject,
+  nAtATime,
+  retry,
 } from "@definitelytyped/utils";
 import * as pacote from "pacote";
 import * as semver from "semver";
@@ -233,11 +235,20 @@ interface Registry {
 async function generateRegistry(typings: readonly TypingsData[]): Promise<Registry> {
   return {
     entries: Object.fromEntries(
-      await Promise.all(
-        typings.map(async (typing) => [
-          typing.typesDirectoryName,
-          filterTags((await pacote.packument(typing.name, { cache: cacheDir }))["dist-tags"]),
-        ]),
+      await nAtATime(
+        10,
+        typings,
+        async (typing) => {
+          const tags = await retry(
+            async () => (await pacote.packument(typing.name, { cache: cacheDir }))["dist-tags"],
+            /*count*/ 2,
+            /*delaySeconds*/ 5,
+          );
+          return [
+            typing.typesDirectoryName,
+            filterTags(tags),
+          ];
+        },
       ),
     ),
   };

packages/retag/src/index.ts

@@ -14,7 +14,7 @@ import {
   LoggerWithErrors,
   cacheDir,
   nAtATime,
-  sleep,
+  retry,
 } from "@definitelytyped/utils";
 import { AnyPackage, TypingsData, AllPackages, getDefinitelyTyped } from "@definitelytyped/definitions-parser";
 import * as pacote from "pacote";
@@ -83,19 +83,6 @@ async function tag(dry: boolean, definitelyTypedPath: string, name?: string) {
   // Don't tag notNeeded packages
 }
 
-async function retry<T>(fn: () => Promise<T>, count: number, delaySeconds: number): Promise<T> {
-  let lastError: any;
-  for (let i = 0; i < count; i++) {
-    try {
-      return await fn();
-    } catch (e) {
-      await sleep(delaySeconds);
-      lastError = e;
-    }
-  }
-  throw lastError;
-}
-
 export async function updateTypeScriptVersionTags(
   pkg: AnyPackage,
   version: string,

packages/utils/src/async.ts

@@ -1,5 +1,6 @@
 import { ProgressBar } from "./progress";
 import { initArray } from "./collections";
+import { sleep } from "./miscellany";
 
 /** Progress options needed for `nAtATime`. Other options will be inferred. */
 interface ProgressOptions<T, U> {
@@ -78,3 +79,16 @@ export function logUncaughtErrors<T>(promise: Promise<T> | (() => Promise<T>)):
     process.exit(1);
   });
 }
+
+export async function retry<T>(fn: () => Awaitable<T>, count: number, delaySeconds: number): Promise<T> {
+  let lastError: any;
+  for (let i = 0; i < count; i++) {
+    try {
+      return await fn();
+    } catch (e) {
+      await sleep(delaySeconds);
+      lastError = e;
+    }
+  }
+  throw lastError;
+}