different rate limiter

Taken from goenning#45 (comment)

src/index.ts

@@ -169,7 +169,7 @@ export const index = async (input: string = process.argv[2], options: IndexOptio
 
   for (const url of indexablePages) {
     // wait 1 second between requests to avoid rate limiting
-    await new Promise((resolve) => setTimeout(resolve, 1000));
+    // await new Promise((resolve) => setTimeout(resolve, 1000));
 
     console.log(`📄 Processing url: ${url}`);
     const status = await getPublishMetadata(accessToken, url, {

src/shared/utils.ts

@@ -42,6 +42,15 @@ export async function fetchRetry(url: string, options: RequestInit, retries: num
       const body = await response.text();
       throw new Error(`Server error code ${response.status}\n${body}`);
     }
+
+    if (response.status === 429) {
+      console.log("Rate limited. Retrying in 5 seconds...");
+      // Retry after 5 seconds if rate limited
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+
+      // Retry the request ignoring the retries limit
+      return fetchRetry(url, options, retries);
+    }
     return response;
   } catch (err) {
     if (retries <= 0) {