update data, update dependencies, fetch tweaks (#1457)

react-native-community · Dec 30, 2024 · c681ac5 · c681ac5
1 parent 4e112cd
commit c681ac5
Show file tree

Hide file tree

Showing 9 changed files with 22,014 additions and 16,991 deletions.
diff --git a/assets/data.json b/assets/data.json
diff --git a/index.js b/index.js
diff --git a/package.json b/package.json
@@ -6,14 +6,14 @@
     "preview": "next build && next start",
     "lint": "eslint .",
     "data:update": "node --no-warnings scripts/build-and-score-data.js",
-    "data:test": "node --no-warnings scripts/validate-libraries.js",
+    "data:test": "node scripts/validate-libraries.js",
     "data:validate": "ajv validate -s react-native-libraries.schema.json -d react-native-libraries.json --verbose --allowUnionTypes",
-    "libraries:cleanup": "node --no-warnings scripts/cleanup-libraries-json.js && yarn libraries:format",
-    "libraries:recalculate": "node --no-warnings scripts/recalculate-popularity.js",
+    "libraries:cleanup": "node scripts/cleanup-libraries-json.js && yarn libraries:format",
+    "libraries:recalculate": "node scripts/recalculate-popularity.js",
     "libraries:format": "prettier --write react-native-libraries.json",
-    "libraries:check": "node --no-warnings scripts/check-resources.js",
-    "ci:cleanup-blobs": "node --no-warnings scripts/cleanup-data-blobs.js",
-    "ci:validate": "node --no-warnings scripts/validate-new-entries.js",
+    "libraries:check": "node scripts/check-resources.js",
+    "ci:cleanup-blobs": "node scripts/cleanup-data-blobs.js",
+    "ci:validate": "node scripts/validate-new-entries.js",
     "precommit": "simple-git-hooks && lint-staged"
   },
   "resolutions": {
@@ -29,10 +29,10 @@
     "@react-native-picker/picker": "^2.10.2",
     "@sentry/react": "^8.46.0",
     "@vercel/blob": "^0.27.0",
-    "expo": "^52.0.19",
-    "expo-font": "~13.0.1",
+    "expo": "^52.0.23",
+    "expo-font": "~13.0.2",
     "lodash": "^4.17.21",
-    "next": "^15.1.0",
+    "next": "^15.1.3",
     "node-emoji": "^2.2.0",
     "react": "^18.3.1",
     "react-content-loader": "^7.0.2",
@@ -47,18 +47,17 @@
   },
   "devDependencies": {
     "@expo/next-adapter": "^6.0.0",
-    "@next/bundle-analyzer": "^15.1.0",
+    "@next/bundle-analyzer": "^15.1.3",
     "@types/node": "^22.10.2",
     "@types/react": "^18.3.12",
     "ajv-cli": "^5.0.0",
     "browserslist": "^4.24.3",
     "cheerio": "^1.0.0",
-    "cross-fetch": "^4.0.0",
+    "cross-fetch": "^4.1.0",
     "dotenv": "^16.4.7",
     "eslint": "^8.57.0",
-    "eslint-config-next": "^15.1.0",
+    "eslint-config-next": "^15.1.3",
     "eslint-config-universe": "14.0.0",
-    "isomorphic-fetch": "^3.0.0",
     "lint-staged": "^15.2.11",
     "next-compose-plugins": "^2.2.1",
     "next-fonts": "^1.5.1",

diff --git a/pages/index.tsx b/pages/index.tsx
@@ -1,4 +1,4 @@
-import fetch from 'isomorphic-fetch';
+import fetch from 'cross-fetch';
 import { NextPageContext } from 'next';
 import dynamic from 'next/dynamic';
 import { useRouter } from 'next/router';

diff --git a/pages/popular.tsx b/pages/popular.tsx
@@ -1,4 +1,4 @@
-import fetch from 'isomorphic-fetch';
+import fetch from 'cross-fetch';
 import { NextPageContext } from 'next';
 import { StyleSheet } from 'react-native';
 

diff --git a/scripts/build-and-score-data.js b/scripts/build-and-score-data.js
@@ -82,6 +82,9 @@ const buildAndScoreData = async () => {
   // https://github.com/npm/registry/blob/main/docs/download-counts.md#bulk-queries
   let bulkList = [];
 
+  // https://github.com/npm/registry/blob/main/docs/download-counts.md#limits
+  const CHUNK_SIZE = 80;
+
   // Fetch scoped packages data
   data = await Promise.all(
     data.map(project => {
@@ -97,9 +100,6 @@ const buildAndScoreData = async () => {
     })
   );
 
-  // https://github.com/npm/registry/blob/main/docs/download-counts.md#limits
-  const CHUNK_SIZE = 50;
-
   // Assemble and fetch regular packages data in bulk queries
   bulkList = [...Array(Math.ceil(bulkList.length / CHUNK_SIZE))].map(_ =>
     bulkList.splice(0, CHUNK_SIZE)
@@ -108,7 +108,7 @@ const buildAndScoreData = async () => {
   const downloadsList = (
     await Promise.all(
       bulkList.map(async (chunk, index) => {
-        await sleep(2000 * index);
+        await sleep(Math.max(2500 * index, 15000));
         return await fetchNpmDataBulk(chunk);
       })
     )
@@ -117,7 +117,7 @@ const buildAndScoreData = async () => {
   const downloadsListWeek = (
     await Promise.all(
       bulkList.map(async (chunk, index) => {
-        await sleep(2000 * index);
+        await sleep(Math.max(2500 * index, 15000));
         return await fetchNpmDataBulk(chunk, 'week');
       })
     )
@@ -127,12 +127,8 @@ const buildAndScoreData = async () => {
   data = data.map(project => ({
     ...project,
     npm: {
-      ...(downloadsList.find(entry => entry.name === project.npmPkg)?.npm ??
-        latestData.libraries.find(entry => entry.name === project.npmPkg)?.npm ??
-        {}),
-      ...(downloadsListWeek.find(entry => entry.name === project.npmPkg)?.npm ??
-        latestData.libraries.find(entry => entry.name === project.npmPkg)?.npm ??
-        {}),
+      ...(downloadsList.find(entry => entry.name === project.npmPkg)?.npm ?? {}),
+      ...(downloadsListWeek.find(entry => entry.name === project.npmPkg)?.npm ?? {}),
     },
   }));
 
@@ -216,10 +212,21 @@ const buildAndScoreData = async () => {
     const existingData = libraries.map(lib => lib.npmPkg);
     const newData = data.map(lib => lib.npmPkg);
     const missingData = existingData.filter(npmPkg => !newData.includes(npmPkg));
+    const currentData = [...libraries.filter(lib => missingData.includes(lib.npmPkg)), ...data];
+
+    const dataWithFallback = currentData.map(entry =>
+      Object.keys(entry.npm).length > 0
+        ? entry
+        : {
+            ...entry,
+            npm:
+              latestData.libraries.find(prevEntry => entry.npmPkg === prevEntry.npmPkg)?.npm ?? {},
+          }
+    );
 
     fileContent = JSON.stringify(
       {
-        libraries: [...libraries.filter(lib => missingData.includes(lib.npmPkg)), ...data],
+        libraries: dataWithFallback,
         topics: topicCounts,
         topicsList: Object.keys(topicCounts).sort(),
       },
@@ -276,12 +283,6 @@ function getDataForFetch(wantedPackage) {
 async function loadRepositoryDataAsync() {
   const data = getDataForFetch(wantedPackageName);
 
-  let githubResultsFileExists = false;
-  try {
-    fs.statSync(GITHUB_RESULTS_PATH);
-    githubResultsFileExists = true;
-  } catch {}
-
   const { apiLimit, apiLimitRemaining, apiLimitCost } = await fetchGithubRateLimit();
 
   // 5000 requests per hour is the authenticated API request rate limit
@@ -295,22 +296,21 @@ async function loadRepositoryDataAsync() {
   }
 
   console.info(
-    `${apiLimitRemaining} of ${apiLimit} GitHub API requests remaining for the hour at a cost of ${apiLimitCost} per request`
+    `${apiLimitRemaining} of ${apiLimit} GitHub API requests remaining for the hour at a cost of ${apiLimitCost} per request.`
   );
 
   await loadGitHubLicenses();
 
   let result;
-  if (LOAD_GITHUB_RESULTS_FROM_DISK && githubResultsFileExists) {
-    result = fs.readFileSync(GITHUB_RESULTS_PATH);
-    console.log('Loaded Github results from disk, skipped API calls');
-  } else {
-    result = await fetchGithubDataThrottled({ data, chunkSize: 25, staggerMs: 5000 });
-
-    if (LOAD_GITHUB_RESULTS_FROM_DISK) {
-      fs.writeFileSync(GITHUB_RESULTS_PATH, JSON.stringify(result, null, 2));
-      console.log('Saved Github results from disk');
+  if (LOAD_GITHUB_RESULTS_FROM_DISK) {
+    try {
+      result = fs.readFileSync(GITHUB_RESULTS_PATH);
+      console.log('Loaded GitHub results from disk, skipping API calls.');
+    } catch (error) {
+      console.warn('Failed to load data from disk!', error);
     }
+  } else {
+    result = await fetchGithubDataThrottled({ data, chunkSize: 25, staggerMs: 2500 });
   }
 
   return result;

diff --git a/scripts/cleanup-libraries-json.js b/scripts/cleanup-libraries-json.js
@@ -24,7 +24,6 @@ const processedLibraries = libraries
   // Remove all properties with `false` value, except those listed in emptyPropertiesToKeep
   .map(lib =>
     pickBy(lib, (value, key) => {
-      console.log(key);
       if (emptyPropertiesToKeep.includes(key)) {
         return true;
       } else {

diff --git a/scripts/fetch-npm-data.js b/scripts/fetch-npm-data.js
@@ -2,7 +2,7 @@ import fetch from 'cross-fetch';
 
 import { sleep, REQUEST_SLEEP } from './helpers.js';
 
-const ATTEMPTS_LIMIT = 5;
+const ATTEMPTS_LIMIT = 3;
 
 const urlForPackage = (npmPkg, period = 'month') => {
   return `https://api.npmjs.org/downloads/point/last-${period}/${npmPkg}`;
@@ -42,7 +42,8 @@ export const fetchNpmDataBulk = async (namesArray, period = 'month', attemptsCou
     });
   } catch (error) {
     if (attemptsCount >= ATTEMPTS_LIMIT) {
-      console.error('[NPM] Looks like we have reach the NPM API rate limit!', error);
+      console.error('[NPM] Looks like we have reach the NPM API rate limit!');
+      console.error(error);
       return namesArray.map(name => ({ name, npm: null }));
     }
     await sleep(REQUEST_SLEEP, REQUEST_SLEEP * 2);
@@ -81,8 +82,9 @@ export const fetchNpmData = async (pkgData, attemptsCount = 0) => {
       },
     };
   } catch (error) {
-    if (attemptsCount > ATTEMPTS_LIMIT) {
-      console.error('[NPM] Looks like we have reach the NPM API rate limit!', error);
+    if (attemptsCount >= ATTEMPTS_LIMIT) {
+      console.error('[NPM] Looks like we have reach the NPM API rate limit!');
+      console.error(error);
       return { ...pkgData, npm: null };
     }
     await sleep(REQUEST_SLEEP, REQUEST_SLEEP * 2);