fix: change primary db cache to mongo

rayriffy · Feb 14, 2025 · 6243b59 · 6243b59
1 parent 28a8594
commit 6243b59
Show file tree

Hide file tree

Showing 7 changed files with 89 additions and 89 deletions.
diff --git a/scraper/package.json b/scraper/package.json
@@ -17,9 +17,8 @@
     "dist"
   ],
   "dependencies": {
-    "kysely": "0.27.4",
+    "mongodb": "6.13.0",
     "p-queue": "8.0.1",
-    "pg": "8.13.1",
     "puppeteer": "24.1.1",
     "puppeteer-extra": "3.3.6",
     "puppeteer-extra-plugin-stealth": "2.11.2",

diff --git a/scraper/src/commands/fetch.ts b/scraper/src/commands/fetch.ts
@@ -7,10 +7,19 @@ import { chunk } from '../constants/chunk'
 import { cacheDirectory } from '../constants/cacheDirectory'
 import { hentaiDirectory } from '../constants/hentaiDirectory'
 import { parseUrl } from "../functions/parseUrl";
-import {getGalleriesViaBrowser} from "../functions/getGalleriesViaBrowser";
-import {getGalleriesViaFetch} from "../functions/getGalleriesViaFetch";
+import { getGalleriesViaBrowser } from "../functions/getGalleriesViaBrowser";
+import { getGalleriesViaFetch } from "../functions/getGalleriesViaFetch";
+import { collections } from "../constants/mongo";
+import { writeItem } from "../functions/writeItem";
 
 export const fetch = async (entryPoint: string, browserMode: boolean) => {
+  if (process.env.MONGODB_URL === undefined) {
+    console.error(
+      'no database url provided, please provide postgres connection url'
+    )
+    return process.exit(1)
+  }
+
   const { default: codes } = (await import(
     path.join(process.cwd(), entryPoint)
   )) as {
@@ -51,22 +60,37 @@ export const fetch = async (entryPoint: string, browserMode: boolean) => {
   if (!fs.existsSync(hentaiDirectory))
     await fs.promises.mkdir(hentaiDirectory, { recursive: true })
 
-  // let hasError
   const idsNeedsToBeFetched = codes
     .map(code => typeof code === 'object' ? code.code : code)
+    .map(code => Number(code))
     .filter(code => {
       return !fs.existsSync(path.join(hentaiDirectory, `${code}.json`))
     })
 
   console.log(`${idsNeedsToBeFetched.length} galleries needs to be fetched`)
 
-  const fetchResult = await (browserMode ? getGalleriesViaBrowser : getGalleriesViaFetch)(idsNeedsToBeFetched)
+  if (idsNeedsToBeFetched.length > 0) {
+    // look for hot-cache first
+    const mongoItems = await collections.galleries.find({
+      id: { $in: idsNeedsToBeFetched }
+    }, {
+      projection: { _id: 0 }
+    }).toArray()
+
+    await Promise.all(mongoItems.map(item =>
+      writeItem(item.id, item)
+    ))
 
-  if (fetchResult.failure > 0) {
-    console.error("there's some error during fetching! crashing...")
-    process.exit(1)
-  } else {
-    console.log('fetched all galleries')
+    console.log(`${mongoItems.length} found in cache! ${idsNeedsToBeFetched.length - mongoItems.length} needs to be fetched further`)
+
+    const fetchResult = await (browserMode ? getGalleriesViaBrowser : getGalleriesViaFetch)(idsNeedsToBeFetched)
+
+    if (fetchResult.failure > 0) {
+      console.error("there's some error during fetching! crashing...")
+      process.exit(1)
+    } else {
+      console.log('fetched all galleries')
+    }
   }
 
   /**
@@ -102,5 +126,6 @@ export const fetch = async (entryPoint: string, browserMode: boolean) => {
     path.join(cacheDirectory, 'searchKey.json'),
     JSON.stringify(orderedHentai)
   )
+
   console.log('completed!')
 }
diff --git a/scraper/src/commands/seed.ts b/scraper/src/commands/seed.ts
diff --git a/scraper/src/commands/sync.ts b/scraper/src/commands/sync.ts
@@ -4,61 +4,48 @@ import path from 'path'
 import { Hentai } from '@riffyh/commons'
 
 import { hentaiDirectory } from '../constants/hentaiDirectory'
-import { createDBConnection } from '../functions/createDBConnection'
-import { jsonb } from '../functions/jsonb'
+import { collections } from "../constants/mongo";
 
 export const sync = async () => {
-  if (process.env.DATABASE_URL === undefined) {
+  if (process.env.MONGODB_URL === undefined) {
     console.error(
       'no database url provided, please provide postgres connection url'
     )
-    process.exit(1)
-  } else {
-    const db = createDBConnection()
+    return process.exit(1)
+  }
+
+  try {
+    const remoteIds = (await collections.galleries.find({}, {
+      projection: { _id: 0, id: 1 },
+    }).toArray()).map(o => o.id)
 
-    try {
-      // list ids in local cache
-      const ids = await db
-        .selectFrom('Hentai')
-        .select('id')
-        .execute()
-        .then(o => o.map(p => p.id))
+    // push new data to local cache
+    const itemsToPush = fs
+      .readdirSync(hentaiDirectory)
+      .filter(o => !o.startsWith('.') && o.endsWith('.json'))
+      .filter(
+        o =>
+          !remoteIds.includes(
+            (
+              JSON.parse(
+                fs.readFileSync(path.join(hentaiDirectory, o), 'utf8')
+              ) as Hentai
+            ).id
+          )
+      )
+      .map(file => {
+        return JSON.parse(
+          fs.readFileSync(path.join(hentaiDirectory, file), 'utf8')
+        ) as Hentai
+      })
 
-      // push new data to local cache
-      const itemsToPush = fs
-        .readdirSync(hentaiDirectory)
-        .filter(o => !o.startsWith('.') && o.endsWith('.json'))
-        .filter(
-          o =>
-            !ids.includes(
-              (
-                JSON.parse(
-                  fs.readFileSync(path.join(hentaiDirectory, o), 'utf8')
-                ) as Hentai
-              ).id
-            )
-        )
-        .map(file => {
-          return JSON.parse(
-            fs.readFileSync(path.join(hentaiDirectory, file), 'utf8')
-          ) as Hentai
-        })
+    console.log(itemsToPush.length + ' items to push')
 
-      console.log(itemsToPush.length + ' items to push')
+    if (itemsToPush.length > 0)
+      await collections.galleries.insertMany(itemsToPush)
 
-      await db
-        .insertInto('Hentai')
-        .values(
-          itemsToPush.map(item => ({
-            id: item.id,
-            data: jsonb(JSON.stringify(item)),
-          }))
-        )
-        .execute()
-    } catch (e) {
-      console.log(e)
-    } finally {
-      await db.destroy()
-    }
+    process.exit(0)
+  } catch (e) {
+    console.log(e)
   }
 }
diff --git a/scraper/src/constants/mongo.ts b/scraper/src/constants/mongo.ts
@@ -0,0 +1,20 @@
+import { MongoClient } from 'mongodb'
+
+import type { Hentai } from "@riffyh/commons";
+
+const globalMongo = global as unknown as {
+  mongo?: MongoClient
+}
+
+export const mongo = globalMongo.mongo || new MongoClient(Bun.env.MONGODB_URL!)
+
+globalMongo.mongo = mongo
+
+// We can call `.db` and `.collection` as much as we like.
+// Until we actually make a query, it won’t connect to the database.
+
+const db = mongo.db('riffyh-data')
+
+export const collections = {
+  galleries: db.collection<Hentai>('nh-galleries'),
+}
diff --git a/scraper/src/functions/createDBConnection.ts b/scraper/src/functions/createDBConnection.ts
diff --git a/scraper/src/index.ts b/scraper/src/index.ts
@@ -5,7 +5,6 @@ import { hideBin } from 'yargs/helpers'
 
 import { fetch } from './commands/fetch'
 import { sync } from './commands/sync'
-import { seed } from './commands/seed'
 
 yargs(hideBin(process.argv))
   .command<{ file: string, browser: boolean }>(
@@ -21,6 +20,5 @@ yargs(hideBin(process.argv))
     argv => fetch(argv.file, argv.browser)
   )
   .command('sync', 'sync data to database', () => {}, sync)
-  .command('seed', 'generate cache table on database', () => {}, seed)
   .demandCommand(1)
   .parse()