diff --git a/package-lock.json b/package-lock.json index 37e8840..451f96f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,7 @@ "pagination.djs": "^4.0.10", "reflect-metadata": "^0.1.13", "sqlite3": "^5.1.6", - "toad-scheduler": "^2.2.0", + "toad-scheduler": "3.0", "typeorm": "^0.3.16" }, "devDependencies": { @@ -6017,9 +6017,9 @@ } }, "node_modules/toad-scheduler": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/toad-scheduler/-/toad-scheduler-2.3.0.tgz", - "integrity": "sha512-Tl4+tr42ZpWBwKDIG5rdPjdx08TWt7B2UBsmzPuwcQ1uqJg/pY2spymYJJz1NmH/EgddxTppZzJtI+zZrzqDTA==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toad-scheduler/-/toad-scheduler-3.0.0.tgz", + "integrity": "sha512-BYmrORvaGmjziir/ra8qD3qAKczJveqK5ZOO/wn7oS5qLI4iibOza3DBGgu+EwNp9zdlBTKSeZPmxZxzGy1dxw==" }, "node_modules/tr46": { "version": "0.0.3", diff --git a/package.json b/package.json index 8024e0c..a6fb9b4 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "pagination.djs": "^4.0.10", "reflect-metadata": "^0.1.13", "sqlite3": "^5.1.6", - "toad-scheduler": "^2.2.0", + "toad-scheduler": "3.0", "typeorm": "^0.3.16" }, "devDependencies": { diff --git a/src/db/data-source.ts b/src/db/data-source.ts index e73d658..447e622 100644 --- a/src/db/data-source.ts +++ b/src/db/data-source.ts @@ -22,7 +22,7 @@ export const AppDataSource = new DataSource({ type: "sqlite", database: "data/database.sqlite", synchronize: false, - logging: "all", + logging: ["error"], entities: [ Video, DiscordUser, diff --git a/src/utils/schedule.ts b/src/utils/schedule.ts index 6d853ce..a9b404a 100644 --- a/src/utils/schedule.ts +++ b/src/utils/schedule.ts @@ -67,6 +67,8 @@ export async function scrape() { }, }); + const streamerIds: Set = new Set(streamers.map((s: Streamer) => s.id)); + const task = new AsyncTask("scrape Holodex", async () => { // fetch all upcoming streams from holodex // relevant docs: https://holodex.stoplight.io/docs/holodex/f1e355dc4cb79-quickly-access-live-upcoming-for-a-set-of-channels @@ -80,10 +82,9 @@ export async function scrape() { } ); const videos: HolodexVideo[] = await response.json(); - // add the data to the db for (const video of videos) { - if (video.channel.org !== "Hololive") { + if (!streamerIds.has(video.channel.id)) { continue; // TODO find a better workaround. Issue: when the original video creator is not from Hololive, the streamer isn't in the db and then it crashes } // videoMembers is the Video.members field in the db @@ -136,7 +137,6 @@ export async function scrape() { } db_vid.participantStreamers = participants; - console.log("saving"); await videoRepo.save(db_vid); // mention users for the 1st ping