diff --git a/package.json b/package.json index eb36cf3..b825545 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "floatplane-plex-downloader", - "version": "5.10.1", + "version": "5.10.2", "private": true, "type": "module", "scripts": { diff --git a/src/float.ts b/src/float.ts index cba79c8..85be4f9 100644 --- a/src/float.ts +++ b/src/float.ts @@ -102,7 +102,7 @@ process.on("SIGTERM", process.exit); if (settings.floatplane.waitForNewVideos === true) { const waitLoop = async () => { await downloadNewVideos(); - setTimeout(waitLoop, 10 * 1000); + setTimeout(waitLoop, 5 * 60 * 1000); console.log("[" + new Date().toLocaleTimeString() + "]" + " Checking for new videos in 5 minutes..."); }; waitLoop(); diff --git a/src/lib/Caches.ts b/src/lib/Caches.ts new file mode 100644 index 0000000..b2c1974 --- /dev/null +++ b/src/lib/Caches.ts @@ -0,0 +1,61 @@ +import type { JSONSafeValue } from "@inrixia/db"; +import { writeFile } from "fs/promises"; +import { readFileSync } from "fs"; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type FetchItem = (id: string, options?: any) => Promise; + +export class ItemCache { + private cache: Record; + private cachePath: string; + private fetchItem: FetchItem; + private expireAgeMs: number; + + constructor(cachePath: string, fetchItem: FetchItem, expireAgeMins: number = 24 * 60) { + this.cachePath = cachePath; + this.fetchItem = fetchItem; + this.expireAgeMs = expireAgeMins * 60 * 1000; + + try { + this.cache = JSON.parse(readFileSync(this.cachePath).toString()); + } catch (err) { + this.cache = {}; + } + } + + private async writeOut() { + try { + await writeFile(this.cachePath, JSON.stringify(this.cache)); + } catch (err) { + return; + } + } + + private async set(key: string, i: T) { + this.cache[key] = { t: Date.now(), i }; + await this.writeOut(); + return i; + } + + private deepCopy(i: T): T { + return JSON.parse(JSON.stringify(i)); + } + + public async get(id: string, options?: unknown, noCache: boolean = false): Promise { + const key = options !== undefined ? JSON.stringify([id, options]) : id; + if (noCache) { + delete this.cache[key]; + return this.get(id, options); + } + const cacheItem = this.cache[key]; + if (cacheItem !== undefined) { + // Remove expired entries older than expireAge + if (Date.now() - cacheItem.t > this.expireAgeMs) { + delete this.cache[key]; + return this.get(id, options); + } + return this.deepCopy(cacheItem.i); + } + return this.set(key, await this.fetchItem(id, options)); + } +} diff --git a/src/lib/Subscription.ts b/src/lib/Subscription.ts index b1b146e..2856daf 100644 --- a/src/lib/Subscription.ts +++ b/src/lib/Subscription.ts @@ -4,12 +4,13 @@ import chalk from "chalk"; import { rm } from "fs/promises"; import type { ChannelOptions, SubscriptionSettings } from "./types.js"; -import type { ContentPost, VideoContent } from "floatplane/content"; +import type { ContentPost } from "floatplane/content"; import type { BlogPost } from "floatplane/creator"; import { Video } from "./Video.js"; import { settings } from "./helpers.js"; +import { ItemCache } from "./Caches.js"; const removeRepeatedSentences = (postTitle: string, attachmentTitle: string) => { const separators = /(?:\s+|^)((?:[^.,;:!?-]+[\s]*[.,;:!?-]+)+)(?:\s+|$)/g; @@ -24,16 +25,29 @@ const removeRepeatedSentences = (postTitle: string, attachmentTitle: string) => return `${postTitle.trim()} - ${uniqueAttachmentTitleSentences.join("").trim()}`.trim().replace(/[\s]*[.,;:!?-]+[\s]*$/, ""); }; -const t24Hrs = 24 * 60 * 60 * 1000; - +type BlogPosts = typeof fApi.creator.blogPostsIterable; export default class Subscription { public channels: SubscriptionSettings["channels"]; public readonly creatorId: string; + private static AttachmentsCache = new ItemCache("./db/attachmentCache.json", fApi.content.video, 24 * 60); + + private static PostCache = new ItemCache("./db/postCache.json", fApi.creator.blogPosts, 60); + private static async *PostIterable(creatorGUID: Parameters["0"], options: Parameters["1"]): ReturnType { + let fetchAfter = 0; + // First request should always not hit cache incase looking for new videos + let blogPosts = await Subscription.PostCache.get(creatorGUID, { ...options, fetchAfter }, true); + while (blogPosts.length > 0) { + yield* blogPosts; + fetchAfter += 20; + // After that use the cached data + blogPosts = await Subscription.PostCache.get(creatorGUID, { ...options, fetchAfter }); + } + } + constructor(subscription: SubscriptionSettings) { this.creatorId = subscription.creatorId; - this.channels = subscription.channels; } @@ -67,22 +81,6 @@ export default class Subscription { private static getIgnoreBeforeTimestamp = (channel: ChannelOptions) => Date.now() - (channel.daysToKeepVideos ?? 0) * 24 * 60 * 60 * 1000; - private static attachmentCache = new Map(); - private static fetchAttachment = async (attachmentId: string): Promise => { - if (Subscription.attachmentCache.has(attachmentId)) { - const { attachment, t } = Subscription.attachmentCache.get(attachmentId)!; - // Remove expired entries older than 24hrs - if (Date.now() - t > t24Hrs) { - Subscription.attachmentCache.delete(attachmentId); - return Subscription.fetchAttachment(attachmentId); - } - return attachment; - } - const attachment = await fApi.content.video(attachmentId); - Subscription.attachmentCache.set(attachmentId, { t: Date.now(), attachment }); - return attachment; - }; - private async *matchChannel(blogPost: BlogPost): AsyncGenerator