From a8e30aed1d272a985165496223564ba2912d1bae Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Fri, 21 Jun 2024 20:24:25 -0500 Subject: [PATCH] feat: add updated storage limits for casts, reactions, follows, to be applied at a future date --- .changeset/silly-dryers-draw.md | 6 +++++ .../src/storage/stores/castStore.test.ts | 13 +++++++++++ .../src/storage/stores/linkStore.test.ts | 13 +++++++++++ .../src/storage/stores/reactionStore.test.ts | 13 +++++++++++ packages/core/src/limits.ts | 22 ++++++++++++++++--- 5 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 .changeset/silly-dryers-draw.md diff --git a/.changeset/silly-dryers-draw.md b/.changeset/silly-dryers-draw.md new file mode 100644 index 0000000000..b1597c64d4 --- /dev/null +++ b/.changeset/silly-dryers-draw.md @@ -0,0 +1,6 @@ +--- +"@farcaster/core": patch +"@farcaster/hubble": patch +--- + +feat: add updated storage limits for casts, reactions, follows, to be applied at a future date diff --git a/apps/hubble/src/storage/stores/castStore.test.ts b/apps/hubble/src/storage/stores/castStore.test.ts index 41f869cc31..1340f1bdf9 100644 --- a/apps/hubble/src/storage/stores/castStore.test.ts +++ b/apps/hubble/src/storage/stores/castStore.test.ts @@ -6,12 +6,14 @@ import { CastRemoveMessage, CastType, Factories, + getDefaultStoreLimit, getFarcasterTime, HubError, MergeMessageHubEvent, Message, PruneMessageHubEvent, RevokeMessageHubEvent, + StoreType, } from "@farcaster/hub-nodejs"; import { jestRocksDB } from "../db/jestUtils.js"; import { getMessage, makeTsHash } from "../db/message.js"; @@ -753,6 +755,17 @@ describe("pruneMessages", () => { describe("with size limit", () => { const sizePrunedStore = new CastStore(db, eventHandler, { pruneSizeLimit: 3 }); + test("size limit changes in the future", () => { + expect(getDefaultStoreLimit(StoreType.CASTS)).toEqual(5000); + const originalDate = Date.now; + try { + Date.now = () => new Date("2024-08-22").getTime(); + expect(getDefaultStoreLimit(StoreType.CASTS)).toEqual(4000); + } finally { + Date.now = originalDate; + } + }); + test("no-ops when no messages have been merged", async () => { const result = await sizePrunedStore.pruneMessages(fid); expect(result.isOk()).toBeTruthy(); diff --git a/apps/hubble/src/storage/stores/linkStore.test.ts b/apps/hubble/src/storage/stores/linkStore.test.ts index 49a6bf60c2..06cb7865c7 100644 --- a/apps/hubble/src/storage/stores/linkStore.test.ts +++ b/apps/hubble/src/storage/stores/linkStore.test.ts @@ -2,6 +2,7 @@ import { bytesDecrement, bytesIncrement, Factories, + getDefaultStoreLimit, getFarcasterTime, HubError, LinkAddMessage, @@ -12,6 +13,7 @@ import { MessageType, PruneMessageHubEvent, RevokeMessageHubEvent, + StoreType, } from "@farcaster/hub-nodejs"; import { err, ok } from "neverthrow"; import { jestRocksDB } from "../db/jestUtils.js"; @@ -963,6 +965,17 @@ describe("pruneMessages", () => { describe("with size limit", () => { const sizePrunedStore = new LinkStore(db, eventHandler, { pruneSizeLimit: 3 }); + test("size limit changes in the future", () => { + expect(getDefaultStoreLimit(StoreType.LINKS)).toEqual(2500); + const originalDate = Date.now; + try { + Date.now = () => new Date("2024-08-22").getTime(); + expect(getDefaultStoreLimit(StoreType.LINKS)).toEqual(2000); + } finally { + Date.now = originalDate; + } + }); + test("no-ops when no messages have been merged", async () => { const result = await sizePrunedStore.pruneMessages(fid); expect(result._unsafeUnwrap()).toEqual([]); diff --git a/apps/hubble/src/storage/stores/reactionStore.test.ts b/apps/hubble/src/storage/stores/reactionStore.test.ts index e20e1b4827..1ad34f31a4 100644 --- a/apps/hubble/src/storage/stores/reactionStore.test.ts +++ b/apps/hubble/src/storage/stores/reactionStore.test.ts @@ -16,6 +16,8 @@ import { ReactionType, RevokeMessageHubEvent, MessageData, + getDefaultStoreLimit, + StoreType, } from "@farcaster/hub-nodejs"; import { ResultAsync, err, ok } from "neverthrow"; import { jestRocksDB } from "../db/jestUtils.js"; @@ -900,6 +902,17 @@ describe("pruneMessages", () => { describe("with size limit", () => { const sizePrunedStore = new ReactionStore(db, eventHandler, { pruneSizeLimit: 3 }); + test("size limit changes in the future", () => { + expect(getDefaultStoreLimit(StoreType.REACTIONS)).toEqual(2500); + const originalDate = Date.now; + try { + Date.now = () => new Date("2024-08-22").getTime(); + expect(getDefaultStoreLimit(StoreType.REACTIONS)).toEqual(2000); + } finally { + Date.now = originalDate; + } + }); + test("no-ops when no messages have been merged", async () => { const result = await sizePrunedStore.pruneMessages(fid); expect(result._unsafeUnwrap()).toEqual([]); diff --git a/packages/core/src/limits.ts b/packages/core/src/limits.ts index d94c2a3893..d54647aba2 100644 --- a/packages/core/src/limits.ts +++ b/packages/core/src/limits.ts @@ -1,5 +1,15 @@ import { StoreType } from "./protobufs"; +// Protocol version 2024.7.24 includes updated storage limits for casts, reactions, and follows. +// Date is used here to update the limit at a future time so hubs can stay in sync between versions +const CASTS_LINKS_REACTIONS_LIMIT_DATE = new Date("2024-08-21").getTime(); + +const VERSIONED_LIMTS: { [key in StoreType]?: number } = { + [StoreType.CASTS]: 4000, + [StoreType.REACTIONS]: 2000, + [StoreType.LINKS]: 2000, +}; + const CASTS_SIZE_LIMIT_DEFAULT = 5_000; const LINKS_SIZE_LIMIT_DEFAULT = 2_500; const REACTIONS_SIZE_LIMIT_DEFAULT = 2_500; @@ -36,11 +46,17 @@ export const getStoreLimits = (units: number) => [ export const getDefaultStoreLimit = (storeType: StoreType) => { switch (storeType) { case StoreType.CASTS: - return CASTS_SIZE_LIMIT_DEFAULT; + return Date.now() > CASTS_LINKS_REACTIONS_LIMIT_DATE + ? VERSIONED_LIMTS[StoreType.CASTS] ?? Math.floor(0.8 * CASTS_SIZE_LIMIT_DEFAULT) + : CASTS_SIZE_LIMIT_DEFAULT; case StoreType.LINKS: - return LINKS_SIZE_LIMIT_DEFAULT; + return Date.now() > CASTS_LINKS_REACTIONS_LIMIT_DATE + ? VERSIONED_LIMTS[StoreType.LINKS] ?? Math.floor(0.8 * LINKS_SIZE_LIMIT_DEFAULT) + : LINKS_SIZE_LIMIT_DEFAULT; case StoreType.REACTIONS: - return REACTIONS_SIZE_LIMIT_DEFAULT; + return Date.now() > CASTS_LINKS_REACTIONS_LIMIT_DATE + ? VERSIONED_LIMTS[StoreType.REACTIONS] ?? Math.floor(0.8 * REACTIONS_SIZE_LIMIT_DEFAULT) + : REACTIONS_SIZE_LIMIT_DEFAULT; case StoreType.USER_DATA: return USER_DATA_SIZE_LIMIT_DEFAULT; case StoreType.USERNAME_PROOFS: