feat: add updated storage limits for casts, reactions, follows

.changeset/silly-dryers-draw.md

@@ -0,0 +1,6 @@
+---
+"@farcaster/core": patch
+"@farcaster/hubble": patch
+---
+
+feat: add updated storage limits for casts, reactions, follows, to be applied at a future date

apps/hubble/src/storage/stores/castStore.test.ts

@@ -6,12 +6,14 @@ import {
   CastRemoveMessage,
   CastType,
   Factories,
+  getDefaultStoreLimit,
   getFarcasterTime,
   HubError,
   MergeMessageHubEvent,
   Message,
   PruneMessageHubEvent,
   RevokeMessageHubEvent,
+  StoreType,
 } from "@farcaster/hub-nodejs";
 import { jestRocksDB } from "../db/jestUtils.js";
 import { getMessage, makeTsHash } from "../db/message.js";
@@ -753,6 +755,17 @@ describe("pruneMessages", () => {
   describe("with size limit", () => {
     const sizePrunedStore = new CastStore(db, eventHandler, { pruneSizeLimit: 3 });
 
+    test("size limit changes in the future", () => {
+      expect(getDefaultStoreLimit(StoreType.CASTS)).toEqual(5000);
+      const originalDate = Date.now;
+      try {
+        Date.now = () => new Date("2024-08-22").getTime();
+        expect(getDefaultStoreLimit(StoreType.CASTS)).toEqual(4000);
+      } finally {
+        Date.now = originalDate;
+      }
+    });
+
     test("no-ops when no messages have been merged", async () => {
       const result = await sizePrunedStore.pruneMessages(fid);
       expect(result.isOk()).toBeTruthy();

apps/hubble/src/storage/stores/linkStore.test.ts

@@ -2,6 +2,7 @@ import {
   bytesDecrement,
   bytesIncrement,
   Factories,
+  getDefaultStoreLimit,
   getFarcasterTime,
   HubError,
   LinkAddMessage,
@@ -12,6 +13,7 @@ import {
   MessageType,
   PruneMessageHubEvent,
   RevokeMessageHubEvent,
+  StoreType,
 } from "@farcaster/hub-nodejs";
 import { err, ok } from "neverthrow";
 import { jestRocksDB } from "../db/jestUtils.js";
@@ -963,6 +965,17 @@ describe("pruneMessages", () => {
   describe("with size limit", () => {
     const sizePrunedStore = new LinkStore(db, eventHandler, { pruneSizeLimit: 3 });
 
+    test("size limit changes in the future", () => {
+      expect(getDefaultStoreLimit(StoreType.LINKS)).toEqual(2500);
+      const originalDate = Date.now;
+      try {
+        Date.now = () => new Date("2024-08-22").getTime();
+        expect(getDefaultStoreLimit(StoreType.LINKS)).toEqual(2000);
+      } finally {
+        Date.now = originalDate;
+      }
+    });
+
     test("no-ops when no messages have been merged", async () => {
       const result = await sizePrunedStore.pruneMessages(fid);
       expect(result._unsafeUnwrap()).toEqual([]);

apps/hubble/src/storage/stores/reactionStore.test.ts

@@ -16,6 +16,8 @@ import {
   ReactionType,
   RevokeMessageHubEvent,
   MessageData,
+  getDefaultStoreLimit,
+  StoreType,
 } from "@farcaster/hub-nodejs";
 import { ResultAsync, err, ok } from "neverthrow";
 import { jestRocksDB } from "../db/jestUtils.js";
@@ -900,6 +902,17 @@ describe("pruneMessages", () => {
   describe("with size limit", () => {
     const sizePrunedStore = new ReactionStore(db, eventHandler, { pruneSizeLimit: 3 });
 
+    test("size limit changes in the future", () => {
+      expect(getDefaultStoreLimit(StoreType.REACTIONS)).toEqual(2500);
+      const originalDate = Date.now;
+      try {
+        Date.now = () => new Date("2024-08-22").getTime();
+        expect(getDefaultStoreLimit(StoreType.REACTIONS)).toEqual(2000);
+      } finally {
+        Date.now = originalDate;
+      }
+    });
+
     test("no-ops when no messages have been merged", async () => {
       const result = await sizePrunedStore.pruneMessages(fid);
       expect(result._unsafeUnwrap()).toEqual([]);

packages/core/src/limits.ts

@@ -1,5 +1,15 @@
 import { StoreType } from "./protobufs";
 
+// Protocol version 2024.7.24 includes updated storage limits for casts, reactions, and follows.
+// Date is used here to update the limit at a future time so hubs can stay in sync between versions
+const CASTS_LINKS_REACTIONS_LIMIT_DATE = new Date("2024-08-21").getTime();
+
+const VERSIONED_LIMTS: { [key in StoreType]?: number } = {
+  [StoreType.CASTS]: 4000,
+  [StoreType.REACTIONS]: 2000,
+  [StoreType.LINKS]: 2000,
+};
+
 const CASTS_SIZE_LIMIT_DEFAULT = 5_000;
 const LINKS_SIZE_LIMIT_DEFAULT = 2_500;
 const REACTIONS_SIZE_LIMIT_DEFAULT = 2_500;
@@ -36,11 +46,17 @@ export const getStoreLimits = (units: number) => [
 export const getDefaultStoreLimit = (storeType: StoreType) => {
   switch (storeType) {
     case StoreType.CASTS:
-      return CASTS_SIZE_LIMIT_DEFAULT;
+      return Date.now() > CASTS_LINKS_REACTIONS_LIMIT_DATE
+        ? VERSIONED_LIMTS[StoreType.CASTS] ?? Math.floor(0.8 * CASTS_SIZE_LIMIT_DEFAULT)
+        : CASTS_SIZE_LIMIT_DEFAULT;
     case StoreType.LINKS:
-      return LINKS_SIZE_LIMIT_DEFAULT;
+      return Date.now() > CASTS_LINKS_REACTIONS_LIMIT_DATE
+        ? VERSIONED_LIMTS[StoreType.LINKS] ?? Math.floor(0.8 * LINKS_SIZE_LIMIT_DEFAULT)
+        : LINKS_SIZE_LIMIT_DEFAULT;
     case StoreType.REACTIONS:
-      return REACTIONS_SIZE_LIMIT_DEFAULT;
+      return Date.now() > CASTS_LINKS_REACTIONS_LIMIT_DATE
+        ? VERSIONED_LIMTS[StoreType.REACTIONS] ?? Math.floor(0.8 * REACTIONS_SIZE_LIMIT_DEFAULT)
+        : REACTIONS_SIZE_LIMIT_DEFAULT;
     case StoreType.USER_DATA:
       return USER_DATA_SIZE_LIMIT_DEFAULT;
     case StoreType.USERNAME_PROOFS: