feat(signature-store): add redis-backed signature store

src/data/signature-fetcher.test.ts

@@ -23,6 +23,7 @@ import {
   ContiguousDataSource,
   ContiguousDataIndex,
   ChainSource,
+  SignatureStore,
 } from '../types.js';
 
 describe('SignatureFetcher', () => {
@@ -31,6 +32,7 @@ describe('SignatureFetcher', () => {
   let dataIndex: ContiguousDataIndex;
   let chainSource: ChainSource;
   let signatureFetcher: SignatureFetcher;
+  let signatureStore: SignatureStore;
 
   beforeEach(() => {
     log = winston.createLogger({ silent: true });
@@ -44,12 +46,16 @@ describe('SignatureFetcher', () => {
     chainSource = {
       getTxField: mock.fn(),
     } as unknown as ChainSource;
+    signatureStore = {
+      get: mock.fn(),
+    } as unknown as SignatureStore;
 
     signatureFetcher = new SignatureFetcher({
       log,
       dataSource,
       dataIndex,
       chainSource,
+      signatureStore,
     });
   });
 

src/data/signature-fetcher.ts

@@ -21,6 +21,7 @@ import {
   ContiguousDataIndex,
   ContiguousDataAttributes,
   ChainSource,
+  SignatureStore,
 } from '../types.js';
 import winston from 'winston';
 import { toB64Url } from '../lib/encoding.js';
@@ -30,28 +31,38 @@ export class SignatureFetcher implements SignatureSource {
   private dataSource: ContiguousDataSource;
   private dataIndex: ContiguousDataIndex;
   private chainSource: ChainSource;
+  private signatureStore: SignatureStore;
 
   constructor({
     log,
     dataSource,
     dataIndex,
     chainSource,
+    signatureStore,
   }: {
     log: winston.Logger;
     dataSource: ContiguousDataSource;
     dataIndex: ContiguousDataIndex;
     chainSource: ChainSource;
+    signatureStore: SignatureStore;
   }) {
     this.log = log.child({ class: 'SignatureFetcher' });
     this.dataSource = dataSource;
     this.dataIndex = dataIndex;
     this.chainSource = chainSource;
+    this.signatureStore = signatureStore;
   }
 
   async getDataItemSignature(id: string): Promise<string | undefined> {
     try {
-      this.log.debug('Fetching data item signature', { id });
+      this.log.debug('Fetching data item signature from store', { id });
+      const signatureFromStore = await this.signatureStore.get(id);
+
+      if (signatureFromStore !== undefined) {
+        return signatureFromStore;
+      }
 
+      this.log.debug('Fetching data item signature', { id });
       const dataItemAttributes = await this.dataIndex.getDataItemAttributes(id);
 
       if (dataItemAttributes === undefined) {
@@ -96,8 +107,14 @@ export class SignatureFetcher implements SignatureSource {
 
   async getTransactionSignature(id: string): Promise<string | undefined> {
     try {
-      this.log.debug('Fetching transaction signature', { id });
+      this.log.debug('Fetching transaction signature from store', { id });
+      const signatureFromStore = await this.signatureStore.get(id);
+
+      if (signatureFromStore !== undefined) {
+        return signatureFromStore;
+      }
 
+      this.log.debug('Fetching transaction signature', { id });
       const transactionAttributes =
         await this.dataIndex.getTransactionAttributes(id);
 

src/init/header-stores.ts

@@ -26,6 +26,7 @@ import { KvBlockStore } from '../store/kv-block-store.js';
 import { KvTransactionStore } from '../store/kv-transaction-store.js';
 import { FsBlockStore } from '../store/fs-block-store.js';
 import { FsTransactionStore } from '../store/fs-transaction-store.js';
+import { KvSignatureStore } from '../store/kv-signature-store.js';
 
 const createKvBufferStore = ({
   pathKey,
@@ -116,3 +117,14 @@ export const makeTxStore = ({
     });
   }
 };
+
+export const makeSignatureStore = ({ log }: { log: winston.Logger }) => {
+  return new KvSignatureStore({
+    log,
+    kvBufferStore: new RedisKvStore({
+      redisUrl: config.REDIS_CACHE_URL,
+      ttlSeconds: 60 * 60 * 4, // 4 hours
+      log,
+    }),
+  });
+};

src/routes/ar-io.ts

@@ -22,6 +22,7 @@ import * as config from '../config.js';
 import * as system from '../system.js';
 import * as events from '../events.js';
 import { release } from '../version.js';
+import { signatureStore } from '../system.js';
 
 export const arIoRouter = Router();
 
@@ -181,6 +182,11 @@ arIoRouter.post(
       }
 
       for (const dataItemHeader of dataItemHeaders) {
+        // cache signatures in signature store
+        if (config.WRITE_ANS104_DATA_ITEM_DB_SIGNATURES === false) {
+          signatureStore.set(dataItemHeader.id, dataItemHeader.signature);
+        }
+
         system.dataItemIndexer.queueDataItem(
           {
             ...dataItemHeader,

src/store/kv-signature-store.ts

@@ -0,0 +1,117 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+import winston from 'winston';
+
+import { KVBufferStore, SignatureStore } from '../types.js';
+import { fromB64Url, toB64Url } from '../lib/encoding.js';
+
+const getSignatureKey = (id: string) => `sig:${id}`;
+
+export class KvSignatureStore implements SignatureStore {
+  private log: winston.Logger;
+  private kvBufferStore: KVBufferStore;
+
+  constructor({
+    log,
+    kvBufferStore,
+  }: {
+    log: winston.Logger;
+    kvBufferStore: KVBufferStore;
+  }) {
+    this.log = log.child({ class: this.constructor.name });
+    this.kvBufferStore = kvBufferStore;
+  }
+
+  async has(key: string) {
+    try {
+      const exists = await this.kvBufferStore.has(key);
+
+      return exists;
+    } catch (error: any) {
+      this.log.error(
+        'Failed to verify if signature exists in key/value store',
+        {
+          key,
+          message: error.message,
+          stack: error.stack,
+        },
+      );
+    }
+
+    return false;
+  }
+
+  async get(id: string) {
+    try {
+      const key = getSignatureKey(id);
+
+      if (await this.has(key)) {
+        const signatureBuffer = await this.kvBufferStore.get(key);
+
+        if (signatureBuffer === undefined) {
+          throw new Error('Missing signature in key/value store');
+        }
+
+        return toB64Url(signatureBuffer);
+      }
+    } catch (error: any) {
+      this.log.error('Failed to get signature from key/value store', {
+        id,
+        message: error.message,
+        stack: error.stack,
+      });
+    }
+
+    return undefined;
+  }
+
+  async set(id: string, signature: string) {
+    try {
+      const key = getSignatureKey(id);
+
+      if (!(await this.has(key))) {
+        const signatureBuffer = fromB64Url(signature);
+
+        return this.kvBufferStore.set(key, signatureBuffer);
+      }
+    } catch (error: any) {
+      this.log.error('Failed to set signature in key/value store', {
+        id,
+        message: error.message,
+        stack: error.stack,
+      });
+    }
+  }
+
+  // Currenly unused
+  async del(id: string) {
+    try {
+      const key = getSignatureKey(id);
+
+      if (await this.has(key)) {
+        return this.kvBufferStore.del(key);
+      }
+    } catch (error: any) {
+      this.log.error('Failed to delete signature from key/value store', {
+        id,
+        message: error.message,
+        stack: error.stack,
+      });
+    }
+  }
+}

src/system.ts

@@ -35,7 +35,11 @@ import { StandaloneSqliteDatabase } from './database/standalone-sqlite.js';
 import * as events from './events.js';
 import { MatchTags } from './filters.js';
 import { UniformFailureSimulator } from './lib/chaos.js';
-import { makeBlockStore, makeTxStore } from './init/header-stores.js';
+import {
+  makeBlockStore,
+  makeTxStore,
+  makeSignatureStore,
+} from './init/header-stores.js';
 import { currentUnixTimestamp } from './lib/time.js';
 import log from './log.js';
 import * as metrics from './metrics.js';
@@ -533,11 +537,13 @@ export const mempoolWatcher = config.ENABLE_MEMPOOL_WATCHER
     })
   : undefined;
 
+export const signatureStore = makeSignatureStore({ log });
 export const signatureFetcher = new SignatureFetcher({
   log,
   dataSource: contiguousDataSource,
   dataIndex: contiguousDataIndex,
   chainSource: arweaveClient,
+  signatureStore,
 });
 
 const dataSqliteWalCleanupWorker = config.ENABLE_DATA_DB_WAL_CLEANUP

src/types.d.ts

@@ -123,6 +123,13 @@ export interface PartialJsonTransactionStore {
   del(txId: string): Promise<void>;
 }
 
+export interface SignatureStore {
+  has(key: string): Promise<boolean>;
+  get(id: string): Promise<string | undefined>;
+  set(id: string, signature: string): Promise<void>;
+  del(id: string): Promise<void>;
+}
+
 export interface ChunkDataStore {
   has(dataRoot: string, relativeOffset: number): Promise<boolean>;
   get(dataRoot: string, relativeOffset: number): Promise<ChunkData | undefined>;