From 4d417a4f14ca3f39894115253bbe2dce174f5404 Mon Sep 17 00:00:00 2001 From: Karl Prieb Date: Thu, 5 Sep 2024 12:52:27 -0300 Subject: [PATCH] feat(signature-store): add redis-backed signature store --- src/data/signature-fetcher.test.ts | 6 ++ src/data/signature-fetcher.ts | 21 +++++- src/init/header-stores.ts | 12 +++ src/routes/ar-io.ts | 6 ++ src/store/kv-signature-store.ts | 117 +++++++++++++++++++++++++++++ src/system.ts | 8 +- src/types.d.ts | 7 ++ 7 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 src/store/kv-signature-store.ts diff --git a/src/data/signature-fetcher.test.ts b/src/data/signature-fetcher.test.ts index c18b371f..654f8f46 100644 --- a/src/data/signature-fetcher.test.ts +++ b/src/data/signature-fetcher.test.ts @@ -23,6 +23,7 @@ import { ContiguousDataSource, ContiguousDataIndex, ChainSource, + SignatureStore, } from '../types.js'; describe('SignatureFetcher', () => { @@ -31,6 +32,7 @@ describe('SignatureFetcher', () => { let dataIndex: ContiguousDataIndex; let chainSource: ChainSource; let signatureFetcher: SignatureFetcher; + let signatureStore: SignatureStore; beforeEach(() => { log = winston.createLogger({ silent: true }); @@ -44,12 +46,16 @@ describe('SignatureFetcher', () => { chainSource = { getTxField: mock.fn(), } as unknown as ChainSource; + signatureStore = { + get: mock.fn(), + } as unknown as SignatureStore; signatureFetcher = new SignatureFetcher({ log, dataSource, dataIndex, chainSource, + signatureStore, }); }); diff --git a/src/data/signature-fetcher.ts b/src/data/signature-fetcher.ts index 0e17258e..6ebe36d0 100644 --- a/src/data/signature-fetcher.ts +++ b/src/data/signature-fetcher.ts @@ -21,6 +21,7 @@ import { ContiguousDataIndex, ContiguousDataAttributes, ChainSource, + SignatureStore, } from '../types.js'; import winston from 'winston'; import { toB64Url } from '../lib/encoding.js'; @@ -30,28 +31,38 @@ export class SignatureFetcher implements SignatureSource { private dataSource: ContiguousDataSource; private dataIndex: ContiguousDataIndex; private chainSource: ChainSource; + private signatureStore: SignatureStore; constructor({ log, dataSource, dataIndex, chainSource, + signatureStore, }: { log: winston.Logger; dataSource: ContiguousDataSource; dataIndex: ContiguousDataIndex; chainSource: ChainSource; + signatureStore: SignatureStore; }) { this.log = log.child({ class: 'SignatureFetcher' }); this.dataSource = dataSource; this.dataIndex = dataIndex; this.chainSource = chainSource; + this.signatureStore = signatureStore; } async getDataItemSignature(id: string): Promise { try { - this.log.debug('Fetching data item signature', { id }); + this.log.debug('Fetching data item signature from store', { id }); + const signatureFromStore = await this.signatureStore.get(id); + + if (signatureFromStore !== undefined) { + return signatureFromStore; + } + this.log.debug('Fetching data item signature', { id }); const dataItemAttributes = await this.dataIndex.getDataItemAttributes(id); if (dataItemAttributes === undefined) { @@ -96,8 +107,14 @@ export class SignatureFetcher implements SignatureSource { async getTransactionSignature(id: string): Promise { try { - this.log.debug('Fetching transaction signature', { id }); + this.log.debug('Fetching transaction signature from store', { id }); + const signatureFromStore = await this.signatureStore.get(id); + + if (signatureFromStore !== undefined) { + return signatureFromStore; + } + this.log.debug('Fetching transaction signature', { id }); const transactionAttributes = await this.dataIndex.getTransactionAttributes(id); diff --git a/src/init/header-stores.ts b/src/init/header-stores.ts index 9c0b801b..acb19eea 100644 --- a/src/init/header-stores.ts +++ b/src/init/header-stores.ts @@ -26,6 +26,7 @@ import { KvBlockStore } from '../store/kv-block-store.js'; import { KvTransactionStore } from '../store/kv-transaction-store.js'; import { FsBlockStore } from '../store/fs-block-store.js'; import { FsTransactionStore } from '../store/fs-transaction-store.js'; +import { KvSignatureStore } from '../store/kv-signature-store.js'; const createKvBufferStore = ({ pathKey, @@ -116,3 +117,14 @@ export const makeTxStore = ({ }); } }; + +export const makeSignatureStore = ({ log }: { log: winston.Logger }) => { + return new KvSignatureStore({ + log, + kvBufferStore: new RedisKvStore({ + redisUrl: config.REDIS_CACHE_URL, + ttlSeconds: 60 * 60 * 4, // 4 hours + log, + }), + }); +}; diff --git a/src/routes/ar-io.ts b/src/routes/ar-io.ts index b91cc834..4c449d90 100644 --- a/src/routes/ar-io.ts +++ b/src/routes/ar-io.ts @@ -22,6 +22,7 @@ import * as config from '../config.js'; import * as system from '../system.js'; import * as events from '../events.js'; import { release } from '../version.js'; +import { signatureStore } from '../system.js'; export const arIoRouter = Router(); @@ -181,6 +182,11 @@ arIoRouter.post( } for (const dataItemHeader of dataItemHeaders) { + // cache signatures in signature store + if (config.WRITE_ANS104_DATA_ITEM_DB_SIGNATURES === false) { + signatureStore.set(dataItemHeader.id, dataItemHeader.signature); + } + system.dataItemIndexer.queueDataItem( { ...dataItemHeader, diff --git a/src/store/kv-signature-store.ts b/src/store/kv-signature-store.ts new file mode 100644 index 00000000..15e11a3b --- /dev/null +++ b/src/store/kv-signature-store.ts @@ -0,0 +1,117 @@ +/** + * AR.IO Gateway + * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import winston from 'winston'; + +import { KVBufferStore, SignatureStore } from '../types.js'; +import { fromB64Url, toB64Url } from '../lib/encoding.js'; + +const getSignatureKey = (id: string) => `sig:${id}`; + +export class KvSignatureStore implements SignatureStore { + private log: winston.Logger; + private kvBufferStore: KVBufferStore; + + constructor({ + log, + kvBufferStore, + }: { + log: winston.Logger; + kvBufferStore: KVBufferStore; + }) { + this.log = log.child({ class: this.constructor.name }); + this.kvBufferStore = kvBufferStore; + } + + async has(key: string) { + try { + const exists = await this.kvBufferStore.has(key); + + return exists; + } catch (error: any) { + this.log.error( + 'Failed to verify if signature exists in key/value store', + { + key, + message: error.message, + stack: error.stack, + }, + ); + } + + return false; + } + + async get(id: string) { + try { + const key = getSignatureKey(id); + + if (await this.has(key)) { + const signatureBuffer = await this.kvBufferStore.get(key); + + if (signatureBuffer === undefined) { + throw new Error('Missing signature in key/value store'); + } + + return toB64Url(signatureBuffer); + } + } catch (error: any) { + this.log.error('Failed to get signature from key/value store', { + id, + message: error.message, + stack: error.stack, + }); + } + + return undefined; + } + + async set(id: string, signature: string) { + try { + const key = getSignatureKey(id); + + if (!(await this.has(key))) { + const signatureBuffer = fromB64Url(signature); + + return this.kvBufferStore.set(key, signatureBuffer); + } + } catch (error: any) { + this.log.error('Failed to set signature in key/value store', { + id, + message: error.message, + stack: error.stack, + }); + } + } + + // Currenly unused + async del(id: string) { + try { + const key = getSignatureKey(id); + + if (await this.has(key)) { + return this.kvBufferStore.del(key); + } + } catch (error: any) { + this.log.error('Failed to delete signature from key/value store', { + id, + message: error.message, + stack: error.stack, + }); + } + } +} diff --git a/src/system.ts b/src/system.ts index 61a3063e..f59b57b5 100644 --- a/src/system.ts +++ b/src/system.ts @@ -35,7 +35,11 @@ import { StandaloneSqliteDatabase } from './database/standalone-sqlite.js'; import * as events from './events.js'; import { MatchTags } from './filters.js'; import { UniformFailureSimulator } from './lib/chaos.js'; -import { makeBlockStore, makeTxStore } from './init/header-stores.js'; +import { + makeBlockStore, + makeTxStore, + makeSignatureStore, +} from './init/header-stores.js'; import { currentUnixTimestamp } from './lib/time.js'; import log from './log.js'; import * as metrics from './metrics.js'; @@ -533,11 +537,13 @@ export const mempoolWatcher = config.ENABLE_MEMPOOL_WATCHER }) : undefined; +export const signatureStore = makeSignatureStore({ log }); export const signatureFetcher = new SignatureFetcher({ log, dataSource: contiguousDataSource, dataIndex: contiguousDataIndex, chainSource: arweaveClient, + signatureStore, }); const dataSqliteWalCleanupWorker = config.ENABLE_DATA_DB_WAL_CLEANUP diff --git a/src/types.d.ts b/src/types.d.ts index c5700180..c630275b 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -123,6 +123,13 @@ export interface PartialJsonTransactionStore { del(txId: string): Promise; } +export interface SignatureStore { + has(key: string): Promise; + get(id: string): Promise; + set(id: string, signature: string): Promise; + del(id: string): Promise; +} + export interface ChunkDataStore { has(dataRoot: string, relativeOffset: number): Promise; get(dataRoot: string, relativeOffset: number): Promise;