feat(redis): add redis implementation of KvBufferStore

docker-compose.yaml

@@ -19,6 +19,9 @@ services:
       - TVAL_GRAPHQL_HOST=${GRAPHQL_HOST:-core}
       - TVAL_GRAPHQL_PORT=${GRAPHQL_PORT:-4000}
       - TVAL_ARNS_ROOT_HOST=${ARNS_ROOT_HOST:-}
+    depends_on:
+      - core
+      - observer
 
   core:
     image: ghcr.io/ar-io/ar-io-core:latest
@@ -53,7 +56,19 @@ services:
       - SANDBOX_PROTOCOL=${SANDBOX_PROTOCOL:-}
       - START_WRITERS=${START_WRITERS:-}
       - CONTRACT_ID=${CONTRACT_ID:-}
-      - CHAIN_CACHE_TYPE=${CHAIN_CACHE_TYPE:-}
+      - CHAIN_CACHE_TYPE=${CHAIN_CACHE_TYPE:-redis}
+      - REDIS_CACHE_URL=${REDIS_CACHE_URL:-redis://redis:6379}
+      - REDIS_CACHE_TTL_SECONDS=${REDIS_CACHE_TTL_SECONDS:-}
+    depends_on:
+      - redis
+
+  redis:
+    image: redis:latest
+    command: redis-server --appendonly yes --maxmemory ${REDIS_MAX_MEMORY:-2gb}
+    ports:
+      - 6379:6379
+    volumes:
+      - ${REDIS_DATA_PATH:-./data/redis}:/data
 
   observer:
     image: ghcr.io/ar-io/ar-io-observer:${OBSERVER_IMAGE_TAG:-14802babee090d674249960df890c54c9406076b}

package.json

@@ -31,6 +31,7 @@
     "node-cache": "^5.1.2",
     "prom-client": "^14.0.1",
     "ramda": "^0.28.0",
+    "redis": "^4.6.10",
     "retry-axios": "^3.0.0",
     "rfc4648": "^1.5.2",
     "sql-bricks": "^3.0.0",

src/config.ts

@@ -80,3 +80,11 @@ export const CONTRACT_ID = env.varOrDefault(
   'bLAgYxAdX2Ry-nt6aH2ixgvJXbpsEYm28NgJgyqfs-U',
 );
 export const CHAIN_CACHE_TYPE = env.varOrDefault('CHAIN_CACHE_TYPE', 'fs');
+export const REDIS_CACHE_URL = env.varOrDefault(
+  'REDIS_CACHE_URL',
+  'redis://localhost:6379',
+);
+export const REDIS_CACHE_TTL_SECONDS = +env.varOrDefault(
+  'REDIS_CACHE_TTL_SECONDS',
+  `${60 * 60 * 8}`, // 8 hours by default
+);

src/lib/kvstore.ts

@@ -15,29 +15,44 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+import winston from 'winston';
+
+import * as config from '../config.js';
 import { FsKVStore } from '../store/fs-kv-store.js';
 import { LmdbKVStore } from '../store/lmdb-kv-store.js';
+import { RedisKvStore } from '../store/redis-kv-store.js';
 import { KVBufferStore } from '../types.js';
 
 export const getKvBufferStore = ({
   pathKey,
   type,
+  log,
 }: {
   pathKey: string;
   type: string;
+  log: winston.Logger;
 }): KVBufferStore => {
+  log.info(`Using ${type} for KVBufferStore for ${pathKey}`);
   switch (type) {
     case 'lmdb': {
       return new LmdbKVStore({
         dbPath: `data/lmdb/${pathKey}`,
       });
     }
+    case 'redis': {
+      return new RedisKvStore({
+        redisUrl: config.REDIS_CACHE_URL,
+        ttlSeconds: config.REDIS_CACHE_TTL_SECONDS,
+        log,
+      });
+    }
     case 'fs': {
       return new FsKVStore({
         baseDir: `data/headers/${pathKey}`,
         tmpDir: `data/tmp/${pathKey}`,
       });
     }
+
     // TODO: implement redis
     default: {
       throw new Error(`Invalid chain cache type: ${type}`);

src/metrics.ts

@@ -132,3 +132,15 @@ export const lastHeightImported = new promClient.Gauge({
   name: 'last_height_imported',
   help: 'Height of the last block imported',
 });
+
+// Redis Cache Metrics
+
+export const redisConnectionErrorsCounter = new promClient.Counter({
+  name: 'redis_connection_errors_total',
+  help: 'Number of errors connecting to redis',
+});
+
+export const redisErrorCounter = new promClient.Counter({
+  name: 'redis_errors_total',
+  help: 'Number of errors redis cache has received',
+});

src/store/redis-kv-store.ts

@@ -0,0 +1,79 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+import { RedisClientType, commandOptions, createClient } from 'redis';
+import winston from 'winston';
+
+import * as metrics from '../metrics.js';
+import { KVBufferStore } from '../types.js';
+
+export class RedisKvStore implements KVBufferStore {
+  private client: RedisClientType;
+  private log: winston.Logger;
+  private ttlSeconds: number;
+
+  constructor({
+    log,
+    redisUrl,
+    ttlSeconds,
+  }: {
+    log: winston.Logger;
+    redisUrl: string;
+    ttlSeconds: number;
+  }) {
+    this.log = log.child({ class: this.constructor.name });
+    this.ttlSeconds = ttlSeconds;
+    this.client = createClient({
+      url: redisUrl,
+    });
+    this.client.on('error', (err) => {
+      this.log.error(`Redis error: ${err}`);
+      metrics.redisErrorCounter.inc();
+    });
+    this.client.connect().catch((err) => {
+      this.log.error(`Redis connection error: ${err}`);
+      metrics.redisConnectionErrorsCounter.inc();
+    });
+  }
+
+  // TODO: close connection to redis safely
+
+  async get(key: string): Promise<Buffer | undefined> {
+    const value = await this.client.get(
+      commandOptions({ returnBuffers: true }),
+      key,
+    );
+    return value ?? undefined;
+  }
+
+  async has(key: string): Promise<boolean> {
+    return (await this.client.exists(key)) === 1;
+  }
+
+  async del(key: string): Promise<void> {
+    if (await this.has(key)) {
+      await this.client.del(key);
+    }
+  }
+
+  async set(key: string, buffer: Buffer): Promise<void> {
+    // set the key with a TTL for every key
+    await this.client.set(key, buffer, {
+      EX: this.ttlSeconds,
+    });
+  }
+}

src/system.ts

@@ -75,13 +75,15 @@ export const arweaveClient = new ArweaveCompositeClient({
   blockStore: new KvBlockStore({
     log,
     kvBufferStore: getKvBufferStore({
+      log,
       pathKey: 'partial-blocks',
       type: config.CHAIN_CACHE_TYPE,
     }),
   }),
   txStore: new KvTransactionStore({
     log,
     kvBufferStore: getKvBufferStore({
+      log,
       pathKey: 'partial-txs',
       type: config.CHAIN_CACHE_TYPE,
     }),

yarn.lock

@@ -1044,6 +1044,40 @@
   dependencies:
     "@randlabs/communication-bridge" "^1.0.0"
 
+"@redis/[email protected]":
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71"
+  integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==
+
+"@redis/[email protected]":
+  version "1.5.11"
+  resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.11.tgz#5ee8620fea56c67cb427228c35d8403518efe622"
+  integrity sha512-cV7yHcOAtNQ5x/yQl7Yw1xf53kO0FNDTdDU6bFIMbW6ljB7U7ns0YRM+QIkpoqTAt6zK5k9Fq0QWlUbLcq9AvA==
+  dependencies:
+    cluster-key-slot "1.1.2"
+    generic-pool "3.9.0"
+    yallist "4.0.0"
+
+"@redis/[email protected]":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519"
+  integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==
+
+"@redis/[email protected]":
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.6.tgz#b7a7725bbb907765d84c99d55eac3fcf772e180e"
+  integrity sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==
+
+"@redis/[email protected]":
+  version "1.1.5"
+  resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.5.tgz#682b68114049ff28fdf2d82c580044dfb74199fe"
+  integrity sha512-hPP8w7GfGsbtYEJdn4n7nXa6xt6hVZnnDktKW4ArMaFQ/m/aR7eFvsLQmG/mn1Upq99btPJk+F27IQ2dYpCoUg==
+
+"@redis/[email protected]":
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad"
+  integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==
+
 "@rushstack/ts-command-line@^4.12.2":
   version "4.12.2"
   resolved "https://registry.yarnpkg.com/@rushstack/ts-command-line/-/ts-command-line-4.12.2.tgz#59b7450c5d75190778cce8b159c7d7043c32cc4e"
@@ -2474,6 +2508,11 @@ [email protected]:
   resolved "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz"
   integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==
 
+[email protected]:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac"
+  integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==
+
 code-point-at@^1.0.0:
   version "1.1.0"
   resolved "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz"
@@ -3568,6 +3607,11 @@ gc-stats@^1.4.0:
     nan "^2.13.2"
     node-pre-gyp "^0.13.0"
 
+[email protected]:
+  version "3.9.0"
+  resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4"
+  integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==
+
 gensync@^1.0.0-beta.2:
   version "1.0.0-beta.2"
   resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
@@ -5511,6 +5555,18 @@ readdirp@~3.6.0:
   dependencies:
     picomatch "^2.2.1"
 
+redis@^4.6.10:
+  version "4.6.10"
+  resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.10.tgz#07f6ea2b2c5455b098e76d1e8c9b3376114e9458"
+  integrity sha512-mmbyhuKgDiJ5TWUhiKhBssz+mjsuSI/lSZNPI9QvZOYzWvYGejtb+W3RlDDf8LD6Bdl5/mZeG8O1feUGhXTxEg==
+  dependencies:
+    "@redis/bloom" "1.2.0"
+    "@redis/client" "1.5.11"
+    "@redis/graph" "1.1.0"
+    "@redis/json" "1.0.6"
+    "@redis/search" "1.1.5"
+    "@redis/time-series" "1.0.5"
+
 regenerator-runtime@^0.13.4:
   version "0.13.9"
   resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz"
@@ -6754,16 +6810,16 @@ y18n@^5.0.5:
   resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz"
   integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==
 
+[email protected], yallist@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz"
+  integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
+
 yallist@^3.0.0, yallist@^3.0.2, yallist@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"
   integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==
 
-yallist@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz"
-  integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
-
 yaml@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.1.1.tgz#1e06fb4ca46e60d9da07e4f786ea370ed3c3cfec"