feat(redis): implement redis cache for arns name resolutions

.eslintrc

@@ -1,6 +1,5 @@
 {
   "root": true,
-  "ignorePatterns": ["resources/license.header.js"],
   "parser": "@typescript-eslint/parser",
   "parserOptions": {
     "project": "tsconfig.json"
@@ -29,15 +28,18 @@
         "allowAny": true
       }
     ],
-    "eqeqeq": 2,
+    "eqeqeq": ["error", "smart"],
     "jest-formatting/padding-around-describe-blocks": 2,
     "jest-formatting/padding-around-test-blocks": 2,
     "header/header": [2, "./resources/license.header.js"],
+    "mocha/max-top-level-suites": "off",
+    "mocha/no-exports": "off",
+    "mocha/no-mocha-arrows": "off",
     "no-console": 0,
     "no-return-await": 2,
     "no-unneeded-ternary": 2,
     "no-unused-vars": "off",
-    "prettier/prettier": 2,
+    "prettier/prettier": ["error", { "endOfLine": "auto" }],
     "unicorn/prefer-node-protocol": 2
   }
 }

.github/workflows/build.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        step: ['lint:check', 'build']
+        step: ['lint:check', 'build', "test"]
     steps:
       - uses: actions/checkout@v4
       - name: Setup yarn

docker-compose.yml

@@ -0,0 +1,35 @@
+services:
+  resolver:
+    image: ghcr.io/ar-io/arns-resolver:${RESOLVER_IMAGE_TAG:-7fe02ecda2027e504248d3f3716579f60b561de5}
+    build:
+      context: .
+    restart: on-failure
+    ports:
+      - ${HOST_PORT:-6000}:${CONTAINER_PORT:-6000}
+    environment:
+      - PORT=${CONTAINER_PORT:-6000}
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+      - IO_PROCESS_ID=${IO_PROCESS_ID:-}
+      - RUN_RESOLVER=${RUN_RESOLVER:-true}
+      - EVALUATION_INTERVAL_MS=${EVALUATION_INTERVAL_MS:-}
+      - ARNS_CACHE_TTL_MS=${RESOLVER_CACHE_TTL_MS:-}
+      - ARNS_CACHE_PATH=${ARNS_CACHE_PATH:-./data/arns}
+      - ARNS_CACHE_TYPE=${ARNS_CACHE_TYPE:-redis}
+      - REDIS_CACHE_URL=${REDIS_CACHE_URL:-redis://redis:6379}
+      - AO_CU_URL=${AO_CU_URL:-}
+      - AO_MU_URL=${AO_MU_URL:-}
+      - AO_GATEWAY_URL=${AO_GATEWAY_URL:-}
+      - AO_GRAPHQL_URL=${AO_GRAPHQL_URL:-}
+    volumes:
+      - ${ARNS_CACHE_PATH:-./data/arns}:/app/data/arns
+    depends_on:
+      - redis
+
+  redis:
+    image: redis:${REDIS_IMAGE_TAG:-7}
+    restart: on-failure
+    ports:
+      - 6379:6379
+    volumes:
+      - ${REDIS_DATA_PATH:-./data/redis}:/data
+

package.json

@@ -9,7 +9,7 @@
     "url": "https://github.com/ar-io/arns-resolver"
   },
   "dependencies": {
-    "@ar.io/sdk": "^2.0.2",
+    "@ar.io/sdk": "^2.1.0",
     "@permaweb/aoconnect": "^0.0.56",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
@@ -21,6 +21,7 @@
     "middleware-async": "^1.3.5",
     "p-limit": "^4.0.0",
     "prom-client": "^14.0.1",
+    "redis": "^4.7.0",
     "swagger-ui-express": "^5.0.0",
     "winston": "^3.7.2",
     "yaml": "^2.3.1"
@@ -33,6 +34,7 @@
     "@types/express-prometheus-middleware": "^1.2.1",
     "@types/jest": "^29.5.12",
     "@types/node": "^16.11.7",
+    "@types/redis": "^4.0.11",
     "@types/swagger-ui-express": "^4.1.3",
     "@typescript-eslint/eslint-plugin": "^5.26.0",
     "@typescript-eslint/parser": "^5.26.0",
@@ -59,6 +61,7 @@
     "lint:check": "eslint src",
     "lint:fix": "eslint --fix src",
     "format:check": "prettier --check .",
-    "format:fix": "prettier --write ."
+    "format:fix": "prettier --write .",
+    "test": "NODE_OPTIONS=\"--import=./register.js\" node --test src/**/*.test.ts"
   }
 }

register.js

@@ -0,0 +1,4 @@
+import { register } from 'node:module';
+import { pathToFileURL } from 'node:url';
+
+register('ts-node/esm', pathToFileURL('./'));

src/cache/arns-store.ts

@@ -0,0 +1,69 @@
+/**
+ * AR.IO ArNS Resolver
+ * Copyright (C) 2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+import winston from 'winston';
+
+import { KVBufferStore } from '../types.js';
+
+export class ArNSStore implements KVBufferStore {
+  private log: winston.Logger;
+  private prefix: string;
+  private kvStore: KVBufferStore;
+
+  constructor({
+    log,
+    kvStore,
+    prefix = 'ArNS',
+  }: {
+    log: winston.Logger;
+    kvStore: KVBufferStore;
+    prefix?: string;
+  }) {
+    this.log = log.child({ class: this.constructor.name });
+    this.kvStore = kvStore;
+    this.prefix = prefix;
+    this.log.info('ArNSStore initialized', {
+      prefix,
+      kvStore: kvStore.constructor.name,
+    });
+  }
+
+  // avoid collisions with other redis keys
+  private hashKey(key: string): string {
+    return `${this.prefix}|${key}`;
+  }
+
+  async get(key: string): Promise<Buffer | undefined> {
+    return this.kvStore.get(this.hashKey(key));
+  }
+
+  async set(key: string, value: Buffer, ttlSeconds?: number): Promise<void> {
+    return this.kvStore.set(this.hashKey(key), value, ttlSeconds);
+  }
+
+  async del(key: string): Promise<void> {
+    return this.kvStore.del(this.hashKey(key));
+  }
+
+  async has(key: string): Promise<boolean> {
+    return this.kvStore.has(this.hashKey(key));
+  }
+
+  async close(): Promise<void> {
+    return this.kvStore.close();
+  }
+}

src/cache/lmdb-kv-store.test.ts

@@ -15,6 +15,9 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
+import { strict as assert } from 'node:assert';
+import { describe, it } from 'node:test';
+
 import { LmdbKVStore } from './lmdb-kv-store.js';
 
 describe('LmdbKVStore', () => {
@@ -23,16 +26,28 @@ describe('LmdbKVStore', () => {
     ttlSeconds: 1,
   });
 
-  it('should set and get value', async () => {
+  it('should set and get value with default ttl', async () => {
     await cache.set('test', Buffer.from('hello'));
     const value = await cache.get('test');
-    expect(value).toEqual(Buffer.from('hello'));
+    assert.deepEqual(value, Buffer.from('hello'));
   });
 
-  it('should remove a value once ttl has expired', async () => {
+  it('should remove a value once default ttl has expired ', async () => {
     await cache.set('expire', Buffer.from('hello'));
     await new Promise((resolve) => setTimeout(resolve, 1000));
     const value = await cache.get('expire');
-    expect(value).toBeUndefined();
+    assert.strictEqual(value, undefined);
+  });
+
+  it('should override the default ttl when a ttl is provided when setting a record', async () => {
+    await cache.set('test', Buffer.from('hello'), 3);
+    // get it right away
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+    const value = await cache.get('test');
+    assert.deepEqual(value, Buffer.from('hello'));
+    // wait for it to expire
+    await new Promise((resolve) => setTimeout(resolve, 3000));
+    const value2 = await cache.get('test');
+    assert.strictEqual(value2, undefined);
   });
 });

src/cache/lmdb-kv-store.ts

@@ -35,11 +35,14 @@ export class LmdbKVStore implements KVBufferStore {
   /**
    * Attach the TTL to the value.
    */
-  private serialize(value: Buffer): Buffer {
-    if (this.ttlSeconds === undefined) return value;
+  private serialize(
+    value: Buffer,
+    ttlSeconds: number | undefined = this.ttlSeconds,
+  ): Buffer {
+    if (ttlSeconds === undefined) return value;
     const expirationTimestamp = Buffer.allocUnsafe(8); // 8 bytes for a timestamp
     expirationTimestamp.writeBigInt64BE(
-      BigInt(Date.now() + this.ttlSeconds * 1000),
+      BigInt(Date.now() + ttlSeconds * 1000),
       0,
     );
     return Buffer.concat([expirationTimestamp, value]);
@@ -93,7 +96,11 @@ export class LmdbKVStore implements KVBufferStore {
   /**
    * Set the value in the database with the TTL.
    */
-  async set(key: string, buffer: Buffer): Promise<void> {
-    await this.db.put(key, this.serialize(buffer));
+  async set(key: string, buffer: Buffer, ttlSeconds?: number): Promise<void> {
+    await this.db.put(key, this.serialize(buffer, ttlSeconds));
+  }
+
+  async close(): Promise<void> {
+    await this.db.close();
   }
 }

src/cache/redis-kv-store.ts

@@ -0,0 +1,80 @@
+/**
+ * AR.IO ArNS Resolver
+ * Copyright (C) 2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+import { RedisClientType, commandOptions, createClient } from 'redis';
+import winston from 'winston';
+
+import { KVBufferStore } from '../types.js';
+
+export class RedisKvStore implements KVBufferStore {
+  private client: RedisClientType;
+  private log: winston.Logger;
+  private defaultTtlSeconds?: number;
+
+  constructor({ log, redisUrl }: { log: winston.Logger; redisUrl: string }) {
+    this.log = log.child({ class: this.constructor.name });
+    this.client = createClient({
+      url: redisUrl,
+    });
+    this.client.on('error', (error: any) => {
+      this.log.error(`Redis error`, {
+        message: error.message,
+        stack: error.stack,
+      });
+      // TODO: add prometheus metric for redis error
+    });
+    this.client.connect().catch((error: any) => {
+      this.log.error(`Redis connection error`, {
+        message: error.message,
+        stack: error.stack,
+      });
+      // TODO: add prometheus metric for redis connection error
+    });
+  }
+
+  async close() {
+    await this.client.quit();
+  }
+
+  async get(key: string): Promise<Buffer | undefined> {
+    const value = await this.client.get(
+      commandOptions({ returnBuffers: true }),
+      key,
+    );
+    return value ?? undefined;
+  }
+
+  async has(key: string): Promise<boolean> {
+    return (await this.client.exists(key)) === 1;
+  }
+
+  async del(key: string): Promise<void> {
+    if (await this.has(key)) {
+      await this.client.del(key);
+    }
+  }
+
+  async set(key: string, buffer: Buffer, ttlSeconds?: number): Promise<void> {
+    if (ttlSeconds !== undefined) {
+      await this.client.set(key, buffer, {
+        EX: ttlSeconds ?? this.defaultTtlSeconds,
+      });
+    } else {
+      await this.client.set(key, buffer);
+    }
+  }
+}

src/config.ts

@@ -31,6 +31,11 @@ export const ARNS_CACHE_TTL_MS = +env.varOrDefault(
   'ARNS_CACHE_TTL_MS',
   `${1000 * 60 * 60}`, // 1 hour by default
 );
+export const ARNS_CACHE_TYPE = env.varOrDefault('ARNS_CACHE_TYPE', 'lmdb');
+export const REDIS_CACHE_URL = env.varOrDefault(
+  'REDIS_CACHE_URL',
+  'redis://localhost:6379',
+);
 export const RUN_RESOLVER = env.varOrDefault('RUN_RESOLVER', 'true') === 'true';
 export const ENABLE_OPENAPI_VALIDATION =
   env.varOrDefault('ENABLE_OPENAPI_VALIDATION', 'true') === 'true';