feat(arns): remove resolver, add redis support for arns cache

With on-demand resolution, we no longer need to support the arns-resolver. We can effectively fetch and cache arns resolutions quickly via AO and cache them locally or in redis. This replaces the default `MemoryArNSCache` with a configurable KvBufferStore that supports `redis` or a local `node-cache`. When resolving an arns name, the `CompositeArNSResolver` will check the provided cache and the TTL of the record, if it is not in the cache it will then use the available arns resolvers (on-demand and/or another gateway) to get resolution data. If an operator would like to disable caching of arns names, and always resolve to the latest they can set ARNS_CACHE_TTL_SECONDS to 0. Additionally, prometheus metrics are available for hit/miss rate for the arns cache and total resolution times.
ar-io · Sep 9, 2024 · 3c62e33 · 3c62e33
1 parent bbf79b1
commit 3c62e33
Show file tree

Hide file tree

Showing 13 changed files with 216 additions and 86 deletions.
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -71,11 +71,11 @@ services:
       - WEBHOOK_INDEX_FILTER=${WEBHOOK_INDEX_FILTER:-}
       - WEBHOOK_BLOCK_FILTER=${WEBHOOK_BLOCK_FILTER:-}
       - CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD=${CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD:-}
-      - TRUSTED_ARNS_RESOLVER_URL=${TRUSTED_ARNS_RESOLVER_URL:-}
       - TRUSTED_ARNS_GATEWAY_URL=${TRUSTED_ARNS_GATEWAY_URL:-https://__NAME__.arweave.net}
       - ARNS_RESOLVER_PRIORITY_ORDER=${ARNS_RESOLVER_PRIORITY_ORDER:-on-demand,gateway}
       - ARNS_CACHE_TTL_SECONDS=${ARNS_CACHE_TTL_SECONDS:-3600}
       - ARNS_CACHE_MAX_KEYS=${ARNS_CACHE_MAX_KEYS:-10000}
+      - ARNS_CACHE_TYPE=${ARNS_CACHE_TYPE:-redis}
       - ENABLE_MEMPOOL_WATCHER=${ENABLE_MEMPOOL_WATCHER:-false}
       - MEMPOOL_POOLING_INTERVAL_MS=${MEMPOOL_POOLING_INTERVAL_MS:-}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
@@ -139,28 +139,6 @@ services:
     networks:
       - ar-io-network
 
-  resolver:
-    image: ghcr.io/ar-io/arns-resolver:${RESOLVER_IMAGE_TAG:-7fe02ecda2027e504248d3f3716579f60b561de5}
-    restart: on-failure
-    ports:
-      - 6000:6000
-    environment:
-      - PORT=6000
-      - LOG_LEVEL=${LOG_LEVEL:-info}
-      - IO_PROCESS_ID=${IO_PROCESS_ID:-}
-      - RUN_RESOLVER=${RUN_RESOLVER:-false}
-      - EVALUATION_INTERVAL_MS=${EVALUATION_INTERVAL_MS:-}
-      - ARNS_CACHE_TTL_MS=${RESOLVER_CACHE_TTL_MS:-}
-      - ARNS_CACHE_PATH=${ARNS_CACHE_PATH:-./data/arns}
-      - AO_CU_URL=${AO_CU_URL:-}
-      - AO_MU_URL=${AO_MU_URL:-}
-      - AO_GATEWAY_URL=${AO_GATEWAY_URL:-}
-      - AO_GRAPHQL_URL=${AO_GRAPHQL_URL:-}
-    volumes:
-      - ${ARNS_CACHE_PATH:-./data/arns}:/app/data/arns
-    networks:
-      - ar-io-network
-
   litestream:
     image: ghcr.io/ar-io/ar-io-litestream:${LITESTREAM_IMAGE_TAG:-latest}
     build:

diff --git a/src/config.ts b/src/config.ts
@@ -258,6 +258,8 @@ export const WEBHOOK_BLOCK_FILTER = createFilter(
 // ArNS Resolution
 //
 
+export const ARNS_CACHE_TYPE = env.varOrDefault('ARNS_CACHE_TYPE', 'node');
+
 export const ARNS_CACHE_TTL_SECONDS = +env.varOrDefault(
   'ARNS_CACHE_TTL_SECONDS',
   `${60 * 60}`, // 1 hour
@@ -269,25 +271,14 @@ export const ARNS_CACHE_MAX_KEYS = +env.varOrDefault(
 );
 
 export const ARNS_RESOLVER_PRIORITY_ORDER = env
-  .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'resolver,on-demand,gateway')
+  .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'on-demand,gateway')
   .split(',');
 
 export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault(
   'TRUSTED_ARNS_GATEWAY_URL',
   'https://__NAME__.arweave.dev',
 );
 
-// @deprecated - use ARNS_RESOLVER_PRIORITY_ORDER instead to specify the order
-// of resolvers to try if the first one is not available.
-export const TRUSTED_ARNS_RESOLVER_TYPE = env.varOrDefault(
-  'TRUSTED_ARNS_RESOLVER_TYPE',
-  'gateway',
-);
-
-export const TRUSTED_ARNS_RESOLVER_URL = env.varOrUndefined(
-  'TRUSTED_ARNS_RESOLVER_URL',
-);
-
 //
 // Mempool watcher
 //

diff --git a/src/init/resolvers.ts b/src/init/resolvers.ts
@@ -16,30 +16,61 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 import { Logger } from 'winston';
-import { StandaloneArNSResolver } from '../resolution/standalone-arns-resolver.js';
 import { OnDemandArNSResolver } from '../resolution/on-demand-arns-resolver.js';
 import { TrustedGatewayArNSResolver } from '../resolution/trusted-gateway-arns-resolver.js';
-import { NameResolver } from '../types.js';
+import { KVBufferStore, NameResolver } from '../types.js';
 import { AoIORead } from '@ar.io/sdk';
 import { CompositeArNSResolver } from '../resolution/composite-arns-resolver.js';
+import { RedisKvStore } from '../store/redis-kv-store.js';
+import { NodeKvStore } from '../store/node-kv-store.js';
 
-const supportedResolvers = ['on-demand', 'resolver', 'gateway'] as const;
+const supportedResolvers = ['on-demand', 'gateway'] as const;
 export type ArNSResolverType = (typeof supportedResolvers)[number];
 
 export const isArNSResolverType = (type: string): type is ArNSResolverType => {
   return supportedResolvers.includes(type as ArNSResolverType);
 };
 
+export const createArNSKvStore = ({
+  log,
+  type,
+  redisUrl,
+  ttlSeconds,
+  maxKeys,
+}: {
+  type: 'redis' | 'node' | string;
+  log: Logger;
+  redisUrl: string;
+  ttlSeconds: number;
+  maxKeys: number;
+}): KVBufferStore => {
+  // or node cache
+  log.info(`Using ${type} as KVBufferStore for arns`, {
+    type,
+    redisUrl,
+    ttlSeconds,
+    maxKeys,
+  });
+  if (type === 'redis') {
+    return new RedisKvStore({
+      log,
+      redisUrl,
+      ttlSeconds,
+    });
+  }
+  return new NodeKvStore({ ttlSeconds, maxKeys });
+};
+
 export const createArNSResolver = ({
   log,
+  cache,
   resolutionOrder,
-  standaloneArnResolverUrl,
   trustedGatewayUrl,
   networkProcess,
 }: {
   log: Logger;
+  cache: KVBufferStore;
   resolutionOrder: (ArNSResolverType | string)[];
-  standaloneArnResolverUrl?: string;
   trustedGatewayUrl?: string;
   networkProcess?: AoIORead;
 }): NameResolver => {
@@ -49,13 +80,6 @@ export const createArNSResolver = ({
       log,
       networkProcess,
     }),
-    resolver:
-      standaloneArnResolverUrl !== undefined
-        ? new StandaloneArNSResolver({
-            log,
-            resolverUrl: standaloneArnResolverUrl,
-          })
-        : undefined,
     gateway:
       trustedGatewayUrl !== undefined
         ? new TrustedGatewayArNSResolver({
@@ -82,5 +106,6 @@ export const createArNSResolver = ({
   return new CompositeArNSResolver({
     log,
     resolvers,
+    cache,
   });
 };
diff --git a/src/metrics.ts b/src/metrics.ts
@@ -180,6 +180,21 @@ export const redisErrorCounter = new promClient.Counter({
   help: 'Number of errors redis cache has received',
 });
 
+export const arnsCacheHitCounter = new promClient.Counter({
+  name: 'arns_cache_hit_total',
+  help: 'Number of hits in the arns cache',
+});
+
+export const arnsCacheMissCounter = new promClient.Counter({
+  name: 'arns_cache_miss_total',
+  help: 'Number of misses in the arns cache',
+});
+
+export const arnsResolutionTime = new promClient.Summary({
+  name: 'arns_resolution_time',
+  help: 'Time it takes to resolve an arns name',
+});
+
 // Data source metrics
 
 export const getDataErrorsTotal = new promClient.Counter({

diff --git a/src/middleware/arns.ts b/src/middleware/arns.ts
@@ -23,7 +23,7 @@ import { headerNames } from '../constants.js';
 import { sendNotFound } from '../routes/data/handlers.js';
 import { DATA_PATH_REGEX } from '../constants.js';
 import { NameResolver } from '../types.js';
-
+import * as metrics from '../metrics.js';
 const EXCLUDED_SUBDOMAINS = new Set('www');
 
 export const createArnsMiddleware = ({
@@ -53,7 +53,7 @@ export const createArnsMiddleware = ({
     if (
       EXCLUDED_SUBDOMAINS.has(arnsSubdomain) ||
       // Avoid collisions with sandbox URLs by ensuring the subdomain length
-      // is below the mininimum length of a sandbox subdomain. Undernames are
+      // is below the minimum length of a sandbox subdomain. Undernames are
       // are an exception because they can be longer and '_' cannot appear in
       // base32.
       (arnsSubdomain.length > 48 && !arnsSubdomain.match(/_/))
@@ -67,15 +67,18 @@ export const createArnsMiddleware = ({
       return;
     }
 
+    const start = Date.now();
     const { resolvedId, ttl, processId } =
       await nameResolver.resolve(arnsSubdomain);
+    metrics.arnsResolutionTime.observe(Date.now() - start);
     if (resolvedId === undefined) {
       sendNotFound(res);
       return;
     }
     res.header(headerNames.arnsResolvedId, resolvedId);
     res.header(headerNames.arnsTtlSeconds, ttl.toString());
     res.header(headerNames.arnsProcessId, processId);
+    // TODO: add a header for arns cache status
     res.header('Cache-Control', `public, max-age=${ttl}`);
     dataHandler(req, res, next);
   });
diff --git a/src/resolution/composite-arns-resolver.ts b/src/resolution/composite-arns-resolver.ts
@@ -16,34 +16,75 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 import winston from 'winston';
-import { NameResolution, NameResolver } from '../types.js';
+import { KVBufferStore, NameResolution, NameResolver } from '../types.js';
+import * as metrics from '../metrics.js';
 
 export class CompositeArNSResolver implements NameResolver {
   private log: winston.Logger;
   private resolvers: NameResolver[];
+  private cache: KVBufferStore;
 
   constructor({
     log,
     resolvers,
+    cache,
   }: {
     log: winston.Logger;
     resolvers: NameResolver[];
+    cache: KVBufferStore;
   }) {
-    this.log = log.child({ class: 'CompositeArNSResolver' });
+    this.log = log.child({ class: this.constructor.name });
     this.resolvers = resolvers;
+    this.cache = cache;
+  }
+
+  private hashKey(key: string): string {
+    return `arns|${key}`;
   }
 
   async resolve(name: string): Promise<NameResolution> {
     this.log.info('Resolving name...', { name });
 
     try {
+      const cachedResolutionBuffer = await this.cache.get(this.hashKey(name));
+      if (cachedResolutionBuffer) {
+        const cachedResolution: NameResolution = JSON.parse(
+          cachedResolutionBuffer.toString(),
+        );
+        if (
+          cachedResolution !== undefined &&
+          cachedResolution.resolvedAt !== undefined &&
+          cachedResolution.ttl !== undefined &&
+          cachedResolution.resolvedAt + cachedResolution.ttl * 1000 > Date.now()
+        ) {
+          metrics.arnsCacheHitCounter.inc();
+          this.log.info('Cache hit for arns name', { name });
+          return cachedResolution;
+        }
+      }
+      metrics.arnsCacheMissCounter.inc();
+      this.log.info('Cache miss for arns name', { name });
+
       for (const resolver of this.resolvers) {
-        this.log.debug('Attempting to resolve name with resolver', {
-          resolver,
-        });
-        const resolution = await resolver.resolve(name);
-        if (resolution.resolvedId !== undefined) {
-          return resolution;
+        try {
+          this.log.info('Attempting to resolve name with resolver', {
+            type: resolver.constructor.name,
+            name,
+          });
+          const resolution = await resolver.resolve(name);
+          if (resolution.resolvedId !== undefined) {
+            const hashKey = this.hashKey(name);
+            const resolutionBuffer = Buffer.from(JSON.stringify(resolution));
+            await this.cache.set(hashKey, resolutionBuffer);
+            this.log.info('Resolved name', { name, resolution });
+            return resolution;
+          }
+        } catch (error: any) {
+          this.log.error('Error resolving name with resolver', {
+            resolver,
+            message: error.message,
+            stack: error.stack,
+          });
         }
       }
       this.log.warn('Unable to resolve name against all resolvers', { name });

diff --git a/src/resolution/on-demand-arns-resolver.ts b/src/resolution/on-demand-arns-resolver.ts
@@ -19,33 +19,44 @@ import winston from 'winston';
 
 import { isValidDataId } from '../lib/validation.js';
 import { NameResolution, NameResolver } from '../types.js';
-import { ANT, AoIORead, AOProcess, IO } from '@ar.io/sdk';
+import { ANT, AoClient, AoIORead, AOProcess, IO } from '@ar.io/sdk';
 import * as config from '../config.js';
 import { connect } from '@permaweb/aoconnect';
 
 export class OnDemandArNSResolver implements NameResolver {
   private log: winston.Logger;
   private networkProcess: AoIORead;
+  private ao: AoClient;
 
   constructor({
     log,
+    ao = connect({
+      MU_URL: config.AO_MU_URL,
+      CU_URL: config.AO_CU_URL,
+      GRAPHQL_URL: config.AO_GRAPHQL_URL,
+      GATEWAY_URL: config.AO_GATEWAY_URL,
+    }),
     networkProcess = IO.init({
-      processId: config.IO_PROCESS_ID,
+      process: new AOProcess({
+        processId: config.IO_PROCESS_ID,
+        ao: ao,
+      }),
     }),
   }: {
     log: winston.Logger;
     networkProcess?: AoIORead;
+    ao?: AoClient;
   }) {
     this.log = log.child({
       class: 'OnDemandArNSResolver',
     });
     this.networkProcess = networkProcess;
+    this.ao = ao;
   }
 
   async resolve(name: string): Promise<NameResolution> {
     this.log.info('Resolving name...', { name });
     try {
-      const start = Date.now();
       // get the base name which is the last of th array split by _
       const baseName = name.split('_').pop();
       if (baseName === undefined) {
@@ -66,12 +77,7 @@ export class OnDemandArNSResolver implements NameResolver {
       const ant = ANT.init({
         process: new AOProcess({
           processId: processId,
-          ao: connect({
-            MU_URL: config.AO_MU_URL,
-            CU_URL: config.AO_CU_URL,
-            GRAPHQL_URL: config.AO_GRAPHQL_URL,
-            GATEWAY_URL: config.AO_GATEWAY_URL,
-          }),
+          ao: this.ao,
         }),
       });
 
@@ -93,13 +99,6 @@ export class OnDemandArNSResolver implements NameResolver {
       if (!isValidDataId(resolvedId)) {
         throw new Error('Invalid resolved data ID');
       }
-
-      this.log.info('Resolved name', {
-        name,
-        resolvedId,
-        ttl,
-        durationMs: Date.now() - start,
-      });
       return {
         name,
         resolvedId,

diff --git a/src/store/fs-kv-store.ts b/src/store/fs-kv-store.ts
@@ -66,4 +66,8 @@ export class FsKVStore implements KVBufferStore {
       await fse.move(tmpPath, this.bufferPath(key));
     }
   }
+
+  async close(): Promise<void> {
+    // No-op
+  }
 }