feat(PE-6695/PE-6696): remove resolver, add redis support for arns cache

docker-compose.yaml

@@ -71,11 +71,11 @@ services:
       - WEBHOOK_INDEX_FILTER=${WEBHOOK_INDEX_FILTER:-}
       - WEBHOOK_BLOCK_FILTER=${WEBHOOK_BLOCK_FILTER:-}
       - CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD=${CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD:-}
-      - TRUSTED_ARNS_RESOLVER_URL=${TRUSTED_ARNS_RESOLVER_URL:-}
       - TRUSTED_ARNS_GATEWAY_URL=${TRUSTED_ARNS_GATEWAY_URL:-https://__NAME__.arweave.net}
       - ARNS_RESOLVER_PRIORITY_ORDER=${ARNS_RESOLVER_PRIORITY_ORDER:-on-demand,gateway}
       - ARNS_CACHE_TTL_SECONDS=${ARNS_CACHE_TTL_SECONDS:-3600}
       - ARNS_CACHE_MAX_KEYS=${ARNS_CACHE_MAX_KEYS:-10000}
+      - ARNS_CACHE_TYPE=${ARNS_CACHE_TYPE:-redis}
       - ENABLE_MEMPOOL_WATCHER=${ENABLE_MEMPOOL_WATCHER:-false}
       - MEMPOOL_POOLING_INTERVAL_MS=${MEMPOOL_POOLING_INTERVAL_MS:-}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
@@ -139,28 +139,6 @@ services:
     networks:
       - ar-io-network
 
-  resolver:
-    image: ghcr.io/ar-io/arns-resolver:${RESOLVER_IMAGE_TAG:-7fe02ecda2027e504248d3f3716579f60b561de5}
-    restart: on-failure
-    ports:
-      - 6000:6000
-    environment:
-      - PORT=6000
-      - LOG_LEVEL=${LOG_LEVEL:-info}
-      - IO_PROCESS_ID=${IO_PROCESS_ID:-}
-      - RUN_RESOLVER=${RUN_RESOLVER:-false}
-      - EVALUATION_INTERVAL_MS=${EVALUATION_INTERVAL_MS:-}
-      - ARNS_CACHE_TTL_MS=${RESOLVER_CACHE_TTL_MS:-}
-      - ARNS_CACHE_PATH=${ARNS_CACHE_PATH:-./data/arns}
-      - AO_CU_URL=${AO_CU_URL:-}
-      - AO_MU_URL=${AO_MU_URL:-}
-      - AO_GATEWAY_URL=${AO_GATEWAY_URL:-}
-      - AO_GRAPHQL_URL=${AO_GRAPHQL_URL:-}
-    volumes:
-      - ${ARNS_CACHE_PATH:-./data/arns}:/app/data/arns
-    networks:
-      - ar-io-network
-
   litestream:
     image: ghcr.io/ar-io/ar-io-litestream:${LITESTREAM_IMAGE_TAG:-latest}
     build:

src/config.ts

@@ -258,6 +258,8 @@ export const WEBHOOK_BLOCK_FILTER = createFilter(
 // ArNS Resolution
 //
 
+export const ARNS_CACHE_TYPE = env.varOrDefault('ARNS_CACHE_TYPE', 'node');
+
 export const ARNS_CACHE_TTL_SECONDS = +env.varOrDefault(
   'ARNS_CACHE_TTL_SECONDS',
   `${60 * 60}`, // 1 hour
@@ -269,23 +271,36 @@ export const ARNS_CACHE_MAX_KEYS = +env.varOrDefault(
 );
 
 export const ARNS_RESOLVER_PRIORITY_ORDER = env
-  .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'resolver,on-demand,gateway')
+  .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'on-demand,gateway')
   .split(',');
 
-export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault(
-  'TRUSTED_ARNS_GATEWAY_URL',
-  'https://__NAME__.arweave.dev',
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS = +env.varOrDefault(
+  'ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS',
+  `${5 * 1000}`, // 5 seconds
 );
 
-// @deprecated - use ARNS_RESOLVER_PRIORITY_ORDER instead to specify the order
-// of resolvers to try if the first one is not available.
-export const TRUSTED_ARNS_RESOLVER_TYPE = env.varOrDefault(
-  'TRUSTED_ARNS_RESOLVER_TYPE',
-  'gateway',
-);
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE =
+  +env.varOrDefault(
+    'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE',
+    '50',
+  );
 
-export const TRUSTED_ARNS_RESOLVER_URL = env.varOrUndefined(
-  'TRUSTED_ARNS_RESOLVER_URL',
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS =
+  +env.varOrDefault(
+    'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS',
+    `${60 * 1000}`, // 1 minute
+  );
+
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS =
+  +env.varOrDefault(
+    'ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS',
+    `${5 * 60 * 1000}`, // 5 minutes
+  );
+
+// TODO: support multiple gateway urls
+export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault(
+  'TRUSTED_ARNS_GATEWAY_URL',
+  'https://__NAME__.arweave.net',
 );
 
 //

src/init/resolvers.ts

@@ -16,30 +16,61 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 import { Logger } from 'winston';
-import { StandaloneArNSResolver } from '../resolution/standalone-arns-resolver.js';
 import { OnDemandArNSResolver } from '../resolution/on-demand-arns-resolver.js';
 import { TrustedGatewayArNSResolver } from '../resolution/trusted-gateway-arns-resolver.js';
-import { NameResolver } from '../types.js';
+import { KVBufferStore, NameResolver } from '../types.js';
 import { AoIORead } from '@ar.io/sdk';
 import { CompositeArNSResolver } from '../resolution/composite-arns-resolver.js';
+import { RedisKvStore } from '../store/redis-kv-store.js';
+import { NodeKvStore } from '../store/node-kv-store.js';
+import { KvArnsStore } from '../store/kv-arns-store.js';
 
-const supportedResolvers = ['on-demand', 'resolver', 'gateway'] as const;
+const supportedResolvers = ['on-demand', 'gateway'] as const;
 export type ArNSResolverType = (typeof supportedResolvers)[number];
 
 export const isArNSResolverType = (type: string): type is ArNSResolverType => {
   return supportedResolvers.includes(type as ArNSResolverType);
 };
 
+export const createArNSKvStore = ({
+  log,
+  type,
+  redisUrl,
+  ttlSeconds,
+  maxKeys,
+}: {
+  type: 'redis' | 'node' | string;
+  log: Logger;
+  redisUrl: string;
+  ttlSeconds: number;
+  maxKeys: number;
+}): KVBufferStore => {
+  log.info(`Using ${type} as KVBufferStore for arns`, {
+    type,
+    redisUrl,
+    ttlSeconds,
+    maxKeys,
+  });
+  if (type === 'redis') {
+    return new RedisKvStore({
+      log,
+      redisUrl,
+      ttlSeconds,
+    });
+  }
+  return new NodeKvStore({ ttlSeconds, maxKeys });
+};
+
 export const createArNSResolver = ({
   log,
+  cache,
   resolutionOrder,
-  standaloneArnResolverUrl,
   trustedGatewayUrl,
   networkProcess,
 }: {
   log: Logger;
+  cache: KvArnsStore;
   resolutionOrder: (ArNSResolverType | string)[];
-  standaloneArnResolverUrl?: string;
   trustedGatewayUrl?: string;
   networkProcess?: AoIORead;
 }): NameResolver => {
@@ -49,13 +80,6 @@ export const createArNSResolver = ({
       log,
       networkProcess,
     }),
-    resolver:
-      standaloneArnResolverUrl !== undefined
-        ? new StandaloneArNSResolver({
-            log,
-            resolverUrl: standaloneArnResolverUrl,
-          })
-        : undefined,
     gateway:
       trustedGatewayUrl !== undefined
         ? new TrustedGatewayArNSResolver({
@@ -82,5 +106,6 @@ export const createArNSResolver = ({
   return new CompositeArNSResolver({
     log,
     resolvers,
+    cache,
   });
 };

src/metrics.ts

@@ -180,6 +180,21 @@ export const redisErrorCounter = new promClient.Counter({
   help: 'Number of errors redis cache has received',
 });
 
+export const arnsCacheHitCounter = new promClient.Counter({
+  name: 'arns_cache_hit_total',
+  help: 'Number of hits in the arns cache',
+});
+
+export const arnsCacheMissCounter = new promClient.Counter({
+  name: 'arns_cache_miss_total',
+  help: 'Number of misses in the arns cache',
+});
+
+export const arnsResolutionTime = new promClient.Summary({
+  name: 'arns_resolution_time_ms',
+  help: 'Time in ms it takes to resolve an arns name',
+});
+
 // Data source metrics
 
 export const getDataErrorsTotal = new promClient.Counter({

src/middleware/arns.ts

@@ -22,10 +22,19 @@ import * as config from '../config.js';
 import { headerNames } from '../constants.js';
 import { sendNotFound } from '../routes/data/handlers.js';
 import { DATA_PATH_REGEX } from '../constants.js';
-import { NameResolver } from '../types.js';
+import { NameResolution, NameResolver } from '../types.js';
+import * as metrics from '../metrics.js';
+import NodeCache from 'node-cache';
 
 const EXCLUDED_SUBDOMAINS = new Set('www');
 
+// simple cache that stores the arns resolution promises to avoid duplicate requests to the name resolver
+const arnsRequestCache = new NodeCache({
+  stdTTL: 60, // short cache in case we forget to delete
+  checkperiod: 60,
+  useClones: false, // cloning promises is unsafe
+});
+
 export const createArnsMiddleware = ({
   dataHandler,
   nameResolver,
@@ -53,7 +62,7 @@ export const createArnsMiddleware = ({
     if (
       EXCLUDED_SUBDOMAINS.has(arnsSubdomain) ||
       // Avoid collisions with sandbox URLs by ensuring the subdomain length
-      // is below the mininimum length of a sandbox subdomain. Undernames are
+      // is below the minimum length of a sandbox subdomain. Undernames are
       // are an exception because they can be longer and '_' cannot appear in
       // base32.
       (arnsSubdomain.length > 48 && !arnsSubdomain.match(/_/))
@@ -67,15 +76,34 @@ export const createArnsMiddleware = ({
       return;
     }
 
+    const getArnsResolutionPromise = async (): Promise<NameResolution> => {
+      if (arnsRequestCache.has(arnsSubdomain)) {
+        const arnsResolutionPromise =
+          arnsRequestCache.get<Promise<NameResolution>>(arnsSubdomain);
+        if (arnsResolutionPromise) {
+          return arnsResolutionPromise;
+        }
+      }
+      const arnsResolutionPromise = nameResolver.resolve(arnsSubdomain);
+      arnsRequestCache.set(arnsSubdomain, arnsResolutionPromise);
+      return arnsResolutionPromise;
+    };
+
+    const start = Date.now();
     const { resolvedId, ttl, processId } =
-      await nameResolver.resolve(arnsSubdomain);
+      await getArnsResolutionPromise().finally(() => {
+        // remove from cache after resolution
+        arnsRequestCache.del(arnsSubdomain);
+      });
+    metrics.arnsResolutionTime.observe(Date.now() - start);
     if (resolvedId === undefined) {
       sendNotFound(res);
       return;
     }
     res.header(headerNames.arnsResolvedId, resolvedId);
     res.header(headerNames.arnsTtlSeconds, ttl.toString());
     res.header(headerNames.arnsProcessId, processId);
+    // TODO: add a header for arns cache status
     res.header('Cache-Control', `public, max-age=${ttl}`);
     dataHandler(req, res, next);
   });

src/resolution/composite-arns-resolver.ts

@@ -17,56 +17,90 @@
  */
 import winston from 'winston';
 import { NameResolution, NameResolver } from '../types.js';
+import * as metrics from '../metrics.js';
+import { KvArnsStore } from '../store/kv-arns-store.js';
 
 export class CompositeArNSResolver implements NameResolver {
   private log: winston.Logger;
   private resolvers: NameResolver[];
+  private cache: KvArnsStore;
 
   constructor({
     log,
     resolvers,
+    cache,
   }: {
     log: winston.Logger;
     resolvers: NameResolver[];
+    cache: KvArnsStore;
   }) {
-    this.log = log.child({ class: 'CompositeArNSResolver' });
+    this.log = log.child({ class: this.constructor.name });
     this.resolvers = resolvers;
+    this.cache = cache;
   }
 
   async resolve(name: string): Promise<NameResolution> {
     this.log.info('Resolving name...', { name });
+    let resolution: NameResolution | undefined;
 
     try {
+      const cachedResolutionBuffer = await this.cache.get(name);
+      if (cachedResolutionBuffer) {
+        const cachedResolution: NameResolution = JSON.parse(
+          cachedResolutionBuffer.toString(),
+        );
+        resolution = cachedResolution; // hold on tho this in case we need it
+        if (
+          cachedResolution !== undefined &&
+          cachedResolution.resolvedAt !== undefined &&
+          cachedResolution.ttl !== undefined &&
+          cachedResolution.resolvedAt + cachedResolution.ttl * 1000 > Date.now()
+        ) {
+          metrics.arnsCacheHitCounter.inc();
+          this.log.info('Cache hit for arns name', { name });
+          return cachedResolution;
+        }
+      }
+      metrics.arnsCacheMissCounter.inc();
+      this.log.info('Cache miss for arns name', { name });
+
       for (const resolver of this.resolvers) {
-        this.log.debug('Attempting to resolve name with resolver', {
-          resolver,
-        });
-        const resolution = await resolver.resolve(name);
-        if (resolution.resolvedId !== undefined) {
-          return resolution;
+        try {
+          this.log.info('Attempting to resolve name with resolver', {
+            type: resolver.constructor.name,
+            name,
+          });
+          const resolution = await resolver.resolve(name);
+          if (resolution.resolvedId !== undefined) {
+            this.cache.set(name, Buffer.from(JSON.stringify(resolution)));
+            this.log.info('Resolved name', { name, resolution });
+            return resolution;
+          }
+        } catch (error: any) {
+          this.log.error('Error resolving name with resolver', {
+            resolver,
+            message: error.message,
+            stack: error.stack,
+          });
         }
       }
       this.log.warn('Unable to resolve name against all resolvers', { name });
-      return {
-        name,
-        resolvedId: undefined,
-        resolvedAt: undefined,
-        processId: undefined,
-        ttl: undefined,
-      };
     } catch (error: any) {
       this.log.error('Error resolving name:', {
         name,
         message: error.message,
         stack: error.stack,
       });
-      return {
+    }
+    // return the resolution if it exists, otherwise return an empty resolution
+    return (
+      resolution ?? {
         name,
         resolvedId: undefined,
         resolvedAt: undefined,
         ttl: undefined,
         processId: undefined,
-      };
-    }
+      }
+    );
   }
 }