From 3c62e336d33a42dc33fd22d55c025016ca1bc14b Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Mon, 9 Sep 2024 16:19:02 -0600 Subject: [PATCH] feat(arns): remove resolver, add redis support for arns cache With on-demand resolution, we no longer need to support the arns-resolver. We can effectively fetch and cache arns resolutions quickly via AO and cache them locally or in redis. This replaces the default `MemoryArNSCache` with a configurable KvBufferStore that supports `redis` or a local `node-cache`. When resolving an arns name, the `CompositeArNSResolver` will check the provided cache and the TTL of the record, if it is not in the cache it will then use the available arns resolvers (on-demand and/or another gateway) to get resolution data. If an operator would like to disable caching of arns names, and always resolve to the latest they can set ARNS_CACHE_TTL_SECONDS to 0. Additionally, prometheus metrics are available for hit/miss rate for the arns cache and total resolution times. --- docker-compose.yaml | 24 +-------- src/config.ts | 15 ++---- src/init/resolvers.ts | 49 +++++++++++++----- src/metrics.ts | 15 ++++++ src/middleware/arns.ts | 7 ++- src/resolution/composite-arns-resolver.ts | 57 +++++++++++++++++--- src/resolution/on-demand-arns-resolver.ts | 31 ++++++----- src/store/fs-kv-store.ts | 4 ++ src/store/lmdb-kv-store.ts | 4 ++ src/store/node-kv-store.ts | 63 +++++++++++++++++++++++ src/store/redis-kv-store.ts | 7 +-- src/system.ts | 25 +++++---- src/types.d.ts | 1 + 13 files changed, 216 insertions(+), 86 deletions(-) create mode 100644 src/store/node-kv-store.ts diff --git a/docker-compose.yaml b/docker-compose.yaml index df3ca1f4..cd8c4510 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -71,11 +71,11 @@ services: - WEBHOOK_INDEX_FILTER=${WEBHOOK_INDEX_FILTER:-} - WEBHOOK_BLOCK_FILTER=${WEBHOOK_BLOCK_FILTER:-} - CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD=${CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD:-} - - TRUSTED_ARNS_RESOLVER_URL=${TRUSTED_ARNS_RESOLVER_URL:-} - TRUSTED_ARNS_GATEWAY_URL=${TRUSTED_ARNS_GATEWAY_URL:-https://__NAME__.arweave.net} - ARNS_RESOLVER_PRIORITY_ORDER=${ARNS_RESOLVER_PRIORITY_ORDER:-on-demand,gateway} - ARNS_CACHE_TTL_SECONDS=${ARNS_CACHE_TTL_SECONDS:-3600} - ARNS_CACHE_MAX_KEYS=${ARNS_CACHE_MAX_KEYS:-10000} + - ARNS_CACHE_TYPE=${ARNS_CACHE_TYPE:-redis} - ENABLE_MEMPOOL_WATCHER=${ENABLE_MEMPOOL_WATCHER:-false} - MEMPOOL_POOLING_INTERVAL_MS=${MEMPOOL_POOLING_INTERVAL_MS:-} - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} @@ -139,28 +139,6 @@ services: networks: - ar-io-network - resolver: - image: ghcr.io/ar-io/arns-resolver:${RESOLVER_IMAGE_TAG:-7fe02ecda2027e504248d3f3716579f60b561de5} - restart: on-failure - ports: - - 6000:6000 - environment: - - PORT=6000 - - LOG_LEVEL=${LOG_LEVEL:-info} - - IO_PROCESS_ID=${IO_PROCESS_ID:-} - - RUN_RESOLVER=${RUN_RESOLVER:-false} - - EVALUATION_INTERVAL_MS=${EVALUATION_INTERVAL_MS:-} - - ARNS_CACHE_TTL_MS=${RESOLVER_CACHE_TTL_MS:-} - - ARNS_CACHE_PATH=${ARNS_CACHE_PATH:-./data/arns} - - AO_CU_URL=${AO_CU_URL:-} - - AO_MU_URL=${AO_MU_URL:-} - - AO_GATEWAY_URL=${AO_GATEWAY_URL:-} - - AO_GRAPHQL_URL=${AO_GRAPHQL_URL:-} - volumes: - - ${ARNS_CACHE_PATH:-./data/arns}:/app/data/arns - networks: - - ar-io-network - litestream: image: ghcr.io/ar-io/ar-io-litestream:${LITESTREAM_IMAGE_TAG:-latest} build: diff --git a/src/config.ts b/src/config.ts index 4d3452cd..62814f06 100644 --- a/src/config.ts +++ b/src/config.ts @@ -258,6 +258,8 @@ export const WEBHOOK_BLOCK_FILTER = createFilter( // ArNS Resolution // +export const ARNS_CACHE_TYPE = env.varOrDefault('ARNS_CACHE_TYPE', 'node'); + export const ARNS_CACHE_TTL_SECONDS = +env.varOrDefault( 'ARNS_CACHE_TTL_SECONDS', `${60 * 60}`, // 1 hour @@ -269,7 +271,7 @@ export const ARNS_CACHE_MAX_KEYS = +env.varOrDefault( ); export const ARNS_RESOLVER_PRIORITY_ORDER = env - .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'resolver,on-demand,gateway') + .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'on-demand,gateway') .split(','); export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault( @@ -277,17 +279,6 @@ export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault( 'https://__NAME__.arweave.dev', ); -// @deprecated - use ARNS_RESOLVER_PRIORITY_ORDER instead to specify the order -// of resolvers to try if the first one is not available. -export const TRUSTED_ARNS_RESOLVER_TYPE = env.varOrDefault( - 'TRUSTED_ARNS_RESOLVER_TYPE', - 'gateway', -); - -export const TRUSTED_ARNS_RESOLVER_URL = env.varOrUndefined( - 'TRUSTED_ARNS_RESOLVER_URL', -); - // // Mempool watcher // diff --git a/src/init/resolvers.ts b/src/init/resolvers.ts index 50c67d0f..0dd0abe8 100644 --- a/src/init/resolvers.ts +++ b/src/init/resolvers.ts @@ -16,30 +16,61 @@ * along with this program. If not, see . */ import { Logger } from 'winston'; -import { StandaloneArNSResolver } from '../resolution/standalone-arns-resolver.js'; import { OnDemandArNSResolver } from '../resolution/on-demand-arns-resolver.js'; import { TrustedGatewayArNSResolver } from '../resolution/trusted-gateway-arns-resolver.js'; -import { NameResolver } from '../types.js'; +import { KVBufferStore, NameResolver } from '../types.js'; import { AoIORead } from '@ar.io/sdk'; import { CompositeArNSResolver } from '../resolution/composite-arns-resolver.js'; +import { RedisKvStore } from '../store/redis-kv-store.js'; +import { NodeKvStore } from '../store/node-kv-store.js'; -const supportedResolvers = ['on-demand', 'resolver', 'gateway'] as const; +const supportedResolvers = ['on-demand', 'gateway'] as const; export type ArNSResolverType = (typeof supportedResolvers)[number]; export const isArNSResolverType = (type: string): type is ArNSResolverType => { return supportedResolvers.includes(type as ArNSResolverType); }; +export const createArNSKvStore = ({ + log, + type, + redisUrl, + ttlSeconds, + maxKeys, +}: { + type: 'redis' | 'node' | string; + log: Logger; + redisUrl: string; + ttlSeconds: number; + maxKeys: number; +}): KVBufferStore => { + // or node cache + log.info(`Using ${type} as KVBufferStore for arns`, { + type, + redisUrl, + ttlSeconds, + maxKeys, + }); + if (type === 'redis') { + return new RedisKvStore({ + log, + redisUrl, + ttlSeconds, + }); + } + return new NodeKvStore({ ttlSeconds, maxKeys }); +}; + export const createArNSResolver = ({ log, + cache, resolutionOrder, - standaloneArnResolverUrl, trustedGatewayUrl, networkProcess, }: { log: Logger; + cache: KVBufferStore; resolutionOrder: (ArNSResolverType | string)[]; - standaloneArnResolverUrl?: string; trustedGatewayUrl?: string; networkProcess?: AoIORead; }): NameResolver => { @@ -49,13 +80,6 @@ export const createArNSResolver = ({ log, networkProcess, }), - resolver: - standaloneArnResolverUrl !== undefined - ? new StandaloneArNSResolver({ - log, - resolverUrl: standaloneArnResolverUrl, - }) - : undefined, gateway: trustedGatewayUrl !== undefined ? new TrustedGatewayArNSResolver({ @@ -82,5 +106,6 @@ export const createArNSResolver = ({ return new CompositeArNSResolver({ log, resolvers, + cache, }); }; diff --git a/src/metrics.ts b/src/metrics.ts index 18b56f26..657ca196 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -180,6 +180,21 @@ export const redisErrorCounter = new promClient.Counter({ help: 'Number of errors redis cache has received', }); +export const arnsCacheHitCounter = new promClient.Counter({ + name: 'arns_cache_hit_total', + help: 'Number of hits in the arns cache', +}); + +export const arnsCacheMissCounter = new promClient.Counter({ + name: 'arns_cache_miss_total', + help: 'Number of misses in the arns cache', +}); + +export const arnsResolutionTime = new promClient.Summary({ + name: 'arns_resolution_time', + help: 'Time it takes to resolve an arns name', +}); + // Data source metrics export const getDataErrorsTotal = new promClient.Counter({ diff --git a/src/middleware/arns.ts b/src/middleware/arns.ts index 33fbb667..2470ac7e 100644 --- a/src/middleware/arns.ts +++ b/src/middleware/arns.ts @@ -23,7 +23,7 @@ import { headerNames } from '../constants.js'; import { sendNotFound } from '../routes/data/handlers.js'; import { DATA_PATH_REGEX } from '../constants.js'; import { NameResolver } from '../types.js'; - +import * as metrics from '../metrics.js'; const EXCLUDED_SUBDOMAINS = new Set('www'); export const createArnsMiddleware = ({ @@ -53,7 +53,7 @@ export const createArnsMiddleware = ({ if ( EXCLUDED_SUBDOMAINS.has(arnsSubdomain) || // Avoid collisions with sandbox URLs by ensuring the subdomain length - // is below the mininimum length of a sandbox subdomain. Undernames are + // is below the minimum length of a sandbox subdomain. Undernames are // are an exception because they can be longer and '_' cannot appear in // base32. (arnsSubdomain.length > 48 && !arnsSubdomain.match(/_/)) @@ -67,8 +67,10 @@ export const createArnsMiddleware = ({ return; } + const start = Date.now(); const { resolvedId, ttl, processId } = await nameResolver.resolve(arnsSubdomain); + metrics.arnsResolutionTime.observe(Date.now() - start); if (resolvedId === undefined) { sendNotFound(res); return; @@ -76,6 +78,7 @@ export const createArnsMiddleware = ({ res.header(headerNames.arnsResolvedId, resolvedId); res.header(headerNames.arnsTtlSeconds, ttl.toString()); res.header(headerNames.arnsProcessId, processId); + // TODO: add a header for arns cache status res.header('Cache-Control', `public, max-age=${ttl}`); dataHandler(req, res, next); }); diff --git a/src/resolution/composite-arns-resolver.ts b/src/resolution/composite-arns-resolver.ts index e89723fa..a37d12c9 100644 --- a/src/resolution/composite-arns-resolver.ts +++ b/src/resolution/composite-arns-resolver.ts @@ -16,34 +16,75 @@ * along with this program. If not, see . */ import winston from 'winston'; -import { NameResolution, NameResolver } from '../types.js'; +import { KVBufferStore, NameResolution, NameResolver } from '../types.js'; +import * as metrics from '../metrics.js'; export class CompositeArNSResolver implements NameResolver { private log: winston.Logger; private resolvers: NameResolver[]; + private cache: KVBufferStore; constructor({ log, resolvers, + cache, }: { log: winston.Logger; resolvers: NameResolver[]; + cache: KVBufferStore; }) { - this.log = log.child({ class: 'CompositeArNSResolver' }); + this.log = log.child({ class: this.constructor.name }); this.resolvers = resolvers; + this.cache = cache; + } + + private hashKey(key: string): string { + return `arns|${key}`; } async resolve(name: string): Promise { this.log.info('Resolving name...', { name }); try { + const cachedResolutionBuffer = await this.cache.get(this.hashKey(name)); + if (cachedResolutionBuffer) { + const cachedResolution: NameResolution = JSON.parse( + cachedResolutionBuffer.toString(), + ); + if ( + cachedResolution !== undefined && + cachedResolution.resolvedAt !== undefined && + cachedResolution.ttl !== undefined && + cachedResolution.resolvedAt + cachedResolution.ttl * 1000 > Date.now() + ) { + metrics.arnsCacheHitCounter.inc(); + this.log.info('Cache hit for arns name', { name }); + return cachedResolution; + } + } + metrics.arnsCacheMissCounter.inc(); + this.log.info('Cache miss for arns name', { name }); + for (const resolver of this.resolvers) { - this.log.debug('Attempting to resolve name with resolver', { - resolver, - }); - const resolution = await resolver.resolve(name); - if (resolution.resolvedId !== undefined) { - return resolution; + try { + this.log.info('Attempting to resolve name with resolver', { + type: resolver.constructor.name, + name, + }); + const resolution = await resolver.resolve(name); + if (resolution.resolvedId !== undefined) { + const hashKey = this.hashKey(name); + const resolutionBuffer = Buffer.from(JSON.stringify(resolution)); + await this.cache.set(hashKey, resolutionBuffer); + this.log.info('Resolved name', { name, resolution }); + return resolution; + } + } catch (error: any) { + this.log.error('Error resolving name with resolver', { + resolver, + message: error.message, + stack: error.stack, + }); } } this.log.warn('Unable to resolve name against all resolvers', { name }); diff --git a/src/resolution/on-demand-arns-resolver.ts b/src/resolution/on-demand-arns-resolver.ts index 68f10b4a..7de38922 100644 --- a/src/resolution/on-demand-arns-resolver.ts +++ b/src/resolution/on-demand-arns-resolver.ts @@ -19,33 +19,44 @@ import winston from 'winston'; import { isValidDataId } from '../lib/validation.js'; import { NameResolution, NameResolver } from '../types.js'; -import { ANT, AoIORead, AOProcess, IO } from '@ar.io/sdk'; +import { ANT, AoClient, AoIORead, AOProcess, IO } from '@ar.io/sdk'; import * as config from '../config.js'; import { connect } from '@permaweb/aoconnect'; export class OnDemandArNSResolver implements NameResolver { private log: winston.Logger; private networkProcess: AoIORead; + private ao: AoClient; constructor({ log, + ao = connect({ + MU_URL: config.AO_MU_URL, + CU_URL: config.AO_CU_URL, + GRAPHQL_URL: config.AO_GRAPHQL_URL, + GATEWAY_URL: config.AO_GATEWAY_URL, + }), networkProcess = IO.init({ - processId: config.IO_PROCESS_ID, + process: new AOProcess({ + processId: config.IO_PROCESS_ID, + ao: ao, + }), }), }: { log: winston.Logger; networkProcess?: AoIORead; + ao?: AoClient; }) { this.log = log.child({ class: 'OnDemandArNSResolver', }); this.networkProcess = networkProcess; + this.ao = ao; } async resolve(name: string): Promise { this.log.info('Resolving name...', { name }); try { - const start = Date.now(); // get the base name which is the last of th array split by _ const baseName = name.split('_').pop(); if (baseName === undefined) { @@ -66,12 +77,7 @@ export class OnDemandArNSResolver implements NameResolver { const ant = ANT.init({ process: new AOProcess({ processId: processId, - ao: connect({ - MU_URL: config.AO_MU_URL, - CU_URL: config.AO_CU_URL, - GRAPHQL_URL: config.AO_GRAPHQL_URL, - GATEWAY_URL: config.AO_GATEWAY_URL, - }), + ao: this.ao, }), }); @@ -93,13 +99,6 @@ export class OnDemandArNSResolver implements NameResolver { if (!isValidDataId(resolvedId)) { throw new Error('Invalid resolved data ID'); } - - this.log.info('Resolved name', { - name, - resolvedId, - ttl, - durationMs: Date.now() - start, - }); return { name, resolvedId, diff --git a/src/store/fs-kv-store.ts b/src/store/fs-kv-store.ts index fe840887..a23689e2 100644 --- a/src/store/fs-kv-store.ts +++ b/src/store/fs-kv-store.ts @@ -66,4 +66,8 @@ export class FsKVStore implements KVBufferStore { await fse.move(tmpPath, this.bufferPath(key)); } } + + async close(): Promise { + // No-op + } } diff --git a/src/store/lmdb-kv-store.ts b/src/store/lmdb-kv-store.ts index a9ce5cd5..9542200e 100644 --- a/src/store/lmdb-kv-store.ts +++ b/src/store/lmdb-kv-store.ts @@ -48,4 +48,8 @@ export class LmdbKVStore implements KVBufferStore { async set(key: string, buffer: Buffer): Promise { await this.db.put(key, buffer); } + + async close(): Promise { + await this.db.close(); + } } diff --git a/src/store/node-kv-store.ts b/src/store/node-kv-store.ts new file mode 100644 index 00000000..191abae2 --- /dev/null +++ b/src/store/node-kv-store.ts @@ -0,0 +1,63 @@ +/** + * AR.IO Gateway + * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import NodeCache from 'node-cache'; +import { KVBufferStore } from '../types.js'; + +export class NodeKvStore implements KVBufferStore { + private cache: NodeCache; + + constructor({ + ttlSeconds, + maxKeys, + }: { + ttlSeconds: number; + maxKeys: number; + }) { + this.cache = new NodeCache({ + stdTTL: ttlSeconds, + maxKeys, + deleteOnExpire: true, + useClones: false, // cloning promises is unsafe + checkperiod: Math.min(60 * 5, ttlSeconds), + }); + } + + async get(key: string): Promise { + const value = this.cache.get(key); + if (value === undefined) { + return undefined; + } + return value as Buffer; + } + + async set(key: string, buffer: Buffer): Promise { + this.cache.set(key, buffer); + } + + async del(key: string): Promise { + this.cache.del(key); + } + + async has(key: string): Promise { + return this.cache.has(key); + } + + async close(): Promise { + this.cache.close(); + } +} diff --git a/src/store/redis-kv-store.ts b/src/store/redis-kv-store.ts index a53c21c2..a7e77fc6 100644 --- a/src/store/redis-kv-store.ts +++ b/src/store/redis-kv-store.ts @@ -17,7 +17,6 @@ */ import { RedisClientType, commandOptions, createClient } from 'redis'; import winston from 'winston'; - import * as metrics from '../metrics.js'; import { KVBufferStore } from '../types.js'; @@ -56,8 +55,6 @@ export class RedisKvStore implements KVBufferStore { }); } - // TODO: close connection to redis safely - async get(key: string): Promise { const value = await this.client.get( commandOptions({ returnBuffers: true }), @@ -82,4 +79,8 @@ export class RedisKvStore implements KVBufferStore { EX: this.ttlSeconds, }); } + + async close(): Promise { + await this.client.quit(); + } } diff --git a/src/system.ts b/src/system.ts index c2e0eadf..9abfb4d6 100644 --- a/src/system.ts +++ b/src/system.ts @@ -44,7 +44,6 @@ import { import { currentUnixTimestamp } from './lib/time.js'; import log from './log.js'; import * as metrics from './metrics.js'; -import { MemoryCacheArNSResolver } from './resolution/memory-cache-arns-resolver.js'; import { StreamingManifestPathResolver } from './resolution/streaming-manifest-path-resolver.js'; import { FsChunkDataStore } from './store/fs-chunk-data-store.js'; import { FsDataStore } from './store/fs-data-store.js'; @@ -74,7 +73,7 @@ import { TransactionRepairWorker } from './workers/transaction-repair-worker.js' import { TransactionOffsetImporter } from './workers/transaction-offset-importer.js'; import { TransactionOffsetRepairWorker } from './workers/transaction-offset-repair-worker.js'; import { WebhookEmitter } from './workers/webhook-emitter.js'; -import { createArNSResolver } from './init/resolvers.js'; +import { createArNSKvStore, createArNSResolver } from './init/resolvers.js'; import { MempoolWatcher } from './workers/mempool-watcher.js'; import { ArIODataSource } from './data/ar-io-data-source.js'; import { S3DataSource } from './data/s3-data-source.js'; @@ -551,15 +550,20 @@ export const manifestPathResolver = new StreamingManifestPathResolver({ log, }); -export const nameResolver = new MemoryCacheArNSResolver({ +export const arnsResolverCache = createArNSKvStore({ log, - resolver: createArNSResolver({ - log, - trustedGatewayUrl: config.TRUSTED_GATEWAY_URL, - standaloneArnResolverUrl: config.TRUSTED_ARNS_RESOLVER_URL, - resolutionOrder: config.ARNS_RESOLVER_PRIORITY_ORDER, - networkProcess: arIO, - }), + type: config.ARNS_CACHE_TYPE, + redisUrl: config.REDIS_CACHE_URL, + ttlSeconds: config.ARNS_CACHE_TTL_SECONDS, + maxKeys: config.ARNS_CACHE_MAX_KEYS, +}); + +export const nameResolver = createArNSResolver({ + log, + trustedGatewayUrl: config.TRUSTED_GATEWAY_URL, + resolutionOrder: config.ARNS_RESOLVER_PRIORITY_ORDER, + networkProcess: arIO, + cache: arnsResolverCache, }); const webhookEmitter = new WebhookEmitter({ @@ -616,6 +620,7 @@ export const shutdown = async (express: Server) => { eventEmitter.removeAllListeners(); arIODataSource.stopUpdatingPeers(); dataSqliteWalCleanupWorker?.stop(); + await arnsResolverCache.close(); await mempoolWatcher?.stop(); await blockImporter.stop(); await dataItemIndexer.stop(); diff --git a/src/types.d.ts b/src/types.d.ts index 708bbef8..d1181126 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -633,6 +633,7 @@ export type KVBufferStore = { set(key: string, buffer: Buffer): Promise; del(key: string): Promise; has(key: string): Promise; + close(): Promise; }; export interface SignatureSource {