From 037417e6ada6a942497290a504a344120451ee25 Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Mon, 9 Sep 2024 16:19:02 -0600 Subject: [PATCH 01/10] feat(arns): remove resolver, add redis support for arns cache With on-demand resolution, we no longer need to support the arns-resolver. We can effectively fetch and cache arns resolutions quickly via AO and cache them locally or in redis. This replaces the default `MemoryArNSCache` with a configurable KvBufferStore that supports `redis` or a local `node-cache`. When resolving an arns name, the `CompositeArNSResolver` will check the provided cache and the TTL of the record, if it is not in the cache it will then use the available arns resolvers (on-demand and/or another gateway) to get resolution data. If an operator would like to disable caching of arns names, and always resolve to the latest they can set ARNS_CACHE_TTL_SECONDS to 0. Additionally, prometheus metrics are available for hit/miss rate for the arns cache and total resolution times. --- docker-compose.yaml | 24 +-------- src/config.ts | 18 ++----- src/init/resolvers.ts | 48 ++++++++++++----- src/metrics.ts | 15 ++++++ src/middleware/arns.ts | 7 ++- src/resolution/composite-arns-resolver.ts | 57 +++++++++++++++++--- src/resolution/on-demand-arns-resolver.ts | 31 ++++++----- src/store/fs-kv-store.ts | 4 ++ src/store/lmdb-kv-store.ts | 4 ++ src/store/node-kv-store.ts | 63 +++++++++++++++++++++++ src/store/redis-kv-store.ts | 7 +-- src/system.ts | 25 +++++---- src/types.d.ts | 1 + 13 files changed, 217 insertions(+), 87 deletions(-) create mode 100644 src/store/node-kv-store.ts diff --git a/docker-compose.yaml b/docker-compose.yaml index df3ca1f4..cd8c4510 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -71,11 +71,11 @@ services: - WEBHOOK_INDEX_FILTER=${WEBHOOK_INDEX_FILTER:-} - WEBHOOK_BLOCK_FILTER=${WEBHOOK_BLOCK_FILTER:-} - CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD=${CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD:-} - - TRUSTED_ARNS_RESOLVER_URL=${TRUSTED_ARNS_RESOLVER_URL:-} - TRUSTED_ARNS_GATEWAY_URL=${TRUSTED_ARNS_GATEWAY_URL:-https://__NAME__.arweave.net} - ARNS_RESOLVER_PRIORITY_ORDER=${ARNS_RESOLVER_PRIORITY_ORDER:-on-demand,gateway} - ARNS_CACHE_TTL_SECONDS=${ARNS_CACHE_TTL_SECONDS:-3600} - ARNS_CACHE_MAX_KEYS=${ARNS_CACHE_MAX_KEYS:-10000} + - ARNS_CACHE_TYPE=${ARNS_CACHE_TYPE:-redis} - ENABLE_MEMPOOL_WATCHER=${ENABLE_MEMPOOL_WATCHER:-false} - MEMPOOL_POOLING_INTERVAL_MS=${MEMPOOL_POOLING_INTERVAL_MS:-} - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-} @@ -139,28 +139,6 @@ services: networks: - ar-io-network - resolver: - image: ghcr.io/ar-io/arns-resolver:${RESOLVER_IMAGE_TAG:-7fe02ecda2027e504248d3f3716579f60b561de5} - restart: on-failure - ports: - - 6000:6000 - environment: - - PORT=6000 - - LOG_LEVEL=${LOG_LEVEL:-info} - - IO_PROCESS_ID=${IO_PROCESS_ID:-} - - RUN_RESOLVER=${RUN_RESOLVER:-false} - - EVALUATION_INTERVAL_MS=${EVALUATION_INTERVAL_MS:-} - - ARNS_CACHE_TTL_MS=${RESOLVER_CACHE_TTL_MS:-} - - ARNS_CACHE_PATH=${ARNS_CACHE_PATH:-./data/arns} - - AO_CU_URL=${AO_CU_URL:-} - - AO_MU_URL=${AO_MU_URL:-} - - AO_GATEWAY_URL=${AO_GATEWAY_URL:-} - - AO_GRAPHQL_URL=${AO_GRAPHQL_URL:-} - volumes: - - ${ARNS_CACHE_PATH:-./data/arns}:/app/data/arns - networks: - - ar-io-network - litestream: image: ghcr.io/ar-io/ar-io-litestream:${LITESTREAM_IMAGE_TAG:-latest} build: diff --git a/src/config.ts b/src/config.ts index 4d3452cd..495aa05d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -258,6 +258,8 @@ export const WEBHOOK_BLOCK_FILTER = createFilter( // ArNS Resolution // +export const ARNS_CACHE_TYPE = env.varOrDefault('ARNS_CACHE_TYPE', 'node'); + export const ARNS_CACHE_TTL_SECONDS = +env.varOrDefault( 'ARNS_CACHE_TTL_SECONDS', `${60 * 60}`, // 1 hour @@ -269,23 +271,13 @@ export const ARNS_CACHE_MAX_KEYS = +env.varOrDefault( ); export const ARNS_RESOLVER_PRIORITY_ORDER = env - .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'resolver,on-demand,gateway') + .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'on-demand,gateway') .split(','); +// TODO: support multiple gateway urls export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault( 'TRUSTED_ARNS_GATEWAY_URL', - 'https://__NAME__.arweave.dev', -); - -// @deprecated - use ARNS_RESOLVER_PRIORITY_ORDER instead to specify the order -// of resolvers to try if the first one is not available. -export const TRUSTED_ARNS_RESOLVER_TYPE = env.varOrDefault( - 'TRUSTED_ARNS_RESOLVER_TYPE', - 'gateway', -); - -export const TRUSTED_ARNS_RESOLVER_URL = env.varOrUndefined( - 'TRUSTED_ARNS_RESOLVER_URL', + 'https://__NAME__.arweave.net', ); // diff --git a/src/init/resolvers.ts b/src/init/resolvers.ts index 50c67d0f..79771745 100644 --- a/src/init/resolvers.ts +++ b/src/init/resolvers.ts @@ -16,30 +16,60 @@ * along with this program. If not, see . */ import { Logger } from 'winston'; -import { StandaloneArNSResolver } from '../resolution/standalone-arns-resolver.js'; import { OnDemandArNSResolver } from '../resolution/on-demand-arns-resolver.js'; import { TrustedGatewayArNSResolver } from '../resolution/trusted-gateway-arns-resolver.js'; -import { NameResolver } from '../types.js'; +import { KVBufferStore, NameResolver } from '../types.js'; import { AoIORead } from '@ar.io/sdk'; import { CompositeArNSResolver } from '../resolution/composite-arns-resolver.js'; +import { RedisKvStore } from '../store/redis-kv-store.js'; +import { NodeKvStore } from '../store/node-kv-store.js'; -const supportedResolvers = ['on-demand', 'resolver', 'gateway'] as const; +const supportedResolvers = ['on-demand', 'gateway'] as const; export type ArNSResolverType = (typeof supportedResolvers)[number]; export const isArNSResolverType = (type: string): type is ArNSResolverType => { return supportedResolvers.includes(type as ArNSResolverType); }; +export const createArNSKvStore = ({ + log, + type, + redisUrl, + ttlSeconds, + maxKeys, +}: { + type: 'redis' | 'node' | string; + log: Logger; + redisUrl: string; + ttlSeconds: number; + maxKeys: number; +}): KVBufferStore => { + log.info(`Using ${type} as KVBufferStore for arns`, { + type, + redisUrl, + ttlSeconds, + maxKeys, + }); + if (type === 'redis') { + return new RedisKvStore({ + log, + redisUrl, + ttlSeconds, + }); + } + return new NodeKvStore({ ttlSeconds, maxKeys }); +}; + export const createArNSResolver = ({ log, + cache, resolutionOrder, - standaloneArnResolverUrl, trustedGatewayUrl, networkProcess, }: { log: Logger; + cache: KVBufferStore; resolutionOrder: (ArNSResolverType | string)[]; - standaloneArnResolverUrl?: string; trustedGatewayUrl?: string; networkProcess?: AoIORead; }): NameResolver => { @@ -49,13 +79,6 @@ export const createArNSResolver = ({ log, networkProcess, }), - resolver: - standaloneArnResolverUrl !== undefined - ? new StandaloneArNSResolver({ - log, - resolverUrl: standaloneArnResolverUrl, - }) - : undefined, gateway: trustedGatewayUrl !== undefined ? new TrustedGatewayArNSResolver({ @@ -82,5 +105,6 @@ export const createArNSResolver = ({ return new CompositeArNSResolver({ log, resolvers, + cache, }); }; diff --git a/src/metrics.ts b/src/metrics.ts index 18b56f26..657ca196 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -180,6 +180,21 @@ export const redisErrorCounter = new promClient.Counter({ help: 'Number of errors redis cache has received', }); +export const arnsCacheHitCounter = new promClient.Counter({ + name: 'arns_cache_hit_total', + help: 'Number of hits in the arns cache', +}); + +export const arnsCacheMissCounter = new promClient.Counter({ + name: 'arns_cache_miss_total', + help: 'Number of misses in the arns cache', +}); + +export const arnsResolutionTime = new promClient.Summary({ + name: 'arns_resolution_time', + help: 'Time it takes to resolve an arns name', +}); + // Data source metrics export const getDataErrorsTotal = new promClient.Counter({ diff --git a/src/middleware/arns.ts b/src/middleware/arns.ts index 33fbb667..2470ac7e 100644 --- a/src/middleware/arns.ts +++ b/src/middleware/arns.ts @@ -23,7 +23,7 @@ import { headerNames } from '../constants.js'; import { sendNotFound } from '../routes/data/handlers.js'; import { DATA_PATH_REGEX } from '../constants.js'; import { NameResolver } from '../types.js'; - +import * as metrics from '../metrics.js'; const EXCLUDED_SUBDOMAINS = new Set('www'); export const createArnsMiddleware = ({ @@ -53,7 +53,7 @@ export const createArnsMiddleware = ({ if ( EXCLUDED_SUBDOMAINS.has(arnsSubdomain) || // Avoid collisions with sandbox URLs by ensuring the subdomain length - // is below the mininimum length of a sandbox subdomain. Undernames are + // is below the minimum length of a sandbox subdomain. Undernames are // are an exception because they can be longer and '_' cannot appear in // base32. (arnsSubdomain.length > 48 && !arnsSubdomain.match(/_/)) @@ -67,8 +67,10 @@ export const createArnsMiddleware = ({ return; } + const start = Date.now(); const { resolvedId, ttl, processId } = await nameResolver.resolve(arnsSubdomain); + metrics.arnsResolutionTime.observe(Date.now() - start); if (resolvedId === undefined) { sendNotFound(res); return; @@ -76,6 +78,7 @@ export const createArnsMiddleware = ({ res.header(headerNames.arnsResolvedId, resolvedId); res.header(headerNames.arnsTtlSeconds, ttl.toString()); res.header(headerNames.arnsProcessId, processId); + // TODO: add a header for arns cache status res.header('Cache-Control', `public, max-age=${ttl}`); dataHandler(req, res, next); }); diff --git a/src/resolution/composite-arns-resolver.ts b/src/resolution/composite-arns-resolver.ts index e89723fa..a37d12c9 100644 --- a/src/resolution/composite-arns-resolver.ts +++ b/src/resolution/composite-arns-resolver.ts @@ -16,34 +16,75 @@ * along with this program. If not, see . */ import winston from 'winston'; -import { NameResolution, NameResolver } from '../types.js'; +import { KVBufferStore, NameResolution, NameResolver } from '../types.js'; +import * as metrics from '../metrics.js'; export class CompositeArNSResolver implements NameResolver { private log: winston.Logger; private resolvers: NameResolver[]; + private cache: KVBufferStore; constructor({ log, resolvers, + cache, }: { log: winston.Logger; resolvers: NameResolver[]; + cache: KVBufferStore; }) { - this.log = log.child({ class: 'CompositeArNSResolver' }); + this.log = log.child({ class: this.constructor.name }); this.resolvers = resolvers; + this.cache = cache; + } + + private hashKey(key: string): string { + return `arns|${key}`; } async resolve(name: string): Promise { this.log.info('Resolving name...', { name }); try { + const cachedResolutionBuffer = await this.cache.get(this.hashKey(name)); + if (cachedResolutionBuffer) { + const cachedResolution: NameResolution = JSON.parse( + cachedResolutionBuffer.toString(), + ); + if ( + cachedResolution !== undefined && + cachedResolution.resolvedAt !== undefined && + cachedResolution.ttl !== undefined && + cachedResolution.resolvedAt + cachedResolution.ttl * 1000 > Date.now() + ) { + metrics.arnsCacheHitCounter.inc(); + this.log.info('Cache hit for arns name', { name }); + return cachedResolution; + } + } + metrics.arnsCacheMissCounter.inc(); + this.log.info('Cache miss for arns name', { name }); + for (const resolver of this.resolvers) { - this.log.debug('Attempting to resolve name with resolver', { - resolver, - }); - const resolution = await resolver.resolve(name); - if (resolution.resolvedId !== undefined) { - return resolution; + try { + this.log.info('Attempting to resolve name with resolver', { + type: resolver.constructor.name, + name, + }); + const resolution = await resolver.resolve(name); + if (resolution.resolvedId !== undefined) { + const hashKey = this.hashKey(name); + const resolutionBuffer = Buffer.from(JSON.stringify(resolution)); + await this.cache.set(hashKey, resolutionBuffer); + this.log.info('Resolved name', { name, resolution }); + return resolution; + } + } catch (error: any) { + this.log.error('Error resolving name with resolver', { + resolver, + message: error.message, + stack: error.stack, + }); } } this.log.warn('Unable to resolve name against all resolvers', { name }); diff --git a/src/resolution/on-demand-arns-resolver.ts b/src/resolution/on-demand-arns-resolver.ts index 68f10b4a..7de38922 100644 --- a/src/resolution/on-demand-arns-resolver.ts +++ b/src/resolution/on-demand-arns-resolver.ts @@ -19,33 +19,44 @@ import winston from 'winston'; import { isValidDataId } from '../lib/validation.js'; import { NameResolution, NameResolver } from '../types.js'; -import { ANT, AoIORead, AOProcess, IO } from '@ar.io/sdk'; +import { ANT, AoClient, AoIORead, AOProcess, IO } from '@ar.io/sdk'; import * as config from '../config.js'; import { connect } from '@permaweb/aoconnect'; export class OnDemandArNSResolver implements NameResolver { private log: winston.Logger; private networkProcess: AoIORead; + private ao: AoClient; constructor({ log, + ao = connect({ + MU_URL: config.AO_MU_URL, + CU_URL: config.AO_CU_URL, + GRAPHQL_URL: config.AO_GRAPHQL_URL, + GATEWAY_URL: config.AO_GATEWAY_URL, + }), networkProcess = IO.init({ - processId: config.IO_PROCESS_ID, + process: new AOProcess({ + processId: config.IO_PROCESS_ID, + ao: ao, + }), }), }: { log: winston.Logger; networkProcess?: AoIORead; + ao?: AoClient; }) { this.log = log.child({ class: 'OnDemandArNSResolver', }); this.networkProcess = networkProcess; + this.ao = ao; } async resolve(name: string): Promise { this.log.info('Resolving name...', { name }); try { - const start = Date.now(); // get the base name which is the last of th array split by _ const baseName = name.split('_').pop(); if (baseName === undefined) { @@ -66,12 +77,7 @@ export class OnDemandArNSResolver implements NameResolver { const ant = ANT.init({ process: new AOProcess({ processId: processId, - ao: connect({ - MU_URL: config.AO_MU_URL, - CU_URL: config.AO_CU_URL, - GRAPHQL_URL: config.AO_GRAPHQL_URL, - GATEWAY_URL: config.AO_GATEWAY_URL, - }), + ao: this.ao, }), }); @@ -93,13 +99,6 @@ export class OnDemandArNSResolver implements NameResolver { if (!isValidDataId(resolvedId)) { throw new Error('Invalid resolved data ID'); } - - this.log.info('Resolved name', { - name, - resolvedId, - ttl, - durationMs: Date.now() - start, - }); return { name, resolvedId, diff --git a/src/store/fs-kv-store.ts b/src/store/fs-kv-store.ts index fe840887..a23689e2 100644 --- a/src/store/fs-kv-store.ts +++ b/src/store/fs-kv-store.ts @@ -66,4 +66,8 @@ export class FsKVStore implements KVBufferStore { await fse.move(tmpPath, this.bufferPath(key)); } } + + async close(): Promise { + // No-op + } } diff --git a/src/store/lmdb-kv-store.ts b/src/store/lmdb-kv-store.ts index a9ce5cd5..9542200e 100644 --- a/src/store/lmdb-kv-store.ts +++ b/src/store/lmdb-kv-store.ts @@ -48,4 +48,8 @@ export class LmdbKVStore implements KVBufferStore { async set(key: string, buffer: Buffer): Promise { await this.db.put(key, buffer); } + + async close(): Promise { + await this.db.close(); + } } diff --git a/src/store/node-kv-store.ts b/src/store/node-kv-store.ts new file mode 100644 index 00000000..191abae2 --- /dev/null +++ b/src/store/node-kv-store.ts @@ -0,0 +1,63 @@ +/** + * AR.IO Gateway + * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import NodeCache from 'node-cache'; +import { KVBufferStore } from '../types.js'; + +export class NodeKvStore implements KVBufferStore { + private cache: NodeCache; + + constructor({ + ttlSeconds, + maxKeys, + }: { + ttlSeconds: number; + maxKeys: number; + }) { + this.cache = new NodeCache({ + stdTTL: ttlSeconds, + maxKeys, + deleteOnExpire: true, + useClones: false, // cloning promises is unsafe + checkperiod: Math.min(60 * 5, ttlSeconds), + }); + } + + async get(key: string): Promise { + const value = this.cache.get(key); + if (value === undefined) { + return undefined; + } + return value as Buffer; + } + + async set(key: string, buffer: Buffer): Promise { + this.cache.set(key, buffer); + } + + async del(key: string): Promise { + this.cache.del(key); + } + + async has(key: string): Promise { + return this.cache.has(key); + } + + async close(): Promise { + this.cache.close(); + } +} diff --git a/src/store/redis-kv-store.ts b/src/store/redis-kv-store.ts index a53c21c2..a7e77fc6 100644 --- a/src/store/redis-kv-store.ts +++ b/src/store/redis-kv-store.ts @@ -17,7 +17,6 @@ */ import { RedisClientType, commandOptions, createClient } from 'redis'; import winston from 'winston'; - import * as metrics from '../metrics.js'; import { KVBufferStore } from '../types.js'; @@ -56,8 +55,6 @@ export class RedisKvStore implements KVBufferStore { }); } - // TODO: close connection to redis safely - async get(key: string): Promise { const value = await this.client.get( commandOptions({ returnBuffers: true }), @@ -82,4 +79,8 @@ export class RedisKvStore implements KVBufferStore { EX: this.ttlSeconds, }); } + + async close(): Promise { + await this.client.quit(); + } } diff --git a/src/system.ts b/src/system.ts index c2e0eadf..9abfb4d6 100644 --- a/src/system.ts +++ b/src/system.ts @@ -44,7 +44,6 @@ import { import { currentUnixTimestamp } from './lib/time.js'; import log from './log.js'; import * as metrics from './metrics.js'; -import { MemoryCacheArNSResolver } from './resolution/memory-cache-arns-resolver.js'; import { StreamingManifestPathResolver } from './resolution/streaming-manifest-path-resolver.js'; import { FsChunkDataStore } from './store/fs-chunk-data-store.js'; import { FsDataStore } from './store/fs-data-store.js'; @@ -74,7 +73,7 @@ import { TransactionRepairWorker } from './workers/transaction-repair-worker.js' import { TransactionOffsetImporter } from './workers/transaction-offset-importer.js'; import { TransactionOffsetRepairWorker } from './workers/transaction-offset-repair-worker.js'; import { WebhookEmitter } from './workers/webhook-emitter.js'; -import { createArNSResolver } from './init/resolvers.js'; +import { createArNSKvStore, createArNSResolver } from './init/resolvers.js'; import { MempoolWatcher } from './workers/mempool-watcher.js'; import { ArIODataSource } from './data/ar-io-data-source.js'; import { S3DataSource } from './data/s3-data-source.js'; @@ -551,15 +550,20 @@ export const manifestPathResolver = new StreamingManifestPathResolver({ log, }); -export const nameResolver = new MemoryCacheArNSResolver({ +export const arnsResolverCache = createArNSKvStore({ log, - resolver: createArNSResolver({ - log, - trustedGatewayUrl: config.TRUSTED_GATEWAY_URL, - standaloneArnResolverUrl: config.TRUSTED_ARNS_RESOLVER_URL, - resolutionOrder: config.ARNS_RESOLVER_PRIORITY_ORDER, - networkProcess: arIO, - }), + type: config.ARNS_CACHE_TYPE, + redisUrl: config.REDIS_CACHE_URL, + ttlSeconds: config.ARNS_CACHE_TTL_SECONDS, + maxKeys: config.ARNS_CACHE_MAX_KEYS, +}); + +export const nameResolver = createArNSResolver({ + log, + trustedGatewayUrl: config.TRUSTED_GATEWAY_URL, + resolutionOrder: config.ARNS_RESOLVER_PRIORITY_ORDER, + networkProcess: arIO, + cache: arnsResolverCache, }); const webhookEmitter = new WebhookEmitter({ @@ -616,6 +620,7 @@ export const shutdown = async (express: Server) => { eventEmitter.removeAllListeners(); arIODataSource.stopUpdatingPeers(); dataSqliteWalCleanupWorker?.stop(); + await arnsResolverCache.close(); await mempoolWatcher?.stop(); await blockImporter.stop(); await dataItemIndexer.stop(); diff --git a/src/types.d.ts b/src/types.d.ts index 708bbef8..d1181126 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -633,6 +633,7 @@ export type KVBufferStore = { set(key: string, buffer: Buffer): Promise; del(key: string): Promise; has(key: string): Promise; + close(): Promise; }; export interface SignatureSource { From 3b09c9e21709c61a01e0067e56e09522c916350e Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Mon, 9 Sep 2024 16:40:55 -0600 Subject: [PATCH 02/10] chore(kv): create abstract `KvArnsStore` to handle the hashing keys for arns in cache This is to avoid doing it in the composite resolver --- src/init/resolvers.ts | 3 +- src/resolution/composite-arns-resolver.ts | 13 ++---- src/store/kv-arns-store.ts | 51 +++++++++++++++++++++++ src/system.ts | 15 ++++--- 4 files changed, 66 insertions(+), 16 deletions(-) create mode 100644 src/store/kv-arns-store.ts diff --git a/src/init/resolvers.ts b/src/init/resolvers.ts index 79771745..0045c1f5 100644 --- a/src/init/resolvers.ts +++ b/src/init/resolvers.ts @@ -23,6 +23,7 @@ import { AoIORead } from '@ar.io/sdk'; import { CompositeArNSResolver } from '../resolution/composite-arns-resolver.js'; import { RedisKvStore } from '../store/redis-kv-store.js'; import { NodeKvStore } from '../store/node-kv-store.js'; +import { KvArnsStore } from '../store/kv-arns-store.js'; const supportedResolvers = ['on-demand', 'gateway'] as const; export type ArNSResolverType = (typeof supportedResolvers)[number]; @@ -68,7 +69,7 @@ export const createArNSResolver = ({ networkProcess, }: { log: Logger; - cache: KVBufferStore; + cache: KvArnsStore; resolutionOrder: (ArNSResolverType | string)[]; trustedGatewayUrl?: string; networkProcess?: AoIORead; diff --git a/src/resolution/composite-arns-resolver.ts b/src/resolution/composite-arns-resolver.ts index a37d12c9..b9235f19 100644 --- a/src/resolution/composite-arns-resolver.ts +++ b/src/resolution/composite-arns-resolver.ts @@ -18,6 +18,7 @@ import winston from 'winston'; import { KVBufferStore, NameResolution, NameResolver } from '../types.js'; import * as metrics from '../metrics.js'; +import { KvArnsStore } from '../store/kv-arns-store.js'; export class CompositeArNSResolver implements NameResolver { private log: winston.Logger; @@ -31,22 +32,18 @@ export class CompositeArNSResolver implements NameResolver { }: { log: winston.Logger; resolvers: NameResolver[]; - cache: KVBufferStore; + cache: KvArnsStore; }) { this.log = log.child({ class: this.constructor.name }); this.resolvers = resolvers; this.cache = cache; } - private hashKey(key: string): string { - return `arns|${key}`; - } - async resolve(name: string): Promise { this.log.info('Resolving name...', { name }); try { - const cachedResolutionBuffer = await this.cache.get(this.hashKey(name)); + const cachedResolutionBuffer = await this.cache.get(name); if (cachedResolutionBuffer) { const cachedResolution: NameResolution = JSON.parse( cachedResolutionBuffer.toString(), @@ -73,9 +70,7 @@ export class CompositeArNSResolver implements NameResolver { }); const resolution = await resolver.resolve(name); if (resolution.resolvedId !== undefined) { - const hashKey = this.hashKey(name); - const resolutionBuffer = Buffer.from(JSON.stringify(resolution)); - await this.cache.set(hashKey, resolutionBuffer); + await this.cache.set(name, Buffer.from(JSON.stringify(resolution))); this.log.info('Resolved name', { name, resolution }); return resolution; } diff --git a/src/store/kv-arns-store.ts b/src/store/kv-arns-store.ts new file mode 100644 index 00000000..286f4a3a --- /dev/null +++ b/src/store/kv-arns-store.ts @@ -0,0 +1,51 @@ +/** + * AR.IO Gateway + * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { KVBufferStore } from '../types.js'; + +export class KvArnsStore implements KVBufferStore { + private kvBufferStore: KVBufferStore; + + constructor({ kvBufferStore }: { kvBufferStore: KVBufferStore }) { + this.kvBufferStore = kvBufferStore; + } + + private hashKey(key: string): string { + return `arns|${key}`; + } + + async get(key: string): Promise { + return this.kvBufferStore.get(this.hashKey(key)); + } + + async set(key: string, value: Buffer): Promise { + return this.kvBufferStore.set(this.hashKey(key), value); + } + + async has(key: string): Promise { + return this.kvBufferStore.has(this.hashKey(key)); + } + + async del(key: string): Promise { + return this.kvBufferStore.del(this.hashKey(key)); + } + + async close(): Promise { + return this.kvBufferStore.close(); + } +} diff --git a/src/system.ts b/src/system.ts index 9abfb4d6..fba134e5 100644 --- a/src/system.ts +++ b/src/system.ts @@ -81,6 +81,7 @@ import { connect } from '@permaweb/aoconnect'; import { DataContentAttributeImporter } from './workers/data-content-attribute-importer.js'; import { SignatureFetcher } from './data/signature-fetcher.js'; import { SQLiteWalCleanupWorker } from './workers/sqlite-wal-cleanup-worker.js'; +import { KvArnsStore } from './store/kv-arns-store.js'; process.on('uncaughtException', (error) => { metrics.uncaughtExceptionCounter.inc(); @@ -550,12 +551,14 @@ export const manifestPathResolver = new StreamingManifestPathResolver({ log, }); -export const arnsResolverCache = createArNSKvStore({ - log, - type: config.ARNS_CACHE_TYPE, - redisUrl: config.REDIS_CACHE_URL, - ttlSeconds: config.ARNS_CACHE_TTL_SECONDS, - maxKeys: config.ARNS_CACHE_MAX_KEYS, +export const arnsResolverCache = new KvArnsStore({ + kvBufferStore: createArNSKvStore({ + log, + type: config.ARNS_CACHE_TYPE, + redisUrl: config.REDIS_CACHE_URL, + ttlSeconds: config.ARNS_CACHE_TTL_SECONDS, + maxKeys: config.ARNS_CACHE_MAX_KEYS, + }), }); export const nameResolver = createArNSResolver({ From ecb7574e059283156bdbd32d9b1cd4a6ad4757ad Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Mon, 9 Sep 2024 16:48:32 -0600 Subject: [PATCH 03/10] chore(arns): remove standalone and memory cache resolvers --- src/resolution/memory-cache-arns-resolver.ts | 96 -------------------- src/resolution/standalone-arns-resolver.ts | 93 ------------------- 2 files changed, 189 deletions(-) delete mode 100644 src/resolution/memory-cache-arns-resolver.ts delete mode 100644 src/resolution/standalone-arns-resolver.ts diff --git a/src/resolution/memory-cache-arns-resolver.ts b/src/resolution/memory-cache-arns-resolver.ts deleted file mode 100644 index 3faf193f..00000000 --- a/src/resolution/memory-cache-arns-resolver.ts +++ /dev/null @@ -1,96 +0,0 @@ -/** - * AR.IO Gateway - * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { default as NodeCache } from 'node-cache'; -import winston from 'winston'; - -import { NameResolution, NameResolver } from '../types.js'; -import * as config from '../config.js'; - -export class MemoryCacheArNSResolver implements NameResolver { - private log: winston.Logger; - private resolver: NameResolver; - private requestCache = new NodeCache({ - checkperiod: Math.min(60 * 5, config.ARNS_CACHE_TTL_SECONDS), // 5 minutes or less - stdTTL: config.ARNS_CACHE_TTL_SECONDS, - useClones: false, // cloning promises is unsafe - maxKeys: config.ARNS_CACHE_MAX_KEYS, - }); - - constructor({ - log, - resolver, - }: { - log: winston.Logger; - resolver: NameResolver; - }) { - this.log = log.child({ class: 'MemoryCacheArNSResolver' }); - this.resolver = resolver; - } - - async resolve(name: string): Promise { - this.log.info('Resolving name...', { name }); - - try { - // Attempt to resolve from memory cache - let resolutionPromise = this.requestCache.get(name); - let resolution = (await resolutionPromise) as NameResolution | undefined; - if (resolution) { - const { resolvedAt, ttl } = resolution; - if (resolvedAt !== undefined && Date.now() < resolvedAt + ttl * 1000) { - this.log.info('Resolved name from cache', resolution); - return resolution; - } - } - - // Fallback to resolver if cache is empty or expired - resolutionPromise = this.resolver.resolve(name); - try { - this.requestCache.set(name, resolutionPromise); - } catch (error: any) { - this.log.warn('Unable to set cache:', { - name, - message: error.message, - stack: error.stack, - }); - } - resolution = (await resolutionPromise) as NameResolution; - const { resolvedAt, ttl } = resolution; - if (resolvedAt !== undefined && Date.now() < resolvedAt + ttl) { - this.log.info('Resolved name from resolver', resolution); - return resolution; - } else { - this.log.warn('Unable to resolve name', { name }); - } - } catch (error: any) { - this.log.warn('Unable to resolve name:', { - name, - message: error.message, - stack: error.stack, - }); - } - - // Return empty resolution if unable to resolve from cache or resolver - return { - name, - resolvedId: undefined, - resolvedAt: undefined, - ttl: undefined, - processId: undefined, - }; - } -} diff --git a/src/resolution/standalone-arns-resolver.ts b/src/resolution/standalone-arns-resolver.ts deleted file mode 100644 index 7f584bed..00000000 --- a/src/resolution/standalone-arns-resolver.ts +++ /dev/null @@ -1,93 +0,0 @@ -/** - * AR.IO Gateway - * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -import { default as axios } from 'axios'; -import winston from 'winston'; - -import { isValidDataId } from '../lib/validation.js'; -import { NameResolution, NameResolver } from '../types.js'; -import { DEFAULT_ARNS_TTL_SECONDS } from './trusted-gateway-arns-resolver.js'; - -export class StandaloneArNSResolver implements NameResolver { - private log: winston.Logger; - private resolverUrl: string; - - constructor({ - log, - resolverUrl, - }: { - log: winston.Logger; - resolverUrl: string; - }) { - this.log = log.child({ - class: 'StandaloneArNSResolver', - resolverUrl: resolverUrl, - }); - this.resolverUrl = resolverUrl; - } - - async resolve(name: string): Promise { - this.log.info('Resolving name...', { name }); - try { - const { data } = await axios<{ - txId: string; - ttlSeconds: number; - processId: string; - }>({ - method: 'GET', - url: `/ar-io/resolver/records/${name}`, - baseURL: this.resolverUrl, - validateStatus: (status) => status === 200, - }); - - const resolvedId = data.txId; - const ttl = data.ttlSeconds || DEFAULT_ARNS_TTL_SECONDS; - const processId = data.processId; - - if (!isValidDataId(resolvedId)) { - throw new Error('Invalid resolved data ID'); - } - - this.log.info('Resolved name', { - name, - resolvedId, - ttl, - }); - return { - name, - resolvedId, - resolvedAt: Date.now(), - processId: processId, - ttl, - }; - } catch (error: any) { - this.log.warn('Unable to resolve name:', { - name, - message: error.message, - stack: error.stack, - }); - } - - return { - name, - resolvedId: undefined, - resolvedAt: undefined, - ttl: undefined, - processId: undefined, - }; - } -} From 379e027c69017d94dcf10f4255293377ba3b7aa3 Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Mon, 9 Sep 2024 16:50:03 -0600 Subject: [PATCH 04/10] chore(prom): update arns resolution time metric name --- src/metrics.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/metrics.ts b/src/metrics.ts index 657ca196..6baf0cff 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -191,8 +191,8 @@ export const arnsCacheMissCounter = new promClient.Counter({ }); export const arnsResolutionTime = new promClient.Summary({ - name: 'arns_resolution_time', - help: 'Time it takes to resolve an arns name', + name: 'arns_resolution_time_ms', + help: 'Time in ms it takes to resolve an arns name', }); // Data source metrics From 0041495fd96264b989039ea0065cd145dfe82389 Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Mon, 9 Sep 2024 17:10:25 -0600 Subject: [PATCH 05/10] chore(logs): add url to redis kv store --- src/store/redis-kv-store.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/store/redis-kv-store.ts b/src/store/redis-kv-store.ts index a7e77fc6..714bfd0d 100644 --- a/src/store/redis-kv-store.ts +++ b/src/store/redis-kv-store.ts @@ -43,6 +43,7 @@ export class RedisKvStore implements KVBufferStore { this.log.error(`Redis error`, { message: error.message, stack: error.stack, + url: redisUrl, }); metrics.redisErrorCounter.inc(); }); @@ -50,6 +51,7 @@ export class RedisKvStore implements KVBufferStore { this.log.error(`Redis connection error`, { message: error.message, stack: error.stack, + url: redisUrl, }); metrics.redisConnectionErrorsCounter.inc(); }); From f1bd0cff5a6a066ff478303a2dfe8276d9032694 Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Tue, 10 Sep 2024 07:45:28 -0600 Subject: [PATCH 06/10] fix(arns): remove await to cache, fix config var to resolver --- src/resolution/composite-arns-resolver.ts | 6 +++--- src/system.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/resolution/composite-arns-resolver.ts b/src/resolution/composite-arns-resolver.ts index b9235f19..d91d57f3 100644 --- a/src/resolution/composite-arns-resolver.ts +++ b/src/resolution/composite-arns-resolver.ts @@ -16,14 +16,14 @@ * along with this program. If not, see . */ import winston from 'winston'; -import { KVBufferStore, NameResolution, NameResolver } from '../types.js'; +import { NameResolution, NameResolver } from '../types.js'; import * as metrics from '../metrics.js'; import { KvArnsStore } from '../store/kv-arns-store.js'; export class CompositeArNSResolver implements NameResolver { private log: winston.Logger; private resolvers: NameResolver[]; - private cache: KVBufferStore; + private cache: KvArnsStore; constructor({ log, @@ -70,7 +70,7 @@ export class CompositeArNSResolver implements NameResolver { }); const resolution = await resolver.resolve(name); if (resolution.resolvedId !== undefined) { - await this.cache.set(name, Buffer.from(JSON.stringify(resolution))); + this.cache.set(name, Buffer.from(JSON.stringify(resolution))); this.log.info('Resolved name', { name, resolution }); return resolution; } diff --git a/src/system.ts b/src/system.ts index fba134e5..6ca047e7 100644 --- a/src/system.ts +++ b/src/system.ts @@ -563,7 +563,7 @@ export const arnsResolverCache = new KvArnsStore({ export const nameResolver = createArNSResolver({ log, - trustedGatewayUrl: config.TRUSTED_GATEWAY_URL, + trustedGatewayUrl: config.TRUSTED_ARNS_GATEWAY_URL, resolutionOrder: config.ARNS_RESOLVER_PRIORITY_ORDER, networkProcess: arIO, cache: arnsResolverCache, From d114821fb5292bfa605999196057acf98f8d8612 Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Tue, 10 Sep 2024 07:55:05 -0600 Subject: [PATCH 07/10] feat(arns): add a promise cache for arns middleware This protects against multiple calls to `nameResolver.resolver(name)` while resolving a single name. The cache is given a short TTL and requests are removed after they are resolved. The best way to verify this behavior is trigger concurrent requests to resolve an arns name (e.g. using `hey`) and observing the logs to resolve the name only appear once for all the requests. Once the original promise is resolved, every outstanding request is resolved with the result and the name & promise are removed from the request cache. --- src/middleware/arns.ts | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/middleware/arns.ts b/src/middleware/arns.ts index 2470ac7e..bf4d0949 100644 --- a/src/middleware/arns.ts +++ b/src/middleware/arns.ts @@ -22,10 +22,19 @@ import * as config from '../config.js'; import { headerNames } from '../constants.js'; import { sendNotFound } from '../routes/data/handlers.js'; import { DATA_PATH_REGEX } from '../constants.js'; -import { NameResolver } from '../types.js'; +import { NameResolution, NameResolver } from '../types.js'; import * as metrics from '../metrics.js'; +import NodeCache from 'node-cache'; + const EXCLUDED_SUBDOMAINS = new Set('www'); +// simple cache that stores the arns resolution promises to avoid duplicate requests to the name resolver +const arnsRequestCache = new NodeCache({ + stdTTL: 60, // short cache in case we forget to delete + checkperiod: 60, + useClones: false, // cloning promises is unsafe +}); + export const createArnsMiddleware = ({ dataHandler, nameResolver, @@ -67,9 +76,25 @@ export const createArnsMiddleware = ({ return; } + const getArnsResolutionPromise = async (): Promise => { + if (arnsRequestCache.has(arnsSubdomain)) { + const arnsResolutionPromise = + arnsRequestCache.get>(arnsSubdomain); + if (arnsResolutionPromise) { + return arnsResolutionPromise; + } + } + const arnsResolutionPromise = nameResolver.resolve(arnsSubdomain); + arnsRequestCache.set(arnsSubdomain, arnsResolutionPromise); + return arnsResolutionPromise; + }; + const start = Date.now(); const { resolvedId, ttl, processId } = - await nameResolver.resolve(arnsSubdomain); + await getArnsResolutionPromise().finally(() => { + // remove from cache after resolution + arnsRequestCache.del(arnsSubdomain); + }); metrics.arnsResolutionTime.observe(Date.now() - start); if (resolvedId === undefined) { sendNotFound(res); From 410cb5fb3e0bef28ebf5c7bff69dee376b09d228 Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Tue, 10 Sep 2024 12:37:39 -0600 Subject: [PATCH 08/10] feat(arns): add circuit breaker for calling the IO network process This should help avoid slamming AO when there are intermittent issues --- src/config.ts | 23 +++++++++++++++++ src/resolution/on-demand-arns-resolver.ts | 31 ++++++++++++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index 495aa05d..af886349 100644 --- a/src/config.ts +++ b/src/config.ts @@ -274,6 +274,29 @@ export const ARNS_RESOLVER_PRIORITY_ORDER = env .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'on-demand,gateway') .split(','); +export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS = +env.varOrDefault( + 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS', + `${60 * 1000}`, // 1 minute +); + +export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE = + +env.varOrDefault( + 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE', + '50', + ); + +export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS = + +env.varOrDefault( + 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS', + `${1000 * 10}`, // 10 seconds + ); + +export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS = + +env.varOrDefault( + 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS', + `${1000 * 60}`, // 1 minute + ); + // TODO: support multiple gateway urls export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault( 'TRUSTED_ARNS_GATEWAY_URL', diff --git a/src/resolution/on-demand-arns-resolver.ts b/src/resolution/on-demand-arns-resolver.ts index 7de38922..bbf0ba2f 100644 --- a/src/resolution/on-demand-arns-resolver.ts +++ b/src/resolution/on-demand-arns-resolver.ts @@ -22,11 +22,17 @@ import { NameResolution, NameResolver } from '../types.js'; import { ANT, AoClient, AoIORead, AOProcess, IO } from '@ar.io/sdk'; import * as config from '../config.js'; import { connect } from '@permaweb/aoconnect'; +import CircuitBreaker from 'opossum'; +import * as metrics from '../metrics.js'; export class OnDemandArNSResolver implements NameResolver { private log: winston.Logger; private networkProcess: AoIORead; private ao: AoClient; + private aoCircuitBreaker: CircuitBreaker< + Parameters, + Awaited> + >; constructor({ log, @@ -42,16 +48,35 @@ export class OnDemandArNSResolver implements NameResolver { ao: ao, }), }), + circuitBreakerOptions = { + timeout: config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS, + errorThresholdPercentage: + config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE, + rollingCountTimeout: + config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS, + resetTimeout: config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS, + }, }: { log: winston.Logger; networkProcess?: AoIORead; ao?: AoClient; + circuitBreakerOptions?: CircuitBreaker.Options; }) { this.log = log.child({ class: 'OnDemandArNSResolver', }); this.networkProcess = networkProcess; this.ao = ao; + this.aoCircuitBreaker = new CircuitBreaker( + ({ name }: { name: string }) => { + return this.networkProcess.getArNSRecord({ name }); + }, + { + name: 'getArNSRecord', + ...circuitBreakerOptions, + }, + ); + metrics.circuitBreakerMetrics.add(this.aoCircuitBreaker); } async resolve(name: string): Promise { @@ -62,10 +87,8 @@ export class OnDemandArNSResolver implements NameResolver { if (baseName === undefined) { throw new Error('Invalid name'); } - // find that name in the network process - const arnsRecord = await this.networkProcess.getArNSRecord({ - name: baseName, - }); + // find that name in the network process, using the circuit breaker if there are persistent AO issues + const arnsRecord = await this.aoCircuitBreaker.fire({ name: baseName }); if (arnsRecord === undefined || arnsRecord.processId === undefined) { throw new Error('Invalid name, arns record not found'); From b8afa1ce134b4f507b2ff4dd56ff14af31496a1a Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Tue, 10 Sep 2024 13:23:08 -0600 Subject: [PATCH 09/10] fix(arns): update circuit breaker default timeouts --- src/config.ts | 6 +++--- src/resolution/on-demand-arns-resolver.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index af886349..c4ddde9d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -276,7 +276,7 @@ export const ARNS_RESOLVER_PRIORITY_ORDER = env export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS = +env.varOrDefault( 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS', - `${60 * 1000}`, // 1 minute + `${5 * 1000}`, // 5 seconds ); export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE = @@ -288,13 +288,13 @@ export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE = export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS = +env.varOrDefault( 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS', - `${1000 * 10}`, // 10 seconds + `${60 * 1000}`, // 1 minute ); export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS = +env.varOrDefault( 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS', - `${1000 * 60}`, // 1 minute + `${5 * 60 * 1000}`, // 5 minutes ); // TODO: support multiple gateway urls diff --git a/src/resolution/on-demand-arns-resolver.ts b/src/resolution/on-demand-arns-resolver.ts index bbf0ba2f..7abc2924 100644 --- a/src/resolution/on-demand-arns-resolver.ts +++ b/src/resolution/on-demand-arns-resolver.ts @@ -72,8 +72,8 @@ export class OnDemandArNSResolver implements NameResolver { return this.networkProcess.getArNSRecord({ name }); }, { - name: 'getArNSRecord', ...circuitBreakerOptions, + name: 'getArNSRecord', }, ); metrics.circuitBreakerMetrics.add(this.aoCircuitBreaker); From b9e4fe63b1e5d876ee5078577c15d709f5a8ffcd Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Tue, 10 Sep 2024 14:01:07 -0600 Subject: [PATCH 10/10] fix(arns): returned cached resolution data if failed to fetch from resolvers If we have the resolution in the cache, and we fail to fetch new data from the resolvers, returned the cached resolution data. This data will expire based on the the ARNS_CACHE_TTL_SECONDS config variable. --- src/resolution/composite-arns-resolver.ts | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/resolution/composite-arns-resolver.ts b/src/resolution/composite-arns-resolver.ts index d91d57f3..d7798fa1 100644 --- a/src/resolution/composite-arns-resolver.ts +++ b/src/resolution/composite-arns-resolver.ts @@ -41,6 +41,7 @@ export class CompositeArNSResolver implements NameResolver { async resolve(name: string): Promise { this.log.info('Resolving name...', { name }); + let resolution: NameResolution | undefined; try { const cachedResolutionBuffer = await this.cache.get(name); @@ -48,6 +49,7 @@ export class CompositeArNSResolver implements NameResolver { const cachedResolution: NameResolution = JSON.parse( cachedResolutionBuffer.toString(), ); + resolution = cachedResolution; // hold on tho this in case we need it if ( cachedResolution !== undefined && cachedResolution.resolvedAt !== undefined && @@ -83,26 +85,22 @@ export class CompositeArNSResolver implements NameResolver { } } this.log.warn('Unable to resolve name against all resolvers', { name }); - return { - name, - resolvedId: undefined, - resolvedAt: undefined, - processId: undefined, - ttl: undefined, - }; } catch (error: any) { this.log.error('Error resolving name:', { name, message: error.message, stack: error.stack, }); - return { + } + // return the resolution if it exists, otherwise return an empty resolution + return ( + resolution ?? { name, resolvedId: undefined, resolvedAt: undefined, ttl: undefined, processId: undefined, - }; - } + } + ); } }