diff --git a/docker-compose.yaml b/docker-compose.yaml
index df3ca1f4..cd8c4510 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -71,11 +71,11 @@ services:
- WEBHOOK_INDEX_FILTER=${WEBHOOK_INDEX_FILTER:-}
- WEBHOOK_BLOCK_FILTER=${WEBHOOK_BLOCK_FILTER:-}
- CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD=${CONTIGUOUS_DATA_CACHE_CLEANUP_THRESHOLD:-}
- - TRUSTED_ARNS_RESOLVER_URL=${TRUSTED_ARNS_RESOLVER_URL:-}
- TRUSTED_ARNS_GATEWAY_URL=${TRUSTED_ARNS_GATEWAY_URL:-https://__NAME__.arweave.net}
- ARNS_RESOLVER_PRIORITY_ORDER=${ARNS_RESOLVER_PRIORITY_ORDER:-on-demand,gateway}
- ARNS_CACHE_TTL_SECONDS=${ARNS_CACHE_TTL_SECONDS:-3600}
- ARNS_CACHE_MAX_KEYS=${ARNS_CACHE_MAX_KEYS:-10000}
+ - ARNS_CACHE_TYPE=${ARNS_CACHE_TYPE:-redis}
- ENABLE_MEMPOOL_WATCHER=${ENABLE_MEMPOOL_WATCHER:-false}
- MEMPOOL_POOLING_INTERVAL_MS=${MEMPOOL_POOLING_INTERVAL_MS:-}
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
@@ -139,28 +139,6 @@ services:
networks:
- ar-io-network
- resolver:
- image: ghcr.io/ar-io/arns-resolver:${RESOLVER_IMAGE_TAG:-7fe02ecda2027e504248d3f3716579f60b561de5}
- restart: on-failure
- ports:
- - 6000:6000
- environment:
- - PORT=6000
- - LOG_LEVEL=${LOG_LEVEL:-info}
- - IO_PROCESS_ID=${IO_PROCESS_ID:-}
- - RUN_RESOLVER=${RUN_RESOLVER:-false}
- - EVALUATION_INTERVAL_MS=${EVALUATION_INTERVAL_MS:-}
- - ARNS_CACHE_TTL_MS=${RESOLVER_CACHE_TTL_MS:-}
- - ARNS_CACHE_PATH=${ARNS_CACHE_PATH:-./data/arns}
- - AO_CU_URL=${AO_CU_URL:-}
- - AO_MU_URL=${AO_MU_URL:-}
- - AO_GATEWAY_URL=${AO_GATEWAY_URL:-}
- - AO_GRAPHQL_URL=${AO_GRAPHQL_URL:-}
- volumes:
- - ${ARNS_CACHE_PATH:-./data/arns}:/app/data/arns
- networks:
- - ar-io-network
-
litestream:
image: ghcr.io/ar-io/ar-io-litestream:${LITESTREAM_IMAGE_TAG:-latest}
build:
diff --git a/src/config.ts b/src/config.ts
index 4d3452cd..c4ddde9d 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -258,6 +258,8 @@ export const WEBHOOK_BLOCK_FILTER = createFilter(
// ArNS Resolution
//
+export const ARNS_CACHE_TYPE = env.varOrDefault('ARNS_CACHE_TYPE', 'node');
+
export const ARNS_CACHE_TTL_SECONDS = +env.varOrDefault(
'ARNS_CACHE_TTL_SECONDS',
`${60 * 60}`, // 1 hour
@@ -269,23 +271,36 @@ export const ARNS_CACHE_MAX_KEYS = +env.varOrDefault(
);
export const ARNS_RESOLVER_PRIORITY_ORDER = env
- .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'resolver,on-demand,gateway')
+ .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'on-demand,gateway')
.split(',');
-export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault(
- 'TRUSTED_ARNS_GATEWAY_URL',
- 'https://__NAME__.arweave.dev',
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS = +env.varOrDefault(
+ 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS',
+ `${5 * 1000}`, // 5 seconds
);
-// @deprecated - use ARNS_RESOLVER_PRIORITY_ORDER instead to specify the order
-// of resolvers to try if the first one is not available.
-export const TRUSTED_ARNS_RESOLVER_TYPE = env.varOrDefault(
- 'TRUSTED_ARNS_RESOLVER_TYPE',
- 'gateway',
-);
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE =
+ +env.varOrDefault(
+ 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE',
+ '50',
+ );
-export const TRUSTED_ARNS_RESOLVER_URL = env.varOrUndefined(
- 'TRUSTED_ARNS_RESOLVER_URL',
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS =
+ +env.varOrDefault(
+ 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS',
+ `${60 * 1000}`, // 1 minute
+ );
+
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS =
+ +env.varOrDefault(
+ 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS',
+ `${5 * 60 * 1000}`, // 5 minutes
+ );
+
+// TODO: support multiple gateway urls
+export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault(
+ 'TRUSTED_ARNS_GATEWAY_URL',
+ 'https://__NAME__.arweave.net',
);
//
diff --git a/src/init/resolvers.ts b/src/init/resolvers.ts
index 50c67d0f..0045c1f5 100644
--- a/src/init/resolvers.ts
+++ b/src/init/resolvers.ts
@@ -16,30 +16,61 @@
* along with this program. If not, see .
*/
import { Logger } from 'winston';
-import { StandaloneArNSResolver } from '../resolution/standalone-arns-resolver.js';
import { OnDemandArNSResolver } from '../resolution/on-demand-arns-resolver.js';
import { TrustedGatewayArNSResolver } from '../resolution/trusted-gateway-arns-resolver.js';
-import { NameResolver } from '../types.js';
+import { KVBufferStore, NameResolver } from '../types.js';
import { AoIORead } from '@ar.io/sdk';
import { CompositeArNSResolver } from '../resolution/composite-arns-resolver.js';
+import { RedisKvStore } from '../store/redis-kv-store.js';
+import { NodeKvStore } from '../store/node-kv-store.js';
+import { KvArnsStore } from '../store/kv-arns-store.js';
-const supportedResolvers = ['on-demand', 'resolver', 'gateway'] as const;
+const supportedResolvers = ['on-demand', 'gateway'] as const;
export type ArNSResolverType = (typeof supportedResolvers)[number];
export const isArNSResolverType = (type: string): type is ArNSResolverType => {
return supportedResolvers.includes(type as ArNSResolverType);
};
+export const createArNSKvStore = ({
+ log,
+ type,
+ redisUrl,
+ ttlSeconds,
+ maxKeys,
+}: {
+ type: 'redis' | 'node' | string;
+ log: Logger;
+ redisUrl: string;
+ ttlSeconds: number;
+ maxKeys: number;
+}): KVBufferStore => {
+ log.info(`Using ${type} as KVBufferStore for arns`, {
+ type,
+ redisUrl,
+ ttlSeconds,
+ maxKeys,
+ });
+ if (type === 'redis') {
+ return new RedisKvStore({
+ log,
+ redisUrl,
+ ttlSeconds,
+ });
+ }
+ return new NodeKvStore({ ttlSeconds, maxKeys });
+};
+
export const createArNSResolver = ({
log,
+ cache,
resolutionOrder,
- standaloneArnResolverUrl,
trustedGatewayUrl,
networkProcess,
}: {
log: Logger;
+ cache: KvArnsStore;
resolutionOrder: (ArNSResolverType | string)[];
- standaloneArnResolverUrl?: string;
trustedGatewayUrl?: string;
networkProcess?: AoIORead;
}): NameResolver => {
@@ -49,13 +80,6 @@ export const createArNSResolver = ({
log,
networkProcess,
}),
- resolver:
- standaloneArnResolverUrl !== undefined
- ? new StandaloneArNSResolver({
- log,
- resolverUrl: standaloneArnResolverUrl,
- })
- : undefined,
gateway:
trustedGatewayUrl !== undefined
? new TrustedGatewayArNSResolver({
@@ -82,5 +106,6 @@ export const createArNSResolver = ({
return new CompositeArNSResolver({
log,
resolvers,
+ cache,
});
};
diff --git a/src/metrics.ts b/src/metrics.ts
index 18b56f26..6baf0cff 100644
--- a/src/metrics.ts
+++ b/src/metrics.ts
@@ -180,6 +180,21 @@ export const redisErrorCounter = new promClient.Counter({
help: 'Number of errors redis cache has received',
});
+export const arnsCacheHitCounter = new promClient.Counter({
+ name: 'arns_cache_hit_total',
+ help: 'Number of hits in the arns cache',
+});
+
+export const arnsCacheMissCounter = new promClient.Counter({
+ name: 'arns_cache_miss_total',
+ help: 'Number of misses in the arns cache',
+});
+
+export const arnsResolutionTime = new promClient.Summary({
+ name: 'arns_resolution_time_ms',
+ help: 'Time in ms it takes to resolve an arns name',
+});
+
// Data source metrics
export const getDataErrorsTotal = new promClient.Counter({
diff --git a/src/middleware/arns.ts b/src/middleware/arns.ts
index 33fbb667..bf4d0949 100644
--- a/src/middleware/arns.ts
+++ b/src/middleware/arns.ts
@@ -22,10 +22,19 @@ import * as config from '../config.js';
import { headerNames } from '../constants.js';
import { sendNotFound } from '../routes/data/handlers.js';
import { DATA_PATH_REGEX } from '../constants.js';
-import { NameResolver } from '../types.js';
+import { NameResolution, NameResolver } from '../types.js';
+import * as metrics from '../metrics.js';
+import NodeCache from 'node-cache';
const EXCLUDED_SUBDOMAINS = new Set('www');
+// simple cache that stores the arns resolution promises to avoid duplicate requests to the name resolver
+const arnsRequestCache = new NodeCache({
+ stdTTL: 60, // short cache in case we forget to delete
+ checkperiod: 60,
+ useClones: false, // cloning promises is unsafe
+});
+
export const createArnsMiddleware = ({
dataHandler,
nameResolver,
@@ -53,7 +62,7 @@ export const createArnsMiddleware = ({
if (
EXCLUDED_SUBDOMAINS.has(arnsSubdomain) ||
// Avoid collisions with sandbox URLs by ensuring the subdomain length
- // is below the mininimum length of a sandbox subdomain. Undernames are
+ // is below the minimum length of a sandbox subdomain. Undernames are
// are an exception because they can be longer and '_' cannot appear in
// base32.
(arnsSubdomain.length > 48 && !arnsSubdomain.match(/_/))
@@ -67,8 +76,26 @@ export const createArnsMiddleware = ({
return;
}
+ const getArnsResolutionPromise = async (): Promise => {
+ if (arnsRequestCache.has(arnsSubdomain)) {
+ const arnsResolutionPromise =
+ arnsRequestCache.get>(arnsSubdomain);
+ if (arnsResolutionPromise) {
+ return arnsResolutionPromise;
+ }
+ }
+ const arnsResolutionPromise = nameResolver.resolve(arnsSubdomain);
+ arnsRequestCache.set(arnsSubdomain, arnsResolutionPromise);
+ return arnsResolutionPromise;
+ };
+
+ const start = Date.now();
const { resolvedId, ttl, processId } =
- await nameResolver.resolve(arnsSubdomain);
+ await getArnsResolutionPromise().finally(() => {
+ // remove from cache after resolution
+ arnsRequestCache.del(arnsSubdomain);
+ });
+ metrics.arnsResolutionTime.observe(Date.now() - start);
if (resolvedId === undefined) {
sendNotFound(res);
return;
@@ -76,6 +103,7 @@ export const createArnsMiddleware = ({
res.header(headerNames.arnsResolvedId, resolvedId);
res.header(headerNames.arnsTtlSeconds, ttl.toString());
res.header(headerNames.arnsProcessId, processId);
+ // TODO: add a header for arns cache status
res.header('Cache-Control', `public, max-age=${ttl}`);
dataHandler(req, res, next);
});
diff --git a/src/resolution/composite-arns-resolver.ts b/src/resolution/composite-arns-resolver.ts
index e89723fa..d7798fa1 100644
--- a/src/resolution/composite-arns-resolver.ts
+++ b/src/resolution/composite-arns-resolver.ts
@@ -17,56 +17,90 @@
*/
import winston from 'winston';
import { NameResolution, NameResolver } from '../types.js';
+import * as metrics from '../metrics.js';
+import { KvArnsStore } from '../store/kv-arns-store.js';
export class CompositeArNSResolver implements NameResolver {
private log: winston.Logger;
private resolvers: NameResolver[];
+ private cache: KvArnsStore;
constructor({
log,
resolvers,
+ cache,
}: {
log: winston.Logger;
resolvers: NameResolver[];
+ cache: KvArnsStore;
}) {
- this.log = log.child({ class: 'CompositeArNSResolver' });
+ this.log = log.child({ class: this.constructor.name });
this.resolvers = resolvers;
+ this.cache = cache;
}
async resolve(name: string): Promise {
this.log.info('Resolving name...', { name });
+ let resolution: NameResolution | undefined;
try {
+ const cachedResolutionBuffer = await this.cache.get(name);
+ if (cachedResolutionBuffer) {
+ const cachedResolution: NameResolution = JSON.parse(
+ cachedResolutionBuffer.toString(),
+ );
+ resolution = cachedResolution; // hold on tho this in case we need it
+ if (
+ cachedResolution !== undefined &&
+ cachedResolution.resolvedAt !== undefined &&
+ cachedResolution.ttl !== undefined &&
+ cachedResolution.resolvedAt + cachedResolution.ttl * 1000 > Date.now()
+ ) {
+ metrics.arnsCacheHitCounter.inc();
+ this.log.info('Cache hit for arns name', { name });
+ return cachedResolution;
+ }
+ }
+ metrics.arnsCacheMissCounter.inc();
+ this.log.info('Cache miss for arns name', { name });
+
for (const resolver of this.resolvers) {
- this.log.debug('Attempting to resolve name with resolver', {
- resolver,
- });
- const resolution = await resolver.resolve(name);
- if (resolution.resolvedId !== undefined) {
- return resolution;
+ try {
+ this.log.info('Attempting to resolve name with resolver', {
+ type: resolver.constructor.name,
+ name,
+ });
+ const resolution = await resolver.resolve(name);
+ if (resolution.resolvedId !== undefined) {
+ this.cache.set(name, Buffer.from(JSON.stringify(resolution)));
+ this.log.info('Resolved name', { name, resolution });
+ return resolution;
+ }
+ } catch (error: any) {
+ this.log.error('Error resolving name with resolver', {
+ resolver,
+ message: error.message,
+ stack: error.stack,
+ });
}
}
this.log.warn('Unable to resolve name against all resolvers', { name });
- return {
- name,
- resolvedId: undefined,
- resolvedAt: undefined,
- processId: undefined,
- ttl: undefined,
- };
} catch (error: any) {
this.log.error('Error resolving name:', {
name,
message: error.message,
stack: error.stack,
});
- return {
+ }
+ // return the resolution if it exists, otherwise return an empty resolution
+ return (
+ resolution ?? {
name,
resolvedId: undefined,
resolvedAt: undefined,
ttl: undefined,
processId: undefined,
- };
- }
+ }
+ );
}
}
diff --git a/src/resolution/memory-cache-arns-resolver.ts b/src/resolution/memory-cache-arns-resolver.ts
deleted file mode 100644
index 3faf193f..00000000
--- a/src/resolution/memory-cache-arns-resolver.ts
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * AR.IO Gateway
- * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-import { default as NodeCache } from 'node-cache';
-import winston from 'winston';
-
-import { NameResolution, NameResolver } from '../types.js';
-import * as config from '../config.js';
-
-export class MemoryCacheArNSResolver implements NameResolver {
- private log: winston.Logger;
- private resolver: NameResolver;
- private requestCache = new NodeCache({
- checkperiod: Math.min(60 * 5, config.ARNS_CACHE_TTL_SECONDS), // 5 minutes or less
- stdTTL: config.ARNS_CACHE_TTL_SECONDS,
- useClones: false, // cloning promises is unsafe
- maxKeys: config.ARNS_CACHE_MAX_KEYS,
- });
-
- constructor({
- log,
- resolver,
- }: {
- log: winston.Logger;
- resolver: NameResolver;
- }) {
- this.log = log.child({ class: 'MemoryCacheArNSResolver' });
- this.resolver = resolver;
- }
-
- async resolve(name: string): Promise {
- this.log.info('Resolving name...', { name });
-
- try {
- // Attempt to resolve from memory cache
- let resolutionPromise = this.requestCache.get(name);
- let resolution = (await resolutionPromise) as NameResolution | undefined;
- if (resolution) {
- const { resolvedAt, ttl } = resolution;
- if (resolvedAt !== undefined && Date.now() < resolvedAt + ttl * 1000) {
- this.log.info('Resolved name from cache', resolution);
- return resolution;
- }
- }
-
- // Fallback to resolver if cache is empty or expired
- resolutionPromise = this.resolver.resolve(name);
- try {
- this.requestCache.set(name, resolutionPromise);
- } catch (error: any) {
- this.log.warn('Unable to set cache:', {
- name,
- message: error.message,
- stack: error.stack,
- });
- }
- resolution = (await resolutionPromise) as NameResolution;
- const { resolvedAt, ttl } = resolution;
- if (resolvedAt !== undefined && Date.now() < resolvedAt + ttl) {
- this.log.info('Resolved name from resolver', resolution);
- return resolution;
- } else {
- this.log.warn('Unable to resolve name', { name });
- }
- } catch (error: any) {
- this.log.warn('Unable to resolve name:', {
- name,
- message: error.message,
- stack: error.stack,
- });
- }
-
- // Return empty resolution if unable to resolve from cache or resolver
- return {
- name,
- resolvedId: undefined,
- resolvedAt: undefined,
- ttl: undefined,
- processId: undefined,
- };
- }
-}
diff --git a/src/resolution/on-demand-arns-resolver.ts b/src/resolution/on-demand-arns-resolver.ts
index 68f10b4a..7abc2924 100644
--- a/src/resolution/on-demand-arns-resolver.ts
+++ b/src/resolution/on-demand-arns-resolver.ts
@@ -19,42 +19,76 @@ import winston from 'winston';
import { isValidDataId } from '../lib/validation.js';
import { NameResolution, NameResolver } from '../types.js';
-import { ANT, AoIORead, AOProcess, IO } from '@ar.io/sdk';
+import { ANT, AoClient, AoIORead, AOProcess, IO } from '@ar.io/sdk';
import * as config from '../config.js';
import { connect } from '@permaweb/aoconnect';
+import CircuitBreaker from 'opossum';
+import * as metrics from '../metrics.js';
export class OnDemandArNSResolver implements NameResolver {
private log: winston.Logger;
private networkProcess: AoIORead;
+ private ao: AoClient;
+ private aoCircuitBreaker: CircuitBreaker<
+ Parameters,
+ Awaited>
+ >;
constructor({
log,
+ ao = connect({
+ MU_URL: config.AO_MU_URL,
+ CU_URL: config.AO_CU_URL,
+ GRAPHQL_URL: config.AO_GRAPHQL_URL,
+ GATEWAY_URL: config.AO_GATEWAY_URL,
+ }),
networkProcess = IO.init({
- processId: config.IO_PROCESS_ID,
+ process: new AOProcess({
+ processId: config.IO_PROCESS_ID,
+ ao: ao,
+ }),
}),
+ circuitBreakerOptions = {
+ timeout: config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS,
+ errorThresholdPercentage:
+ config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE,
+ rollingCountTimeout:
+ config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS,
+ resetTimeout: config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS,
+ },
}: {
log: winston.Logger;
networkProcess?: AoIORead;
+ ao?: AoClient;
+ circuitBreakerOptions?: CircuitBreaker.Options;
}) {
this.log = log.child({
class: 'OnDemandArNSResolver',
});
this.networkProcess = networkProcess;
+ this.ao = ao;
+ this.aoCircuitBreaker = new CircuitBreaker(
+ ({ name }: { name: string }) => {
+ return this.networkProcess.getArNSRecord({ name });
+ },
+ {
+ ...circuitBreakerOptions,
+ name: 'getArNSRecord',
+ },
+ );
+ metrics.circuitBreakerMetrics.add(this.aoCircuitBreaker);
}
async resolve(name: string): Promise {
this.log.info('Resolving name...', { name });
try {
- const start = Date.now();
// get the base name which is the last of th array split by _
const baseName = name.split('_').pop();
if (baseName === undefined) {
throw new Error('Invalid name');
}
- // find that name in the network process
- const arnsRecord = await this.networkProcess.getArNSRecord({
- name: baseName,
- });
+ // find that name in the network process, using the circuit breaker if there are persistent AO issues
+ const arnsRecord = await this.aoCircuitBreaker.fire({ name: baseName });
if (arnsRecord === undefined || arnsRecord.processId === undefined) {
throw new Error('Invalid name, arns record not found');
@@ -66,12 +100,7 @@ export class OnDemandArNSResolver implements NameResolver {
const ant = ANT.init({
process: new AOProcess({
processId: processId,
- ao: connect({
- MU_URL: config.AO_MU_URL,
- CU_URL: config.AO_CU_URL,
- GRAPHQL_URL: config.AO_GRAPHQL_URL,
- GATEWAY_URL: config.AO_GATEWAY_URL,
- }),
+ ao: this.ao,
}),
});
@@ -93,13 +122,6 @@ export class OnDemandArNSResolver implements NameResolver {
if (!isValidDataId(resolvedId)) {
throw new Error('Invalid resolved data ID');
}
-
- this.log.info('Resolved name', {
- name,
- resolvedId,
- ttl,
- durationMs: Date.now() - start,
- });
return {
name,
resolvedId,
diff --git a/src/resolution/standalone-arns-resolver.ts b/src/resolution/standalone-arns-resolver.ts
deleted file mode 100644
index 7f584bed..00000000
--- a/src/resolution/standalone-arns-resolver.ts
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * AR.IO Gateway
- * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-import { default as axios } from 'axios';
-import winston from 'winston';
-
-import { isValidDataId } from '../lib/validation.js';
-import { NameResolution, NameResolver } from '../types.js';
-import { DEFAULT_ARNS_TTL_SECONDS } from './trusted-gateway-arns-resolver.js';
-
-export class StandaloneArNSResolver implements NameResolver {
- private log: winston.Logger;
- private resolverUrl: string;
-
- constructor({
- log,
- resolverUrl,
- }: {
- log: winston.Logger;
- resolverUrl: string;
- }) {
- this.log = log.child({
- class: 'StandaloneArNSResolver',
- resolverUrl: resolverUrl,
- });
- this.resolverUrl = resolverUrl;
- }
-
- async resolve(name: string): Promise {
- this.log.info('Resolving name...', { name });
- try {
- const { data } = await axios<{
- txId: string;
- ttlSeconds: number;
- processId: string;
- }>({
- method: 'GET',
- url: `/ar-io/resolver/records/${name}`,
- baseURL: this.resolverUrl,
- validateStatus: (status) => status === 200,
- });
-
- const resolvedId = data.txId;
- const ttl = data.ttlSeconds || DEFAULT_ARNS_TTL_SECONDS;
- const processId = data.processId;
-
- if (!isValidDataId(resolvedId)) {
- throw new Error('Invalid resolved data ID');
- }
-
- this.log.info('Resolved name', {
- name,
- resolvedId,
- ttl,
- });
- return {
- name,
- resolvedId,
- resolvedAt: Date.now(),
- processId: processId,
- ttl,
- };
- } catch (error: any) {
- this.log.warn('Unable to resolve name:', {
- name,
- message: error.message,
- stack: error.stack,
- });
- }
-
- return {
- name,
- resolvedId: undefined,
- resolvedAt: undefined,
- ttl: undefined,
- processId: undefined,
- };
- }
-}
diff --git a/src/store/fs-kv-store.ts b/src/store/fs-kv-store.ts
index fe840887..a23689e2 100644
--- a/src/store/fs-kv-store.ts
+++ b/src/store/fs-kv-store.ts
@@ -66,4 +66,8 @@ export class FsKVStore implements KVBufferStore {
await fse.move(tmpPath, this.bufferPath(key));
}
}
+
+ async close(): Promise {
+ // No-op
+ }
}
diff --git a/src/store/kv-arns-store.ts b/src/store/kv-arns-store.ts
new file mode 100644
index 00000000..286f4a3a
--- /dev/null
+++ b/src/store/kv-arns-store.ts
@@ -0,0 +1,51 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+import { KVBufferStore } from '../types.js';
+
+export class KvArnsStore implements KVBufferStore {
+ private kvBufferStore: KVBufferStore;
+
+ constructor({ kvBufferStore }: { kvBufferStore: KVBufferStore }) {
+ this.kvBufferStore = kvBufferStore;
+ }
+
+ private hashKey(key: string): string {
+ return `arns|${key}`;
+ }
+
+ async get(key: string): Promise {
+ return this.kvBufferStore.get(this.hashKey(key));
+ }
+
+ async set(key: string, value: Buffer): Promise {
+ return this.kvBufferStore.set(this.hashKey(key), value);
+ }
+
+ async has(key: string): Promise {
+ return this.kvBufferStore.has(this.hashKey(key));
+ }
+
+ async del(key: string): Promise {
+ return this.kvBufferStore.del(this.hashKey(key));
+ }
+
+ async close(): Promise {
+ return this.kvBufferStore.close();
+ }
+}
diff --git a/src/store/lmdb-kv-store.ts b/src/store/lmdb-kv-store.ts
index a9ce5cd5..9542200e 100644
--- a/src/store/lmdb-kv-store.ts
+++ b/src/store/lmdb-kv-store.ts
@@ -48,4 +48,8 @@ export class LmdbKVStore implements KVBufferStore {
async set(key: string, buffer: Buffer): Promise {
await this.db.put(key, buffer);
}
+
+ async close(): Promise {
+ await this.db.close();
+ }
}
diff --git a/src/store/node-kv-store.ts b/src/store/node-kv-store.ts
new file mode 100644
index 00000000..191abae2
--- /dev/null
+++ b/src/store/node-kv-store.ts
@@ -0,0 +1,63 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import NodeCache from 'node-cache';
+import { KVBufferStore } from '../types.js';
+
+export class NodeKvStore implements KVBufferStore {
+ private cache: NodeCache;
+
+ constructor({
+ ttlSeconds,
+ maxKeys,
+ }: {
+ ttlSeconds: number;
+ maxKeys: number;
+ }) {
+ this.cache = new NodeCache({
+ stdTTL: ttlSeconds,
+ maxKeys,
+ deleteOnExpire: true,
+ useClones: false, // cloning promises is unsafe
+ checkperiod: Math.min(60 * 5, ttlSeconds),
+ });
+ }
+
+ async get(key: string): Promise {
+ const value = this.cache.get(key);
+ if (value === undefined) {
+ return undefined;
+ }
+ return value as Buffer;
+ }
+
+ async set(key: string, buffer: Buffer): Promise {
+ this.cache.set(key, buffer);
+ }
+
+ async del(key: string): Promise {
+ this.cache.del(key);
+ }
+
+ async has(key: string): Promise {
+ return this.cache.has(key);
+ }
+
+ async close(): Promise {
+ this.cache.close();
+ }
+}
diff --git a/src/store/redis-kv-store.ts b/src/store/redis-kv-store.ts
index a53c21c2..714bfd0d 100644
--- a/src/store/redis-kv-store.ts
+++ b/src/store/redis-kv-store.ts
@@ -17,7 +17,6 @@
*/
import { RedisClientType, commandOptions, createClient } from 'redis';
import winston from 'winston';
-
import * as metrics from '../metrics.js';
import { KVBufferStore } from '../types.js';
@@ -44,6 +43,7 @@ export class RedisKvStore implements KVBufferStore {
this.log.error(`Redis error`, {
message: error.message,
stack: error.stack,
+ url: redisUrl,
});
metrics.redisErrorCounter.inc();
});
@@ -51,13 +51,12 @@ export class RedisKvStore implements KVBufferStore {
this.log.error(`Redis connection error`, {
message: error.message,
stack: error.stack,
+ url: redisUrl,
});
metrics.redisConnectionErrorsCounter.inc();
});
}
- // TODO: close connection to redis safely
-
async get(key: string): Promise {
const value = await this.client.get(
commandOptions({ returnBuffers: true }),
@@ -82,4 +81,8 @@ export class RedisKvStore implements KVBufferStore {
EX: this.ttlSeconds,
});
}
+
+ async close(): Promise {
+ await this.client.quit();
+ }
}
diff --git a/src/system.ts b/src/system.ts
index c2e0eadf..6ca047e7 100644
--- a/src/system.ts
+++ b/src/system.ts
@@ -44,7 +44,6 @@ import {
import { currentUnixTimestamp } from './lib/time.js';
import log from './log.js';
import * as metrics from './metrics.js';
-import { MemoryCacheArNSResolver } from './resolution/memory-cache-arns-resolver.js';
import { StreamingManifestPathResolver } from './resolution/streaming-manifest-path-resolver.js';
import { FsChunkDataStore } from './store/fs-chunk-data-store.js';
import { FsDataStore } from './store/fs-data-store.js';
@@ -74,7 +73,7 @@ import { TransactionRepairWorker } from './workers/transaction-repair-worker.js'
import { TransactionOffsetImporter } from './workers/transaction-offset-importer.js';
import { TransactionOffsetRepairWorker } from './workers/transaction-offset-repair-worker.js';
import { WebhookEmitter } from './workers/webhook-emitter.js';
-import { createArNSResolver } from './init/resolvers.js';
+import { createArNSKvStore, createArNSResolver } from './init/resolvers.js';
import { MempoolWatcher } from './workers/mempool-watcher.js';
import { ArIODataSource } from './data/ar-io-data-source.js';
import { S3DataSource } from './data/s3-data-source.js';
@@ -82,6 +81,7 @@ import { connect } from '@permaweb/aoconnect';
import { DataContentAttributeImporter } from './workers/data-content-attribute-importer.js';
import { SignatureFetcher } from './data/signature-fetcher.js';
import { SQLiteWalCleanupWorker } from './workers/sqlite-wal-cleanup-worker.js';
+import { KvArnsStore } from './store/kv-arns-store.js';
process.on('uncaughtException', (error) => {
metrics.uncaughtExceptionCounter.inc();
@@ -551,17 +551,24 @@ export const manifestPathResolver = new StreamingManifestPathResolver({
log,
});
-export const nameResolver = new MemoryCacheArNSResolver({
- log,
- resolver: createArNSResolver({
+export const arnsResolverCache = new KvArnsStore({
+ kvBufferStore: createArNSKvStore({
log,
- trustedGatewayUrl: config.TRUSTED_GATEWAY_URL,
- standaloneArnResolverUrl: config.TRUSTED_ARNS_RESOLVER_URL,
- resolutionOrder: config.ARNS_RESOLVER_PRIORITY_ORDER,
- networkProcess: arIO,
+ type: config.ARNS_CACHE_TYPE,
+ redisUrl: config.REDIS_CACHE_URL,
+ ttlSeconds: config.ARNS_CACHE_TTL_SECONDS,
+ maxKeys: config.ARNS_CACHE_MAX_KEYS,
}),
});
+export const nameResolver = createArNSResolver({
+ log,
+ trustedGatewayUrl: config.TRUSTED_ARNS_GATEWAY_URL,
+ resolutionOrder: config.ARNS_RESOLVER_PRIORITY_ORDER,
+ networkProcess: arIO,
+ cache: arnsResolverCache,
+});
+
const webhookEmitter = new WebhookEmitter({
eventEmitter,
targetServersUrls: config.WEBHOOK_TARGET_SERVERS,
@@ -616,6 +623,7 @@ export const shutdown = async (express: Server) => {
eventEmitter.removeAllListeners();
arIODataSource.stopUpdatingPeers();
dataSqliteWalCleanupWorker?.stop();
+ await arnsResolverCache.close();
await mempoolWatcher?.stop();
await blockImporter.stop();
await dataItemIndexer.stop();
diff --git a/src/types.d.ts b/src/types.d.ts
index 708bbef8..d1181126 100644
--- a/src/types.d.ts
+++ b/src/types.d.ts
@@ -633,6 +633,7 @@ export type KVBufferStore = {
set(key: string, buffer: Buffer): Promise;
del(key: string): Promise;
has(key: string): Promise;
+ close(): Promise;
};
export interface SignatureSource {