diff --git a/src/config.ts b/src/config.ts index 495aa05d..af886349 100644 --- a/src/config.ts +++ b/src/config.ts @@ -274,6 +274,29 @@ export const ARNS_RESOLVER_PRIORITY_ORDER = env .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'on-demand,gateway') .split(','); +export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS = +env.varOrDefault( + 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS', + `${60 * 1000}`, // 1 minute +); + +export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE = + +env.varOrDefault( + 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE', + '50', + ); + +export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS = + +env.varOrDefault( + 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS', + `${1000 * 10}`, // 10 seconds + ); + +export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS = + +env.varOrDefault( + 'ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS', + `${1000 * 60}`, // 1 minute + ); + // TODO: support multiple gateway urls export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault( 'TRUSTED_ARNS_GATEWAY_URL', diff --git a/src/metrics.ts b/src/metrics.ts index 6baf0cff..8b18ebed 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -195,6 +195,11 @@ export const arnsResolutionTime = new promClient.Summary({ help: 'Time in ms it takes to resolve an arns name', }); +export const aoCircuitBreakerErrorsCounter = new promClient.Counter({ + name: 'ao_circuit_breaker_errors_total', + help: 'Count of errors in the ao circuit breaker', +}); + // Data source metrics export const getDataErrorsTotal = new promClient.Counter({ diff --git a/src/resolution/on-demand-arns-resolver.ts b/src/resolution/on-demand-arns-resolver.ts index 7de38922..bbf0ba2f 100644 --- a/src/resolution/on-demand-arns-resolver.ts +++ b/src/resolution/on-demand-arns-resolver.ts @@ -22,11 +22,17 @@ import { NameResolution, NameResolver } from '../types.js'; import { ANT, AoClient, AoIORead, AOProcess, IO } from '@ar.io/sdk'; import * as config from '../config.js'; import { connect } from '@permaweb/aoconnect'; +import CircuitBreaker from 'opossum'; +import * as metrics from '../metrics.js'; export class OnDemandArNSResolver implements NameResolver { private log: winston.Logger; private networkProcess: AoIORead; private ao: AoClient; + private aoCircuitBreaker: CircuitBreaker< + Parameters, + Awaited> + >; constructor({ log, @@ -42,16 +48,35 @@ export class OnDemandArNSResolver implements NameResolver { ao: ao, }), }), + circuitBreakerOptions = { + timeout: config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS, + errorThresholdPercentage: + config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE, + rollingCountTimeout: + config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS, + resetTimeout: config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS, + }, }: { log: winston.Logger; networkProcess?: AoIORead; ao?: AoClient; + circuitBreakerOptions?: CircuitBreaker.Options; }) { this.log = log.child({ class: 'OnDemandArNSResolver', }); this.networkProcess = networkProcess; this.ao = ao; + this.aoCircuitBreaker = new CircuitBreaker( + ({ name }: { name: string }) => { + return this.networkProcess.getArNSRecord({ name }); + }, + { + name: 'getArNSRecord', + ...circuitBreakerOptions, + }, + ); + metrics.circuitBreakerMetrics.add(this.aoCircuitBreaker); } async resolve(name: string): Promise { @@ -62,10 +87,8 @@ export class OnDemandArNSResolver implements NameResolver { if (baseName === undefined) { throw new Error('Invalid name'); } - // find that name in the network process - const arnsRecord = await this.networkProcess.getArNSRecord({ - name: baseName, - }); + // find that name in the network process, using the circuit breaker if there are persistent AO issues + const arnsRecord = await this.aoCircuitBreaker.fire({ name: baseName }); if (arnsRecord === undefined || arnsRecord.processId === undefined) { throw new Error('Invalid name, arns record not found');