From 248a2f069421bf420f06ef2fc286eba9b776e213 Mon Sep 17 00:00:00 2001
From: dtfiedler <dylan@ar.io>
Date: Tue, 10 Sep 2024 12:37:39 -0600
Subject: [PATCH] feat(arns): add circuit breaker for calling the IO network
 process

This should help avoid slamming AO when there are intermittent issues
---
 src/config.ts                             | 23 +++++++++++++++++
 src/metrics.ts                            |  5 ++++
 src/resolution/on-demand-arns-resolver.ts | 31 ++++++++++++++++++++---
 3 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/src/config.ts b/src/config.ts
index 495aa05d..af886349 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -274,6 +274,29 @@ export const ARNS_RESOLVER_PRIORITY_ORDER = env
   .varOrDefault('ARNS_RESOLVER_PRIORITY_ORDER', 'on-demand,gateway')
   .split(',');
 
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS = +env.varOrDefault(
+  'ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS',
+  `${60 * 1000}`, // 1 minute
+);
+
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE =
+  +env.varOrDefault(
+    'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE',
+    '50',
+  );
+
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS =
+  +env.varOrDefault(
+    'ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS',
+    `${1000 * 10}`, // 10 seconds
+  );
+
+export const ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS =
+  +env.varOrDefault(
+    'ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS',
+    `${1000 * 60}`, // 1 minute
+  );
+
 // TODO: support multiple gateway urls
 export const TRUSTED_ARNS_GATEWAY_URL = env.varOrDefault(
   'TRUSTED_ARNS_GATEWAY_URL',
diff --git a/src/metrics.ts b/src/metrics.ts
index 6baf0cff..8b18ebed 100644
--- a/src/metrics.ts
+++ b/src/metrics.ts
@@ -195,6 +195,11 @@ export const arnsResolutionTime = new promClient.Summary({
   help: 'Time in ms it takes to resolve an arns name',
 });
 
+export const aoCircuitBreakerErrorsCounter = new promClient.Counter({
+  name: 'ao_circuit_breaker_errors_total',
+  help: 'Count of errors in the ao circuit breaker',
+});
+
 // Data source metrics
 
 export const getDataErrorsTotal = new promClient.Counter({
diff --git a/src/resolution/on-demand-arns-resolver.ts b/src/resolution/on-demand-arns-resolver.ts
index 7de38922..bbf0ba2f 100644
--- a/src/resolution/on-demand-arns-resolver.ts
+++ b/src/resolution/on-demand-arns-resolver.ts
@@ -22,11 +22,17 @@ import { NameResolution, NameResolver } from '../types.js';
 import { ANT, AoClient, AoIORead, AOProcess, IO } from '@ar.io/sdk';
 import * as config from '../config.js';
 import { connect } from '@permaweb/aoconnect';
+import CircuitBreaker from 'opossum';
+import * as metrics from '../metrics.js';
 
 export class OnDemandArNSResolver implements NameResolver {
   private log: winston.Logger;
   private networkProcess: AoIORead;
   private ao: AoClient;
+  private aoCircuitBreaker: CircuitBreaker<
+    Parameters<AoIORead['getArNSRecord']>,
+    Awaited<ReturnType<AoIORead['getArNSRecord']>>
+  >;
 
   constructor({
     log,
@@ -42,16 +48,35 @@ export class OnDemandArNSResolver implements NameResolver {
         ao: ao,
       }),
     }),
+    circuitBreakerOptions = {
+      timeout: config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_TIMEOUT_MS,
+      errorThresholdPercentage:
+        config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_ERROR_THRESHOLD_PERCENTAGE,
+      rollingCountTimeout:
+        config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_ROLLING_COUNT_TIMEOUT_MS,
+      resetTimeout: config.ARNS_ON_DEMAND_CIRCUIT_BREAKER_RESET_TIMEOUT_MS,
+    },
   }: {
     log: winston.Logger;
     networkProcess?: AoIORead;
     ao?: AoClient;
+    circuitBreakerOptions?: CircuitBreaker.Options;
   }) {
     this.log = log.child({
       class: 'OnDemandArNSResolver',
     });
     this.networkProcess = networkProcess;
     this.ao = ao;
+    this.aoCircuitBreaker = new CircuitBreaker(
+      ({ name }: { name: string }) => {
+        return this.networkProcess.getArNSRecord({ name });
+      },
+      {
+        name: 'getArNSRecord',
+        ...circuitBreakerOptions,
+      },
+    );
+    metrics.circuitBreakerMetrics.add(this.aoCircuitBreaker);
   }
 
   async resolve(name: string): Promise<NameResolution> {
@@ -62,10 +87,8 @@ export class OnDemandArNSResolver implements NameResolver {
       if (baseName === undefined) {
         throw new Error('Invalid name');
       }
-      // find that name in the network process
-      const arnsRecord = await this.networkProcess.getArNSRecord({
-        name: baseName,
-      });
+      // find that name in the network process, using the circuit breaker if there are persistent AO issues
+      const arnsRecord = await this.aoCircuitBreaker.fire({ name: baseName });
 
       if (arnsRecord === undefined || arnsRecord.processId === undefined) {
         throw new Error('Invalid name, arns record not found');