From 3e4ddaa3919317f5e78e9694d87f39f7de9f4f17 Mon Sep 17 00:00:00 2001 From: dtfiedler Date: Wed, 4 Sep 2024 13:09:22 -0600 Subject: [PATCH] feat(redis): implement redis cache for arns name resolutions --- .eslintrc | 8 +- .github/workflows/build.yml | 2 +- docker-compose.yml | 35 ++++++ jest.config.json | 17 --- package.json | 7 +- register.js | 4 + src/cache/arns-store.ts | 69 +++++++++++ src/cache/lmdb-kv-store.test.ts | 23 +++- src/cache/lmdb-kv-store.ts | 17 ++- src/cache/redis-kv-store.ts | 80 +++++++++++++ src/config.ts | 5 + src/lib/kv-store.ts | 51 ++++++++ src/server.ts | 121 ++++++++++++------- src/service.ts | 6 - src/system.ts | 206 +++----------------------------- src/types.ts | 3 +- tsconfig.json | 3 +- yarn.lock | 127 ++++++++++++++++---- 18 files changed, 491 insertions(+), 293 deletions(-) create mode 100644 docker-compose.yml delete mode 100644 jest.config.json create mode 100644 register.js create mode 100644 src/cache/arns-store.ts create mode 100644 src/cache/redis-kv-store.ts create mode 100644 src/lib/kv-store.ts diff --git a/.eslintrc b/.eslintrc index dccffa0..375739c 100644 --- a/.eslintrc +++ b/.eslintrc @@ -1,6 +1,5 @@ { "root": true, - "ignorePatterns": ["resources/license.header.js"], "parser": "@typescript-eslint/parser", "parserOptions": { "project": "tsconfig.json" @@ -29,15 +28,18 @@ "allowAny": true } ], - "eqeqeq": 2, + "eqeqeq": ["error", "smart"], "jest-formatting/padding-around-describe-blocks": 2, "jest-formatting/padding-around-test-blocks": 2, "header/header": [2, "./resources/license.header.js"], + "mocha/max-top-level-suites": "off", + "mocha/no-exports": "off", + "mocha/no-mocha-arrows": "off", "no-console": 0, "no-return-await": 2, "no-unneeded-ternary": 2, "no-unused-vars": "off", - "prettier/prettier": 2, + "prettier/prettier": ["error", { "endOfLine": "auto" }], "unicorn/prefer-node-protocol": 2 } } diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 571043f..e307126 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - step: ['lint:check', 'build'] + step: ['lint:check', 'build', "test"] steps: - uses: actions/checkout@v4 - name: Setup yarn diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..60dc396 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +services: + resolver: + image: ghcr.io/ar-io/arns-resolver:${RESOLVER_IMAGE_TAG:-7fe02ecda2027e504248d3f3716579f60b561de5} + build: + context: . + restart: on-failure + ports: + - ${HOST_PORT:-6000}:${CONTAINER_PORT:-6000} + environment: + - PORT=${CONTAINER_PORT:-6000} + - LOG_LEVEL=${LOG_LEVEL:-info} + - IO_PROCESS_ID=${IO_PROCESS_ID:-} + - RUN_RESOLVER=${RUN_RESOLVER:-true} + - EVALUATION_INTERVAL_MS=${EVALUATION_INTERVAL_MS:-} + - ARNS_CACHE_TTL_MS=${RESOLVER_CACHE_TTL_MS:-} + - ARNS_CACHE_PATH=${ARNS_CACHE_PATH:-./data/arns} + - ARNS_CACHE_TYPE=${ARNS_CACHE_TYPE:-redis} + - REDIS_CACHE_URL=${REDIS_CACHE_URL:-redis://redis:6379} + - AO_CU_URL=${AO_CU_URL:-} + - AO_MU_URL=${AO_MU_URL:-} + - AO_GATEWAY_URL=${AO_GATEWAY_URL:-} + - AO_GRAPHQL_URL=${AO_GRAPHQL_URL:-} + volumes: + - ${ARNS_CACHE_PATH:-./data/arns}:/app/data/arns + depends_on: + - redis + + redis: + image: redis:${REDIS_IMAGE_TAG:-7} + restart: on-failure + ports: + - 6379:6379 + volumes: + - ${REDIS_DATA_PATH:-./data/redis}:/data + diff --git a/jest.config.json b/jest.config.json deleted file mode 100644 index ea2563d..0000000 --- a/jest.config.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "preset": "ts-jest", - "testEnvironment": "node", - "extensionsToTreatAsEsm": [".ts"], - "moduleNameMapper": { - "^(\\.{1,2}/.*)\\.js$": "$1" - }, - "transform": { - "^.+\\.m?[tj]sx?$": [ - "ts-jest", - { - "useESM": true - } - ] - }, - "moduleFileExtensions": ["ts", "js", "json", "node"] -} diff --git a/package.json b/package.json index 278b929..9359bbe 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "url": "https://github.com/ar-io/arns-resolver" }, "dependencies": { - "@ar.io/sdk": "^2.0.2", + "@ar.io/sdk": "^2.1.0", "@permaweb/aoconnect": "^0.0.56", "cors": "^2.8.5", "dotenv": "^16.3.1", @@ -21,6 +21,7 @@ "middleware-async": "^1.3.5", "p-limit": "^4.0.0", "prom-client": "^14.0.1", + "redis": "^4.7.0", "swagger-ui-express": "^5.0.0", "winston": "^3.7.2", "yaml": "^2.3.1" @@ -33,6 +34,7 @@ "@types/express-prometheus-middleware": "^1.2.1", "@types/jest": "^29.5.12", "@types/node": "^16.11.7", + "@types/redis": "^4.0.11", "@types/swagger-ui-express": "^4.1.3", "@typescript-eslint/eslint-plugin": "^5.26.0", "@typescript-eslint/parser": "^5.26.0", @@ -59,6 +61,7 @@ "lint:check": "eslint src", "lint:fix": "eslint --fix src", "format:check": "prettier --check .", - "format:fix": "prettier --write ." + "format:fix": "prettier --write .", + "test": "NODE_OPTIONS=\"--import=./register.js\" node --test src/**/*.test.ts" } } diff --git a/register.js b/register.js new file mode 100644 index 0000000..9e43805 --- /dev/null +++ b/register.js @@ -0,0 +1,4 @@ +import { register } from 'node:module'; +import { pathToFileURL } from 'node:url'; + +register('ts-node/esm', pathToFileURL('./')); diff --git a/src/cache/arns-store.ts b/src/cache/arns-store.ts new file mode 100644 index 0000000..bab1360 --- /dev/null +++ b/src/cache/arns-store.ts @@ -0,0 +1,69 @@ +/** + * AR.IO ArNS Resolver + * Copyright (C) 2023 Permanent Data Solutions, Inc. All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import winston from 'winston'; + +import { KVBufferStore } from '../types.js'; + +export class ArNSStore implements KVBufferStore { + private log: winston.Logger; + private prefix: string; + private kvStore: KVBufferStore; + + constructor({ + log, + kvStore, + prefix = 'ArNS', + }: { + log: winston.Logger; + kvStore: KVBufferStore; + prefix?: string; + }) { + this.log = log.child({ class: this.constructor.name }); + this.kvStore = kvStore; + this.prefix = prefix; + this.log.info('ArNSStore initialized', { + prefix, + kvStore: kvStore.constructor.name, + }); + } + + // avoid collisions with other redis keys + private hashKey(key: string): string { + return `${this.prefix}|${key}`; + } + + async get(key: string): Promise { + return this.kvStore.get(this.hashKey(key)); + } + + async set(key: string, value: Buffer, ttlSeconds?: number): Promise { + return this.kvStore.set(this.hashKey(key), value, ttlSeconds); + } + + async del(key: string): Promise { + return this.kvStore.del(this.hashKey(key)); + } + + async has(key: string): Promise { + return this.kvStore.has(this.hashKey(key)); + } + + async close(): Promise { + return this.kvStore.close(); + } +} diff --git a/src/cache/lmdb-kv-store.test.ts b/src/cache/lmdb-kv-store.test.ts index defb521..4d38dae 100644 --- a/src/cache/lmdb-kv-store.test.ts +++ b/src/cache/lmdb-kv-store.test.ts @@ -15,6 +15,9 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +import { strict as assert } from 'node:assert'; +import { describe, it } from 'node:test'; + import { LmdbKVStore } from './lmdb-kv-store.js'; describe('LmdbKVStore', () => { @@ -23,16 +26,28 @@ describe('LmdbKVStore', () => { ttlSeconds: 1, }); - it('should set and get value', async () => { + it('should set and get value with default ttl', async () => { await cache.set('test', Buffer.from('hello')); const value = await cache.get('test'); - expect(value).toEqual(Buffer.from('hello')); + assert.deepEqual(value, Buffer.from('hello')); }); - it('should remove a value once ttl has expired', async () => { + it('should remove a value once default ttl has expired ', async () => { await cache.set('expire', Buffer.from('hello')); await new Promise((resolve) => setTimeout(resolve, 1000)); const value = await cache.get('expire'); - expect(value).toBeUndefined(); + assert.strictEqual(value, undefined); + }); + + it('should override the default ttl when a ttl is provided when setting a record', async () => { + await cache.set('test', Buffer.from('hello'), 3); + // get it right away + await new Promise((resolve) => setTimeout(resolve, 2000)); + const value = await cache.get('test'); + assert.deepEqual(value, Buffer.from('hello')); + // wait for it to expire + await new Promise((resolve) => setTimeout(resolve, 3000)); + const value2 = await cache.get('test'); + assert.strictEqual(value2, undefined); }); }); diff --git a/src/cache/lmdb-kv-store.ts b/src/cache/lmdb-kv-store.ts index e4c0814..49dc47e 100644 --- a/src/cache/lmdb-kv-store.ts +++ b/src/cache/lmdb-kv-store.ts @@ -35,11 +35,14 @@ export class LmdbKVStore implements KVBufferStore { /** * Attach the TTL to the value. */ - private serialize(value: Buffer): Buffer { - if (this.ttlSeconds === undefined) return value; + private serialize( + value: Buffer, + ttlSeconds: number | undefined = this.ttlSeconds, + ): Buffer { + if (ttlSeconds === undefined) return value; const expirationTimestamp = Buffer.allocUnsafe(8); // 8 bytes for a timestamp expirationTimestamp.writeBigInt64BE( - BigInt(Date.now() + this.ttlSeconds * 1000), + BigInt(Date.now() + ttlSeconds * 1000), 0, ); return Buffer.concat([expirationTimestamp, value]); @@ -93,7 +96,11 @@ export class LmdbKVStore implements KVBufferStore { /** * Set the value in the database with the TTL. */ - async set(key: string, buffer: Buffer): Promise { - await this.db.put(key, this.serialize(buffer)); + async set(key: string, buffer: Buffer, ttlSeconds?: number): Promise { + await this.db.put(key, this.serialize(buffer, ttlSeconds)); + } + + async close(): Promise { + await this.db.close(); } } diff --git a/src/cache/redis-kv-store.ts b/src/cache/redis-kv-store.ts new file mode 100644 index 0000000..2cd597b --- /dev/null +++ b/src/cache/redis-kv-store.ts @@ -0,0 +1,80 @@ +/** + * AR.IO ArNS Resolver + * Copyright (C) 2023 Permanent Data Solutions, Inc. All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import { RedisClientType, commandOptions, createClient } from 'redis'; +import winston from 'winston'; + +import { KVBufferStore } from '../types.js'; + +export class RedisKvStore implements KVBufferStore { + private client: RedisClientType; + private log: winston.Logger; + private defaultTtlSeconds?: number; + + constructor({ log, redisUrl }: { log: winston.Logger; redisUrl: string }) { + this.log = log.child({ class: this.constructor.name }); + this.client = createClient({ + url: redisUrl, + }); + this.client.on('error', (error: any) => { + this.log.error(`Redis error`, { + message: error.message, + stack: error.stack, + }); + // TODO: add prometheus metric for redis error + }); + this.client.connect().catch((error: any) => { + this.log.error(`Redis connection error`, { + message: error.message, + stack: error.stack, + }); + // TODO: add prometheus metric for redis connection error + }); + } + + async close() { + await this.client.quit(); + } + + async get(key: string): Promise { + const value = await this.client.get( + commandOptions({ returnBuffers: true }), + key, + ); + return value ?? undefined; + } + + async has(key: string): Promise { + return (await this.client.exists(key)) === 1; + } + + async del(key: string): Promise { + if (await this.has(key)) { + await this.client.del(key); + } + } + + async set(key: string, buffer: Buffer, ttlSeconds?: number): Promise { + if (ttlSeconds !== undefined) { + await this.client.set(key, buffer, { + EX: ttlSeconds ?? this.defaultTtlSeconds, + }); + } else { + await this.client.set(key, buffer); + } + } +} diff --git a/src/config.ts b/src/config.ts index 2457e9f..5f63d45 100644 --- a/src/config.ts +++ b/src/config.ts @@ -31,6 +31,11 @@ export const ARNS_CACHE_TTL_MS = +env.varOrDefault( 'ARNS_CACHE_TTL_MS', `${1000 * 60 * 60}`, // 1 hour by default ); +export const ARNS_CACHE_TYPE = env.varOrDefault('ARNS_CACHE_TYPE', 'lmdb'); +export const REDIS_CACHE_URL = env.varOrDefault( + 'REDIS_CACHE_URL', + 'redis://localhost:6379', +); export const RUN_RESOLVER = env.varOrDefault('RUN_RESOLVER', 'true') === 'true'; export const ENABLE_OPENAPI_VALIDATION = env.varOrDefault('ENABLE_OPENAPI_VALIDATION', 'true') === 'true'; diff --git a/src/lib/kv-store.ts b/src/lib/kv-store.ts new file mode 100644 index 0000000..a1015d6 --- /dev/null +++ b/src/lib/kv-store.ts @@ -0,0 +1,51 @@ +/** + * AR.IO ArNS Resolver + * Copyright (C) 2023 Permanent Data Solutions, Inc. All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import winston from 'winston'; + +import { LmdbKVStore } from '../cache/lmdb-kv-store.js'; +import { RedisKvStore } from '../cache/redis-kv-store.js'; + +function isSupportedKvStoreType(type: string): type is 'lmdb' | 'redis' { + return type === 'lmdb' || type === 'redis'; +} + +export const createKvStore = ({ + log, + type, + path, + redisUrl, + ttlSeconds, +}: { + log: winston.Logger; + type: 'lmdb' | 'redis' | string; + path: string; + redisUrl: string; + ttlSeconds?: number; +}) => { + if (!isSupportedKvStoreType(type)) { + throw new Error(`Unknown kv store type: ${type}`); + } + switch (type) { + case 'lmdb': + return new LmdbKVStore({ dbPath: path, ttlSeconds }); + case 'redis': + return new RedisKvStore({ redisUrl, log }); + default: + throw new Error(`Unknown kv store type: ${type}`); + } +}; diff --git a/src/server.ts b/src/server.ts index b0b79dc..241c672 100644 --- a/src/server.ts +++ b/src/server.ts @@ -15,6 +15,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +import { ANT, AOProcess } from '@ar.io/sdk/node'; +import { connect } from '@permaweb/aoconnect'; import cors from 'cors'; import express from 'express'; import * as OpenApiValidator from 'express-openapi-validator'; @@ -24,13 +26,7 @@ import YAML from 'yaml'; import * as config from './config.js'; import log from './log.js'; -import { adminMiddleware } from './middleware.js'; -import { - cache, - evaluateArNSNames, - getLastEvaluatedTimestamp, - isEvaluationInProgress, -} from './system.js'; +import { cache, contract } from './system.js'; import { ArNSResolvedData } from './types.js'; // HTTP server @@ -88,62 +84,103 @@ app.get('/ar-io/resolver/healthcheck', async (_req, res) => { app.get('/ar-io/resolver/info', (_req, res) => { res.status(200).send({ processId: config.IO_PROCESS_ID, - lastEvaluationTimestamp: getLastEvaluatedTimestamp(), }); }); -app.post('/ar-io/resolver/admin/evaluate', adminMiddleware, (_req, res) => { - // TODO: we could support post request to trigger evaluation for specific names rather than re-evaluate everything - if (isEvaluationInProgress()) { - res.status(202).send({ - message: 'Evaluation in progress', - }); - } else { - log.info('Evaluation triggered by request', { - processId: config.IO_PROCESS_ID, - }); - evaluateArNSNames(); // don't await - res.status(200).send({ - message: 'Evaluation triggered', - }); - } -}); - app.get('/ar-io/resolver/records/:name', async (req, res) => { + const arnsName = req.params.name; + + // THIS IS ESSENTIALLY A READ THROUGH CACHE USING REDIS - TODO: could replace this with a resolver interface with read through logic try { - // TODO: use barrier synchronization to prevent multiple requests for the same record - log.debug('Checking cache for record', { name: req.params.name }); - const resolvedRecordData = await cache.get(req.params.name); + const logger = log.child({ arnsName }); + logger.debug('Checking cache for record...'); + + let resolvedRecordData: ArNSResolvedData | undefined; + const cachedNameResolution = await cache.get(arnsName); + if (cachedNameResolution) { + logger.debug('Found cached arns name resolution'); + resolvedRecordData = JSON.parse(cachedNameResolution.toString()); + } else { + logger.debug('Cache miss for arns name'); + const apexName = arnsName.split('_').slice(-1)[0]; + const record = await contract.getArNSRecord({ name: apexName }); + if (!record) { + res.status(404).json({ + error: 'Record not found', + }); + return; + } + + // get the ant id and use that to get the record from the cache + const antId = record.processId; + const ant = ANT.init({ + process: new AOProcess({ + processId: antId, + ao: connect({ + MU_URL: config.AO_MU_URL, + CU_URL: config.AO_CU_URL, + GRAPHQL_URL: config.AO_GRAPHQL_URL, + GATEWAY_URL: config.AO_GATEWAY_URL, + }), + }), + }); + const undername = arnsName.split('_').slice(0, -1).join('_') || '@'; + const antRecord = await ant.getRecord({ undername }); + if (!antRecord) { + res.status(404).json({ + error: 'Record not found', + }); + return; + } + const owner = await ant.getOwner(); + resolvedRecordData = { + ttlSeconds: antRecord.ttlSeconds, + txId: antRecord.transactionId, + processId: antId, + type: record.type, + owner, + }; + + const resolvedRecordBuffer = Buffer.from( + JSON.stringify(resolvedRecordData), + ); + + // cache the record in the cache + await cache.set( + arnsName, + resolvedRecordBuffer, + resolvedRecordData.ttlSeconds, + ); + } + if (!resolvedRecordData) { res.status(404).json({ error: 'Record not found', }); return; } - const recordData: ArNSResolvedData = JSON.parse( - resolvedRecordData.toString(), - ); - log.debug('Successfully fetched record from cache', { - name: req.params.name, - txId: recordData.txId, - ttlSeconds: recordData.ttlSeconds, + + logger.debug('Successfully fetched record from cache', { + name: arnsName, + txId: resolvedRecordData.txId, + ttlSeconds: resolvedRecordData.ttlSeconds, }); res .status(200) .set({ - 'Cache-Control': `public, max-age=${recordData.ttlSeconds}`, + 'Cache-Control': `public, max-age=${resolvedRecordData.ttlSeconds}`, 'Content-Type': 'application/json', - 'X-ArNS-Resolved-Id': recordData.txId, - 'X-ArNS-Ttl-Seconds': recordData.ttlSeconds, - 'X-ArNS-Process-Id': recordData.processId, + 'X-ArNS-Resolved-Id': resolvedRecordData.txId, + 'X-ArNS-Ttl-Seconds': resolvedRecordData.ttlSeconds, + 'X-ArNS-Process-Id': resolvedRecordData.processId, }) .json({ - ...recordData, - name: req.params.name, + ...resolvedRecordData, + name: arnsName, }); } catch (err: any) { log.error('Failed to get record', { - name: req.params.name, + name: arnsName, message: err?.message, stack: err?.stack, }); diff --git a/src/service.ts b/src/service.ts index d51e821..eab44ae 100644 --- a/src/service.ts +++ b/src/service.ts @@ -18,15 +18,9 @@ import * as config from './config.js'; import log from './log.js'; import { app } from './server.js'; -import { evaluateArNSNames } from './system.js'; if (config.RUN_RESOLVER) { - // set the evaluation to run at the configured interval - setInterval(evaluateArNSNames, config.EVALUATION_INTERVAL_MS); - app.listen(config.PORT, () => { log.info(`Listening on port ${config.PORT}`); }); - - evaluateArNSNames(); } diff --git a/src/system.ts b/src/system.ts index a60f7ac..7ae0855 100644 --- a/src/system.ts +++ b/src/system.ts @@ -15,28 +15,14 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -import { - ANT, - ANTRecord, - AOProcess, - AoIORead, - IO, - ProcessId, - fetchAllArNSRecords, - isLeasedArNSRecord, -} from '@ar.io/sdk/node'; +import { AOProcess, AoIORead, IO } from '@ar.io/sdk/node'; import { connect } from '@permaweb/aoconnect'; -import pLimit from 'p-limit'; -import { LmdbKVStore } from './cache/lmdb-kv-store.js'; +import { ArNSStore } from './cache/arns-store.js'; import * as config from './config.js'; +import { createKvStore } from './lib/kv-store.js'; import log from './log.js'; -import { ArNSResolvedData } from './types.js'; -let lastEvaluationTimestamp: number | undefined; -let evaluationInProgress = false; -export const getLastEvaluatedTimestamp = () => lastEvaluationTimestamp; -export const isEvaluationInProgress = () => evaluationInProgress; export const contract: AoIORead = IO.init({ process: new AOProcess({ processId: config.IO_PROCESS_ID, @@ -50,175 +36,16 @@ export const contract: AoIORead = IO.init({ }), }); -// TODO: this could be done using any KV store - or in memory. For now, we are using LMDB for persistence. -export const cache = new LmdbKVStore({ - dbPath: config.ARNS_CACHE_PATH, - ttlSeconds: config.ARNS_CACHE_TTL_MS / 1000, +export const cache = new ArNSStore({ + log, + kvStore: createKvStore({ + log, + type: config.ARNS_CACHE_TYPE, + path: config.ARNS_CACHE_PATH, + redisUrl: config.REDIS_CACHE_URL, + }), }); -const parallelLimit = pLimit(100); - -export async function evaluateArNSNames() { - if (evaluationInProgress) { - log.debug('Evaluation in progress', { - processId: config.IO_PROCESS_ID, - }); - return; - } - - try { - log.info('Evaluating arns names against ArNS registry', { - processId: config.IO_PROCESS_ID, - }); - // prevent duplicate evaluations - evaluationInProgress = true; - - // monitor the time it takes to evaluate the names - const startTime = Date.now(); - const apexRecords = await fetchAllArNSRecords({ - contract, - }); - - log.info('Retrieved apex records:', { - count: Object.keys(apexRecords).length, - }); - - // get all the unique process ids on the contract - const processIds: Set = new Set( - Object.values(apexRecords) - .map((record) => record.processId) - .filter((id) => id !== undefined), - ); - - log.debug('Identified unique process ids assigned to records:', { - apexRecordCount: Object.keys(apexRecords).length, - processCount: processIds.size, - }); - - // create a map of the contract records and use concurrency to fetch their records - const processRecordMap: Record< - ProcessId, - { owner: string | undefined; records: Record } - > = {}; - await Promise.all( - [...processIds].map((processId: string) => { - return parallelLimit(async () => { - const antContract = ANT.init({ - processId, - }); - const antRecords = await antContract - .getRecords() - .catch((err: any) => { - log.debug('Failed to get records for contract', { - processId, - error: err?.message, - stack: err?.stack, - }); - return {}; - }); - - if (Object.keys(antRecords).length) { - processRecordMap[processId] = { - owner: await antContract.getOwner().catch((err: any) => { - log.debug('Failed to get owner for contract', { - processId, - error: err?.message, - stack: err?.stack, - }); - return undefined; - }), - records: antRecords, - }; - } - }); - }), - ); - - log.info('Retrieved unique process ids assigned to records:', { - processCount: Object.keys(processRecordMap).length, - apexRecordCount: Object.keys(apexRecords).length, - }); - - // filter out any records associated with an invalid contract - const validArNSRecords = Object.entries(apexRecords).filter( - ([_, record]) => record.processId in processRecordMap, - ); - - let successfulEvaluationCount = 0; - - const insertPromises = []; - - // now go through all the record names and assign them to the resolved tx ids - for (const [apexName, apexRecordData] of validArNSRecords) { - const antData = processRecordMap[apexRecordData.processId]; - // TODO: current complexity is O(n^2) - we can do better by flattening records above before this loop - for (const [undername, antRecordData] of Object.entries( - antData.records, - )) { - const resolvedRecordObj: ArNSResolvedData = { - ttlSeconds: antRecordData.ttlSeconds, - txId: antRecordData.transactionId, - processId: apexRecordData.processId, - type: apexRecordData.type, - owner: antData.owner, - ...(isLeasedArNSRecord(apexRecordData) - ? { - endTimestamp: apexRecordData.endTimestamp, - } - : {}), - }; - const resolvedRecordBuffer = Buffer.from( - JSON.stringify(resolvedRecordObj), - ); - const cacheKey = - undername === '@' ? apexName : `${undername}_${apexName}`; - log.debug('Inserting resolved record data into cache', { - apexName, - undername, - resolvedName: cacheKey, - processId: apexRecordData.processId, - txId: antRecordData.transactionId, - }); - // all inserts will get a ttl based on the cache configuration - const promise = cache - .set(cacheKey, resolvedRecordBuffer) - .catch((err) => { - log.error('Failed to set record in cache', { - cacheKey, - error: err?.message, - stack: err?.stack, - }); - }); - insertPromises.push(promise); - } - } - // use pLimit to prevent overwhelming cache - await Promise.all( - insertPromises.map((promise) => - parallelLimit(() => promise.then(() => successfulEvaluationCount++)), - ), - ); - log.info('Finished evaluating arns names', { - durationMs: Date.now() - startTime, - apexRecordCount: Object.keys(apexRecords).length, - evaluatedRecordCount: successfulEvaluationCount, - evaluatedProcessCount: Object.keys(processRecordMap).length, - failedProcessCount: - processIds.size - Object.keys(processRecordMap).length, - }); - lastEvaluationTimestamp = Date.now(); - } catch (err: any) { - log.error('Failed to evaluate arns names', { - error: err?.message, - stack: err?.stack, - }); - } finally { - evaluationInProgress = false; - } - - return; -} - // Exception Handlers process.on('uncaughtException', (error: any) => { @@ -228,12 +55,17 @@ process.on('uncaughtException', (error: any) => { }); }); -process.on('SIGTERM', () => { +process.on('SIGTERM', async () => { log.info('SIGTERM received, exiting...'); process.exit(0); }); -process.on('SIGINT', () => { +process.on('SIGINT', async () => { log.info('SIGINT received, exiting...'); - process.exit(0); + await shutdown(); }); + +export const shutdown = async () => { + await cache.close(); + process.exit(0); +}; diff --git a/src/types.ts b/src/types.ts index b58c968..20ebabd 100644 --- a/src/types.ts +++ b/src/types.ts @@ -18,9 +18,10 @@ export type RecordTxId = string; export type KVBufferStore = { get(key: string): Promise; - set(key: string, buffer: Buffer): Promise; + set(key: string, buffer: Buffer, ttlSeconds?: number): Promise; del(key: string): Promise; has(key: string): Promise; + close(): Promise; }; export type ArNSResolvedData = { ttlSeconds: number; diff --git a/tsconfig.json b/tsconfig.json index 228881f..bedd229 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,7 +20,8 @@ "skipLibCheck": true }, "ts-node": { - "swc": true + "swc": true, + "esm": true }, "include": ["src", "tests"] } diff --git a/yarn.lock b/yarn.lock index 4282a2d..445890d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -33,12 +33,12 @@ call-me-maybe "^1.0.1" js-yaml "^4.1.0" -"@ar.io/sdk@^2.0.2": - version "2.0.2" - resolved "https://registry.yarnpkg.com/@ar.io/sdk/-/sdk-2.0.2.tgz#64aa7c4e6b30bbe36d7cc7d450f32d100ee731b3" - integrity sha512-DGSHo9Bf90Pe2m4lBEJPFuUageAaoO+nA42nhGJ2vCn6dKVM/y3aSqCJ+H5enHjidx0H1MmsgysiBQGEBElMHg== +"@ar.io/sdk@^2.1.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@ar.io/sdk/-/sdk-2.1.0.tgz#3d33fd9f7db6b0e66c0abcd1aadf593b2f4be850" + integrity sha512-bJBzAOhiMGHJIX6YtJFK2qEgFabtF73669UDXHhuj+M7lUYAUINzqC4ZkBB3jCJMjV7nwHboNQqs/c4bRs8mjQ== dependencies: - "@permaweb/aoconnect" "^0.0.55" + "@permaweb/aoconnect" "^0.0.57" arbundles "0.11.0" arweave "1.14.4" axios "1.7.2" @@ -1284,19 +1284,14 @@ ramda "^0.30.0" zod "^3.23.5" -"@permaweb/aoconnect@^0.0.55": - version "0.0.55" - resolved "https://registry.yarnpkg.com/@permaweb/aoconnect/-/aoconnect-0.0.55.tgz#d856a078d3702154ac58541d09478d25ed3acf2c" - integrity sha512-W2GtLZedVseuDkCKk4CmM9SFmi0DdrMKqvhMBm9xo65z+Mzr/t1TEjMJKRNzEA2qh5IdwM43sWJ5fmbBYLg6TQ== +"@permaweb/ao-scheduler-utils@~0.0.20": + version "0.0.24" + resolved "https://registry.yarnpkg.com/@permaweb/ao-scheduler-utils/-/ao-scheduler-utils-0.0.24.tgz#a7d2c1e09f9b6ea5d45127fa395bbbcef6688452" + integrity sha512-G6109Nz8+dQFPuG7mV8mz66kLVA+gl2uTSqU7qpaRwfujrWi6obM94CpmvyvAnrLo3dB29EYiuv7+KOKcns8ig== dependencies: - "@permaweb/ao-scheduler-utils" "~0.0.16" - buffer "^6.0.3" - debug "^4.3.4" - hyper-async "^1.1.2" - mnemonist "^0.39.8" - ramda "^0.29.1" - warp-arbundles "^1.0.4" - zod "^3.22.4" + lru-cache "^10.2.2" + ramda "^0.30.0" + zod "^3.23.5" "@permaweb/aoconnect@^0.0.56": version "0.0.56" @@ -1312,6 +1307,20 @@ warp-arbundles "^1.0.4" zod "^3.22.4" +"@permaweb/aoconnect@^0.0.57": + version "0.0.57" + resolved "https://registry.yarnpkg.com/@permaweb/aoconnect/-/aoconnect-0.0.57.tgz#dd779563e1b994e78509251b74df64dc89ea62ea" + integrity sha512-l1+47cZuQ8pOIMOdRXymcegCmefXjqR8Bc2MY6jIzWv9old/tG6mfCue2W1QviGyhjP3zEVQgr7YofkY2lq35Q== + dependencies: + "@permaweb/ao-scheduler-utils" "~0.0.20" + buffer "^6.0.3" + debug "^4.3.5" + hyper-async "^1.1.2" + mnemonist "^0.39.8" + ramda "^0.30.1" + warp-arbundles "^1.0.4" + zod "^3.23.8" + "@randlabs/communication-bridge@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@randlabs/communication-bridge/-/communication-bridge-1.0.1.tgz#d1ecfc29157afcbb0ca2d73122d67905eecb5bf3" @@ -1324,6 +1333,40 @@ dependencies: "@randlabs/communication-bridge" "1.0.1" +"@redis/bloom@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" + integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== + +"@redis/client@1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.6.0.tgz#dcf4ae1319763db6fdddd6de7f0af68a352c30ea" + integrity sha512-aR0uffYI700OEEH4gYnitAnv3vzVGXCFvYfdpu/CJKvk4pHfLPEy/JSZyrpQ+15WhXe1yJRXLtfQ84s4mEXnPg== + dependencies: + cluster-key-slot "1.1.2" + generic-pool "3.9.0" + yallist "4.0.0" + +"@redis/graph@1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.1.tgz#8c10df2df7f7d02741866751764031a957a170ea" + integrity sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw== + +"@redis/json@1.0.7": + version "1.0.7" + resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.7.tgz#016257fcd933c4cbcb9c49cde8a0961375c6893b" + integrity sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ== + +"@redis/search@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.2.0.tgz#50976fd3f31168f585666f7922dde111c74567b8" + integrity sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw== + +"@redis/time-series@1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.1.0.tgz#cba454c05ec201bd5547aaf55286d44682ac8eb5" + integrity sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g== + "@sinclair/typebox@^0.27.8": version "0.27.8" resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e" @@ -1626,6 +1669,13 @@ resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw== +"@types/redis@^4.0.11": + version "4.0.11" + resolved "https://registry.yarnpkg.com/@types/redis/-/redis-4.0.11.tgz#0bb4c11ac9900a21ad40d2a6768ec6aaf651c0e1" + integrity sha512-bI+gth8La8Wg/QCR1+V1fhrL9+LZUSWfcqpOj2Kc80ZQ4ffbdL173vQd5wovmoV9i071FU9oP2g6etLuEwb6Rg== + dependencies: + redis "*" + "@types/semver@^7.3.12": version "7.5.0" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.0.tgz#591c1ce3a702c45ee15f47a42ade72c2fd78978a" @@ -2432,6 +2482,11 @@ cliui@^8.0.1: strip-ansi "^6.0.1" wrap-ansi "^7.0.0" +cluster-key-slot@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -2622,6 +2677,13 @@ debug@^4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4: dependencies: ms "2.1.2" +debug@^4.3.5: + version "4.3.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.6.tgz#2ab2c38fbaffebf8aa95fdfe6d88438c7a13c52b" + integrity sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg== + dependencies: + ms "2.1.2" + dedent@^1.0.0: version "1.5.1" resolved "https://registry.yarnpkg.com/dedent/-/dedent-1.5.1.tgz#4f3fc94c8b711e9bb2800d185cd6ad20f2a90aff" @@ -3278,6 +3340,11 @@ gc-stats@^1.4.0: nan "^2.13.2" node-pre-gyp "^0.13.0" +generic-pool@3.9.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" + integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== + gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -5008,7 +5075,7 @@ ramda@^0.29.1: resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.29.1.tgz#408a6165b9555b7ba2fc62555804b6c5a2eca196" integrity sha512-OfxIeWzd4xdUNxlWhgFazxsA/nl3mS4/jGZI5n00uWOoSSFRhC1b6gl6xvmzUamgmqELraWp0J/qqVlXYPDPyA== -ramda@^0.30.0: +ramda@^0.30.0, ramda@^0.30.1: version "0.30.1" resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.30.1.tgz#7108ac95673062b060025052cd5143ae8fc605bf" integrity sha512-tEF5I22zJnuclswcZMc8bDIrwRHRzf+NqVEmqg50ShAZMP7MWeR/RGDthfM/p+BlqvF2fXAzpn8i+SJcYD3alw== @@ -5091,6 +5158,18 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +redis@*, redis@^4.7.0: + version "4.7.0" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.7.0.tgz#b401787514d25dd0cfc22406d767937ba3be55d6" + integrity sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ== + dependencies: + "@redis/bloom" "1.2.0" + "@redis/client" "1.6.0" + "@redis/graph" "1.1.1" + "@redis/json" "1.0.7" + "@redis/search" "1.2.0" + "@redis/time-series" "1.1.0" + regexp-tree@^0.1.24, regexp-tree@~0.1.1: version "0.1.27" resolved "https://registry.yarnpkg.com/regexp-tree/-/regexp-tree-0.1.27.tgz#2198f0ef54518ffa743fe74d983b56ffd631b6cd" @@ -5969,16 +6048,16 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yallist@4.0.0, yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yallist@^3.0.0, yallist@^3.0.2, yallist@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== -yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - yaml@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.1.tgz#02fe0975d23cd441242aa7204e09fc28ac2ac33b" @@ -6017,7 +6096,7 @@ yocto-queue@^1.0.0: resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251" integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g== -zod@^3.22.4, zod@^3.23.5: +zod@^3.22.4, zod@^3.23.5, zod@^3.23.8: version "3.23.8" resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d" integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==