diff --git a/docker-compose.yaml b/docker-compose.yaml
index 180f20ac..087b2489 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -19,6 +19,9 @@ services:
- TVAL_GRAPHQL_HOST=${GRAPHQL_HOST:-core}
- TVAL_GRAPHQL_PORT=${GRAPHQL_PORT:-4000}
- TVAL_ARNS_ROOT_HOST=${ARNS_ROOT_HOST:-}
+ depends_on:
+ - core
+ - observer
core:
image: ghcr.io/ar-io/ar-io-core:latest
@@ -53,7 +56,19 @@ services:
- SANDBOX_PROTOCOL=${SANDBOX_PROTOCOL:-}
- START_WRITERS=${START_WRITERS:-}
- CONTRACT_ID=${CONTRACT_ID:-}
- - CHAIN_CACHE_TYPE=${CHAIN_CACHE_TYPE:-}
+ - CHAIN_CACHE_TYPE=${CHAIN_CACHE_TYPE:-redis}
+ - REDIS_CACHE_URL=${REDIS_CACHE_URL:-redis://redis:6379}
+ - REDIS_CACHE_TTL_SECONDS=${REDIS_CACHE_TTL_SECONDS:-}
+ depends_on:
+ - redis
+
+ redis:
+ image: redis:latest
+ command: redis-server --appendonly yes --maxmemory ${REDIS_MAX_MEMORY:-2gb}
+ ports:
+ - 6379:6379
+ volumes:
+ - ${REDIS_DATA_PATH:-./data/redis}:/data
observer:
image: ghcr.io/ar-io/ar-io-observer:${OBSERVER_IMAGE_TAG:-14802babee090d674249960df890c54c9406076b}
diff --git a/package.json b/package.json
index 5bcf2b81..392395e4 100644
--- a/package.json
+++ b/package.json
@@ -31,6 +31,7 @@
"node-cache": "^5.1.2",
"prom-client": "^14.0.1",
"ramda": "^0.28.0",
+ "redis": "^4.6.10",
"retry-axios": "^3.0.0",
"rfc4648": "^1.5.2",
"sql-bricks": "^3.0.0",
diff --git a/src/config.ts b/src/config.ts
index b7466449..c8a62976 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -80,3 +80,11 @@ export const CONTRACT_ID = env.varOrDefault(
'bLAgYxAdX2Ry-nt6aH2ixgvJXbpsEYm28NgJgyqfs-U',
);
export const CHAIN_CACHE_TYPE = env.varOrDefault('CHAIN_CACHE_TYPE', 'fs');
+export const REDIS_CACHE_URL = env.varOrDefault(
+ 'REDIS_CACHE_URL',
+ 'redis://localhost:6379',
+);
+export const REDIS_CACHE_TTL_SECONDS = +env.varOrDefault(
+ 'REDIS_CACHE_TTL_SECONDS',
+ `${60 * 60 * 8}`, // 8 hours by default
+);
diff --git a/src/lib/kvstore.ts b/src/lib/kvstore.ts
index 81ed74a7..c0eefaf4 100644
--- a/src/lib/kvstore.ts
+++ b/src/lib/kvstore.ts
@@ -15,29 +15,44 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
+import winston from 'winston';
+
+import * as config from '../config.js';
import { FsKVStore } from '../store/fs-kv-store.js';
import { LmdbKVStore } from '../store/lmdb-kv-store.js';
+import { RedisKvStore } from '../store/redis-kv-store.js';
import { KVBufferStore } from '../types.js';
export const getKvBufferStore = ({
pathKey,
type,
+ log,
}: {
pathKey: string;
type: string;
+ log: winston.Logger;
}): KVBufferStore => {
+ log.info(`Using ${type} for KVBufferStore for ${pathKey}`);
switch (type) {
case 'lmdb': {
return new LmdbKVStore({
dbPath: `data/lmdb/${pathKey}`,
});
}
+ case 'redis': {
+ return new RedisKvStore({
+ redisUrl: config.REDIS_CACHE_URL,
+ ttlSeconds: config.REDIS_CACHE_TTL_SECONDS,
+ log,
+ });
+ }
case 'fs': {
return new FsKVStore({
baseDir: `data/headers/${pathKey}`,
tmpDir: `data/tmp/${pathKey}`,
});
}
+
// TODO: implement redis
default: {
throw new Error(`Invalid chain cache type: ${type}`);
diff --git a/src/metrics.ts b/src/metrics.ts
index 42195da8..0d0721e7 100644
--- a/src/metrics.ts
+++ b/src/metrics.ts
@@ -132,3 +132,15 @@ export const lastHeightImported = new promClient.Gauge({
name: 'last_height_imported',
help: 'Height of the last block imported',
});
+
+// Redis Cache Metrics
+
+export const redisConnectionErrorsCounter = new promClient.Counter({
+ name: 'redis_connection_errors_total',
+ help: 'Number of errors connecting to redis',
+});
+
+export const redisErrorCounter = new promClient.Counter({
+ name: 'redis_errors_total',
+ help: 'Number of errors redis cache has received',
+});
diff --git a/src/store/redis-kv-store.ts b/src/store/redis-kv-store.ts
new file mode 100644
index 00000000..b53b6002
--- /dev/null
+++ b/src/store/redis-kv-store.ts
@@ -0,0 +1,79 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+import { RedisClientType, commandOptions, createClient } from 'redis';
+import winston from 'winston';
+
+import * as metrics from '../metrics.js';
+import { KVBufferStore } from '../types.js';
+
+export class RedisKvStore implements KVBufferStore {
+ private client: RedisClientType;
+ private log: winston.Logger;
+ private ttlSeconds: number;
+
+ constructor({
+ log,
+ redisUrl,
+ ttlSeconds,
+ }: {
+ log: winston.Logger;
+ redisUrl: string;
+ ttlSeconds: number;
+ }) {
+ this.log = log.child({ class: this.constructor.name });
+ this.ttlSeconds = ttlSeconds;
+ this.client = createClient({
+ url: redisUrl,
+ });
+ this.client.on('error', (err) => {
+ this.log.error(`Redis error: ${err}`);
+ metrics.redisErrorCounter.inc();
+ });
+ this.client.connect().catch((err) => {
+ this.log.error(`Redis connection error: ${err}`);
+ metrics.redisConnectionErrorsCounter.inc();
+ });
+ }
+
+ // TODO: close connection to redis safely
+
+ async get(key: string): Promise {
+ const value = await this.client.get(
+ commandOptions({ returnBuffers: true }),
+ key,
+ );
+ return value ?? undefined;
+ }
+
+ async has(key: string): Promise {
+ return (await this.client.exists(key)) === 1;
+ }
+
+ async del(key: string): Promise {
+ if (await this.has(key)) {
+ await this.client.del(key);
+ }
+ }
+
+ async set(key: string, buffer: Buffer): Promise {
+ // set the key with a TTL for every key
+ await this.client.set(key, buffer, {
+ EX: this.ttlSeconds,
+ });
+ }
+}
diff --git a/src/system.ts b/src/system.ts
index a49f42af..21366967 100644
--- a/src/system.ts
+++ b/src/system.ts
@@ -75,6 +75,7 @@ export const arweaveClient = new ArweaveCompositeClient({
blockStore: new KvBlockStore({
log,
kvBufferStore: getKvBufferStore({
+ log,
pathKey: 'partial-blocks',
type: config.CHAIN_CACHE_TYPE,
}),
@@ -82,6 +83,7 @@ export const arweaveClient = new ArweaveCompositeClient({
txStore: new KvTransactionStore({
log,
kvBufferStore: getKvBufferStore({
+ log,
pathKey: 'partial-txs',
type: config.CHAIN_CACHE_TYPE,
}),
diff --git a/yarn.lock b/yarn.lock
index ca3eadab..1512a4dd 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1044,6 +1044,40 @@
dependencies:
"@randlabs/communication-bridge" "^1.0.0"
+"@redis/bloom@1.2.0":
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71"
+ integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==
+
+"@redis/client@1.5.11":
+ version "1.5.11"
+ resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.11.tgz#5ee8620fea56c67cb427228c35d8403518efe622"
+ integrity sha512-cV7yHcOAtNQ5x/yQl7Yw1xf53kO0FNDTdDU6bFIMbW6ljB7U7ns0YRM+QIkpoqTAt6zK5k9Fq0QWlUbLcq9AvA==
+ dependencies:
+ cluster-key-slot "1.1.2"
+ generic-pool "3.9.0"
+ yallist "4.0.0"
+
+"@redis/graph@1.1.0":
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519"
+ integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==
+
+"@redis/json@1.0.6":
+ version "1.0.6"
+ resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.6.tgz#b7a7725bbb907765d84c99d55eac3fcf772e180e"
+ integrity sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==
+
+"@redis/search@1.1.5":
+ version "1.1.5"
+ resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.5.tgz#682b68114049ff28fdf2d82c580044dfb74199fe"
+ integrity sha512-hPP8w7GfGsbtYEJdn4n7nXa6xt6hVZnnDktKW4ArMaFQ/m/aR7eFvsLQmG/mn1Upq99btPJk+F27IQ2dYpCoUg==
+
+"@redis/time-series@1.0.5":
+ version "1.0.5"
+ resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad"
+ integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==
+
"@rushstack/ts-command-line@^4.12.2":
version "4.12.2"
resolved "https://registry.yarnpkg.com/@rushstack/ts-command-line/-/ts-command-line-4.12.2.tgz#59b7450c5d75190778cce8b159c7d7043c32cc4e"
@@ -2474,6 +2508,11 @@ clone@2.x:
resolved "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz"
integrity sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==
+cluster-key-slot@1.1.2:
+ version "1.1.2"
+ resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac"
+ integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==
+
code-point-at@^1.0.0:
version "1.1.0"
resolved "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz"
@@ -3568,6 +3607,11 @@ gc-stats@^1.4.0:
nan "^2.13.2"
node-pre-gyp "^0.13.0"
+generic-pool@3.9.0:
+ version "3.9.0"
+ resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4"
+ integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==
+
gensync@^1.0.0-beta.2:
version "1.0.0-beta.2"
resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0"
@@ -5511,6 +5555,18 @@ readdirp@~3.6.0:
dependencies:
picomatch "^2.2.1"
+redis@^4.6.10:
+ version "4.6.10"
+ resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.10.tgz#07f6ea2b2c5455b098e76d1e8c9b3376114e9458"
+ integrity sha512-mmbyhuKgDiJ5TWUhiKhBssz+mjsuSI/lSZNPI9QvZOYzWvYGejtb+W3RlDDf8LD6Bdl5/mZeG8O1feUGhXTxEg==
+ dependencies:
+ "@redis/bloom" "1.2.0"
+ "@redis/client" "1.5.11"
+ "@redis/graph" "1.1.0"
+ "@redis/json" "1.0.6"
+ "@redis/search" "1.1.5"
+ "@redis/time-series" "1.0.5"
+
regenerator-runtime@^0.13.4:
version "0.13.9"
resolved "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz"
@@ -6754,16 +6810,16 @@ y18n@^5.0.5:
resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz"
integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==
+yallist@4.0.0, yallist@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz"
+ integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
+
yallist@^3.0.0, yallist@^3.0.2, yallist@^3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"
integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==
-yallist@^4.0.0:
- version "4.0.0"
- resolved "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz"
- integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
-
yaml@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.1.1.tgz#1e06fb4ca46e60d9da07e4f786ea370ed3c3cfec"