feat(db): add config option to support disabling DB writers #37

Adds an experimental config environment variable, START_WRITERS. It
defaults to 'true'. If set to 'false', worker processes that write to
the DB will be disabled. This is to support experimentation with SQLite
replaction via LiteFS.

docker-compose.yaml

@@ -48,3 +48,4 @@ services:
       - ANS104_INDEX_FILTER=${ANS104_INDEX_FILTER:-}
       - ARNS_ROOT_HOST=${ARNS_ROOT_HOST:-}
       - SANDBOX_PROTOCOL=${SANDBOX_PROTOCOL:-}
+      - START_WRITERS=${START_WRITERS:-}

src/app.ts

@@ -37,9 +37,13 @@ import { apolloServer } from './routes/graphql/index.js';
 import * as system from './system.js';
 
 system.arweaveClient.refreshPeers();
-system.blockImporter.start();
-system.txRepairWorker.start();
-system.bundleRepairWorker.start();
+
+// Allow starting without writers to support SQLite replication
+if (config.START_WRITERS) {
+  system.blockImporter.start();
+  system.txRepairWorker.start();
+  system.bundleRepairWorker.start();
+}
 
 // HTTP server
 const app = express();

src/config.ts

@@ -71,3 +71,5 @@ export const ANS104_INDEX_FILTER = createFilter(
 );
 export const ARNS_ROOT_HOST = env.varOrUndefined('ARNS_ROOT_HOST');
 export const SANDBOX_PROTOCOL = env.varOrUndefined('SANDBOX_PROTOCOL');
+export const START_WRITERS =
+  env.varOrDefault('START_WRITERS', 'true') === 'true';

src/data/read-through-data-cache.ts

@@ -158,14 +158,22 @@ export class ReadThroughDataCache implements ContiguousDataSource {
           const hash = hasher.digest('base64url');
 
           this.log.info('Successfully cached data', { id, hash });
-          await this.contiguousDataIndex.saveDataContentAttributes({
-            id,
-            dataRoot: attributes?.dataRoot,
-            hash,
-            dataSize: data.size,
-            contentType: data.sourceContentType,
-            cachedAt: currentUnixTimestamp(),
-          });
+          try {
+            await this.contiguousDataIndex.saveDataContentAttributes({
+              id,
+              dataRoot: attributes?.dataRoot,
+              hash,
+              dataSize: data.size,
+              contentType: data.sourceContentType,
+              cachedAt: currentUnixTimestamp(),
+            });
+          } catch (error: any) {
+            this.log.error('Error saving data content attributes:', {
+              id,
+              message: error.message,
+              stack: error.stack,
+            });
+          }
 
           try {
             await this.dataStore.finalize(cacheStream, hash);