feat(standalone-sqlite): better sqlite error logs

src/app.ts

@@ -82,11 +82,4 @@ apolloServerInstanceGql.start().then(() => {
   });
 });
 
-// Handle shutdown signals
-process.on('SIGINT', async () => {
-  await system.shutdown(server);
-});
-
-process.on('SIGTERM', async () => {
-  await system.shutdown(server);
-});
+export { server };

src/database/standalone-sqlite.ts

@@ -67,6 +67,7 @@ import {
   TransactionAttributes,
 } from '../types.js';
 import * as config from '../config.js';
+import { DetailedError } from '../lib/error.js';
 
 const CPU_COUNT = os.cpus().length;
 const MAX_WORKER_COUNT = 12;
@@ -2598,9 +2599,15 @@ export class StandaloneSqliteDatabase
           self.workers[pool][role].push({ takeWork });
           takeWork();
         })
-        .on('message', (result) => {
-          if (result === '__ERROR__') {
-            job.reject(new Error('Worker error'));
+        .on('message', async (result) => {
+          if (result && result.stack) {
+            const { message, stack, workerMethod, workerArgs } = result;
+            const error = new DetailedError(message, {
+              stack,
+              workerMethod,
+              workerArgs,
+            });
+            job.reject(error);
           } else {
             job.resolve(result);
           }
@@ -3204,14 +3211,29 @@ if (!isMainThread) {
           parentPort?.postMessage(null);
           process.exit(0);
       }
-    } catch (error) {
+    } catch (e: any) {
       if (errorCount > MAX_WORKER_ERRORS) {
         log.error('Too many errors in StandaloneSqlite worker, exiting.');
         process.exit(1);
       }
-      log.error('Error in StandaloneSqlite worker:', error);
+
+      const error = new DetailedError('Error in StandaloneSqlite worker', {
+        stack: e.stack,
+      });
+
+      log.error(error.message, {
+        message: error.message,
+        stack: error.stack,
+        workerMethod: method,
+        workerArgs: args,
+      });
       errorCount++;
-      parentPort?.postMessage('__ERROR__');
+      parentPort?.postMessage({
+        message: error.message,
+        stack: error.stack,
+        workerMethod: method,
+        workerArgs: args,
+      });
     }
   });
 }

src/lib/error.ts

@@ -0,0 +1,37 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2023 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+interface DetailedErrorOptions {
+  stack?: string;
+  [key: string]: any; // Allow any other properties with any value type
+}
+
+export class DetailedError extends Error {
+  constructor(message: string, options?: DetailedErrorOptions) {
+    super(message);
+    this.name = this.constructor.name;
+    Object.assign(this, options);
+    this.stack = options?.stack ?? new Error().stack;
+  }
+
+  toJSON() {
+    return {
+      ...this,
+    };
+  }
+}

src/system.ts

@@ -17,7 +17,6 @@
  */
 import { default as Arweave } from 'arweave';
 import EventEmitter from 'node:events';
-import { Server } from 'node:http';
 import fs from 'node:fs';
 import { AOProcess, IO } from '@ar.io/sdk';
 import awsLite from '@aws-lite/client';
@@ -83,6 +82,7 @@ import { SignatureFetcher } from './data/signature-fetcher.js';
 import { SQLiteWalCleanupWorker } from './workers/sqlite-wal-cleanup-worker.js';
 import { KvArnsStore } from './store/kv-arns-store.js';
 import { parquetExporter } from './routes/ar-io.js';
+import { server } from './app.js';
 
 process.on('uncaughtException', (error) => {
   metrics.uncaughtExceptionCounter.inc();
@@ -642,13 +642,13 @@ if (dataSqliteWalCleanupWorker !== undefined) {
 
 let isShuttingDown = false;
 
-export const shutdown = async (express: Server) => {
+export const shutdown = async (exitCode = 0) => {
   if (isShuttingDown) {
     log.info('Shutdown already in progress');
   } else {
     isShuttingDown = true;
     log.info('Shutting down...');
-    express.close(async () => {
+    server.close(async () => {
       log.debug('Web server stopped successfully');
       eventEmitter.removeAllListeners();
       arIODataSource.stopUpdatingPeers();
@@ -672,7 +672,16 @@ export const shutdown = async (express: Server) => {
       await headerFsCacheCleanupWorker?.stop();
       await contiguousDataFsCacheCleanupWorker?.stop();
 
-      process.exit(0);
+      process.exit(exitCode);
     });
   }
 };
+
+// Handle shutdown signals
+process.on('SIGINT', async () => {
+  await shutdown();
+});
+
+process.on('SIGTERM', async () => {
+  await shutdown();
+});