refactor db initialization

archiver-config.json

@@ -57,6 +57,5 @@
     }
   ],
   "ARCHIVER_MODE": "release",
-  "DevPublicKey": "",
-  "EXISTING_ARCHIVER_DB_PATH": ""
+  "DevPublicKey": ""
 }

debug_mode.patch

@@ -8,8 +8,7 @@ index a892baf..ffac25a 100644
    ],
 -  "ARCHIVER_MODE": "release",
 +  "ARCHIVER_MODE": "debug",
-   "DevPublicKey": "",
-   "EXISTING_ARCHIVER_DB_PATH": ""
+   "DevPublicKey": ""
  }
 \ No newline at end of file
 diff --git a/src/Config.ts b/src/Config.ts

src/Config.ts

@@ -20,7 +20,6 @@ export interface Config {
     receiptDB: string
     originalTxDataDB: string
   }
-  EXISTING_ARCHIVER_DB_PATH: string
   DATASENDER_TIMEOUT: number
   RATE_LIMIT: number // number of allowed request per second,
   N_NODE_REJECT_PERCENT: number
@@ -104,7 +103,6 @@ let config: Config = {
     receiptDB: 'receipts.sqlite3',
     originalTxDataDB: 'originalTxsData.sqlite3',
   },
-  EXISTING_ARCHIVER_DB_PATH: '',
   DATASENDER_TIMEOUT: 1000 * 60 * 5,
   RATE_LIMIT: 100, // 100 req per second,
   N_NODE_REJECT_PERCENT: 5, // Percentage of old nodes to remove from nodelist

src/dbstore/accounts.ts

@@ -1,5 +1,5 @@
 import * as db from './sqlite3storage'
-import { accountDatabase, extractValues, extractValuesFromArray } from './sqlite3storage'
+import { accountDatabase } from '.'
 import * as Logger from '../Logger'
 import { config } from '../Config'
 import { DeSerializeFromJsonString, SerializeToJsonString } from '../utils/serialization'
@@ -22,7 +22,7 @@ export async function insertAccount(account: AccountsCopy): Promise<void> {
   try {
     const fields = Object.keys(account).join(', ')
     const placeholders = Object.keys(account).fill('?').join(', ')
-    const values = extractValues(account)
+    const values = db.extractValues(account)
     const sql = 'INSERT OR REPLACE INTO accounts (' + fields + ') VALUES (' + placeholders + ')'
     await db.run(accountDatabase, sql, values)
     if (config.VERBOSE) {
@@ -41,7 +41,7 @@ export async function bulkInsertAccounts(accounts: AccountsCopy[]): Promise<void
   try {
     const fields = Object.keys(accounts[0]).join(', ')
     const placeholders = Object.keys(accounts[0]).fill('?').join(', ')
-    const values = extractValuesFromArray(accounts)
+    const values = db.extractValuesFromArray(accounts)
     let sql = 'INSERT OR REPLACE INTO accounts (' + fields + ') VALUES (' + placeholders + ')'
     for (let i = 1; i < accounts.length; i++) {
       sql = sql + ', (' + placeholders + ')'

src/dbstore/cycles.ts

@@ -1,5 +1,5 @@
 import * as db from './sqlite3storage'
-import { cycleDatabase, extractValues, extractValuesFromArray } from './sqlite3storage'
+import { cycleDatabase } from '.'
 import { P2P } from '@shardus/types'
 import * as Logger from '../Logger'
 import { config } from '../Config'
@@ -10,7 +10,7 @@ export async function insertCycle(cycle: Cycle): Promise<void> {
   try {
     const fields = Object.keys(cycle).join(', ')
     const placeholders = Object.keys(cycle).fill('?').join(', ')
-    const values = extractValues(cycle)
+    const values = db.extractValues(cycle)
     const sql = 'INSERT OR REPLACE INTO cycles (' + fields + ') VALUES (' + placeholders + ')'
     await db.run(cycleDatabase, sql, values)
     Logger.mainLogger.debug('Successfully inserted Cycle', cycle.cycleRecord.counter, cycle.cycleMarker)
@@ -28,7 +28,7 @@ export async function bulkInsertCycles(cycles: Cycle[]): Promise<void> {
   try {
     const fields = Object.keys(cycles[0]).join(', ')
     const placeholders = Object.keys(cycles[0]).fill('?').join(', ')
-    const values = extractValuesFromArray(cycles)
+    const values = db.extractValuesFromArray(cycles)
     let sql = 'INSERT OR REPLACE INTO cycles (' + fields + ') VALUES (' + placeholders + ')'
     for (let i = 1; i < cycles.length; i++) {
       sql = sql + ', (' + placeholders + ')'

src/dbstore/index.ts

@@ -1,17 +1,28 @@
+import * as fs from 'fs'
+import * as path from 'path'
+import { Database } from 'sqlite3'
 import { Config } from '../Config'
-import {
-  init,
-  runCreate,
-  close,
-  accountDatabase,
-  transactionDatabase,
-  cycleDatabase,
-  receiptDatabase,
-  originalTxDataDatabase,
-} from './sqlite3storage'
+import { createDB, runCreate, close } from './sqlite3storage'
+
+export let cycleDatabase: Database
+export let accountDatabase: Database
+export let transactionDatabase: Database
+export let receiptDatabase: Database
+export let originalTxDataDatabase: Database
 
 export const initializeDB = async (config: Config): Promise<void> => {
-  await init(config)
+  createDirectories(config.ARCHIVER_DB)
+  accountDatabase = await createDB(`${config.ARCHIVER_DB}/${config.ARCHIVER_DATA.accountDB}`, 'Account')
+  cycleDatabase = await createDB(`${config.ARCHIVER_DB}/${config.ARCHIVER_DATA.cycleDB}`, 'Cycle')
+  transactionDatabase = await createDB(
+    `${config.ARCHIVER_DB}/${config.ARCHIVER_DATA.transactionDB}`,
+    'Transaction'
+  )
+  receiptDatabase = await createDB(`${config.ARCHIVER_DB}/${config.ARCHIVER_DATA.receiptDB}`, 'Receipt')
+  originalTxDataDatabase = await createDB(
+    `${config.ARCHIVER_DB}/${config.ARCHIVER_DATA.originalTxDataDB}`,
+    'OriginalTxData'
+  )
   await runCreate(
     transactionDatabase,
     'CREATE TABLE if not exists `transactions` (`txId` TEXT NOT NULL UNIQUE PRIMARY KEY, `appReceiptId` TEXT, `timestamp` BIGINT NOT NULL, `cycleNumber` NUMBER NOT NULL, `data` JSON NOT NULL, `originalTxData` JSON NOT NULL)'
@@ -91,3 +102,9 @@ export const closeDatabase = async (): Promise<void> => {
   await close(receiptDatabase, 'Receipt')
   await close(originalTxDataDatabase, 'OriginalTxData')
 }
+
+function createDirectories(pathname: string): void {
+  const __dirname = path.resolve()
+  pathname = pathname.replace(/^\.*\/|\/?[^/]+\.[a-z]+|\/$/g, '') // Remove leading directory markers, and remove ending /file-name.extension
+  fs.mkdirSync(path.resolve(__dirname, pathname), { recursive: true }) // eslint-disable-line security/detect-non-literal-fs-filename
+}

src/dbstore/originalTxsData.ts

@@ -1,6 +1,6 @@
 // import { Signature } from 'shardus-crypto-types'
 import * as db from './sqlite3storage'
-import { originalTxDataDatabase, extractValues, extractValuesFromArray } from './sqlite3storage'
+import { originalTxDataDatabase } from '.'
 import * as Logger from '../Logger'
 import { config } from '../Config'
 import { DeSerializeFromJsonString } from '../utils/serialization'
@@ -31,7 +31,7 @@ export async function insertOriginalTxData(OriginalTxData: OriginalTxData): Prom
   try {
     const fields = Object.keys(OriginalTxData).join(', ')
     const placeholders = Object.keys(OriginalTxData).fill('?').join(', ')
-    const values = extractValues(OriginalTxData)
+    const values = db.extractValues(OriginalTxData)
     const sql = 'INSERT OR REPLACE INTO originalTxsData (' + fields + ') VALUES (' + placeholders + ')'
     await db.run(originalTxDataDatabase, sql, values)
     if (config.VERBOSE) {
@@ -50,7 +50,7 @@ export async function bulkInsertOriginalTxsData(originalTxsData: OriginalTxData[
   try {
     const fields = Object.keys(originalTxsData[0]).join(', ')
     const placeholders = Object.keys(originalTxsData[0]).fill('?').join(', ')
-    const values = extractValuesFromArray(originalTxsData)
+    const values = db.extractValuesFromArray(originalTxsData)
     let sql = 'INSERT OR REPLACE INTO originalTxsData (' + fields + ') VALUES (' + placeholders + ')'
     for (let i = 1; i < originalTxsData.length; i++) {
       sql = sql + ', (' + placeholders + ')'

src/dbstore/receipts.ts

@@ -1,6 +1,6 @@
 import { Signature } from '@shardus/crypto-utils'
 import * as db from './sqlite3storage'
-import { receiptDatabase, extractValues, extractValuesFromArray } from './sqlite3storage'
+import { receiptDatabase } from '.'
 import * as Logger from '../Logger'
 import { config } from '../Config'
 import { DeSerializeFromJsonString } from '../utils/serialization'
@@ -109,7 +109,7 @@ export async function insertReceipt(receipt: Receipt): Promise<void> {
   try {
     const fields = Object.keys(receipt).join(', ')
     const placeholders = Object.keys(receipt).fill('?').join(', ')
-    const values = extractValues(receipt)
+    const values = db.extractValues(receipt)
     const sql = 'INSERT OR REPLACE INTO receipts (' + fields + ') VALUES (' + placeholders + ')'
     await db.run(receiptDatabase, sql, values)
     if (config.VERBOSE) {
@@ -128,7 +128,7 @@ export async function bulkInsertReceipts(receipts: Receipt[]): Promise<void> {
   try {
     const fields = Object.keys(receipts[0]).join(', ')
     const placeholders = Object.keys(receipts[0]).fill('?').join(', ')
-    const values = extractValuesFromArray(receipts)
+    const values = db.extractValuesFromArray(receipts)
     let sql = 'INSERT OR REPLACE INTO receipts (' + fields + ') VALUES (' + placeholders + ')'
     for (let i = 1; i < receipts.length; i++) {
       sql = sql + ', (' + placeholders + ')'

src/dbstore/sqlite3storage.ts

@@ -1,32 +1,7 @@
-import * as fs from 'fs'
-import * as path from 'path'
-import { Config } from '../Config'
 import { SerializeToJsonString } from '../utils/serialization'
 import { Database } from 'sqlite3'
-// eslint-disable-next-line @typescript-eslint/no-var-requires
 
-export let cycleDatabase: Database
-export let accountDatabase: Database
-export let transactionDatabase: Database
-export let receiptDatabase: Database
-export let originalTxDataDatabase: Database
-
-export async function init(config: Config): Promise<void> {
-  createDirectories(config.ARCHIVER_DB)
-  accountDatabase = await createDB(`${config.ARCHIVER_DB}/${config.ARCHIVER_DATA.accountDB}`, 'Account')
-  cycleDatabase = await createDB(`${config.ARCHIVER_DB}/${config.ARCHIVER_DATA.cycleDB}`, 'Cycle')
-  transactionDatabase = await createDB(
-    `${config.ARCHIVER_DB}/${config.ARCHIVER_DATA.transactionDB}`,
-    'Transaction'
-  )
-  receiptDatabase = await createDB(`${config.ARCHIVER_DB}/${config.ARCHIVER_DATA.receiptDB}`, 'Receipt')
-  originalTxDataDatabase = await createDB(
-    `${config.ARCHIVER_DB}/${config.ARCHIVER_DATA.originalTxDataDB}`,
-    'OriginalTxData'
-  )
-}
-
-const createDB = async (dbPath: string, dbName: string): Promise<Database> => {
+export const createDB = async (dbPath: string, dbName: string): Promise<Database> => {
   console.log('dbName', dbName, 'dbPath', dbPath)
   const db = new Database(dbPath, (err) => {
     if (err) {
@@ -146,9 +121,3 @@ export function extractValuesFromArray(arr: object[]): unknown[] {
     return null
   }
 }
-
-function createDirectories(pathname: string): void {
-  const __dirname = path.resolve()
-  pathname = pathname.replace(/^\.*\/|\/?[^/]+\.[a-z]+|\/$/g, '') // Remove leading directory markers, and remove ending /file-name.extension
-  fs.mkdirSync(path.resolve(__dirname, pathname), { recursive: true }) // eslint-disable-line security/detect-non-literal-fs-filename
-}

src/dbstore/transactions.ts

@@ -1,6 +1,6 @@
 // import { Signature } from 'shardus-crypto-types'
 import * as db from './sqlite3storage'
-import { transactionDatabase, extractValues, extractValuesFromArray } from './sqlite3storage'
+import { transactionDatabase } from '.'
 import * as Logger from '../Logger'
 import { config } from '../Config'
 import { DeSerializeFromJsonString } from '../utils/serialization'
@@ -28,7 +28,7 @@ export async function insertTransaction(transaction: Transaction): Promise<void>
   try {
     const fields = Object.keys(transaction).join(', ')
     const placeholders = Object.keys(transaction).fill('?').join(', ')
-    const values = extractValues(transaction)
+    const values = db.extractValues(transaction)
     const sql = 'INSERT OR REPLACE INTO transactions (' + fields + ') VALUES (' + placeholders + ')'
     await db.run(transactionDatabase, sql, values)
     if (config.VERBOSE) {
@@ -47,7 +47,7 @@ export async function bulkInsertTransactions(transactions: Transaction[]): Promi
   try {
     const fields = Object.keys(transactions[0]).join(', ')
     const placeholders = Object.keys(transactions[0]).fill('?').join(', ')
-    const values = extractValuesFromArray(transactions)
+    const values = db.extractValuesFromArray(transactions)
     let sql = 'INSERT OR REPLACE INTO transactions (' + fields + ') VALUES (' + placeholders + ')'
     for (let i = 1; i < transactions.length; i++) {
       sql = sql + ', (' + placeholders + ')'

src/primary-process/index.ts

@@ -241,7 +241,7 @@ export const offloadReceipt = async (
       }
     } else {
       console.log('Verifying on the worker process 1', txId, timestamp, worker.process.pid)
-      const cloneReceipt = { ...receipt }
+      const cloneReceipt = Utils.deepCopy(receipt)
       delete cloneReceipt.tx.originalTxData
       delete cloneReceipt.executionShardKey
       const stringifiedReceipt = StringUtils.safeStringify(cloneReceipt)