GOLD-101 stringify migration (#25)

* feat: migrate to unified stringify standard lib

* fix: avoid build path import introduced by prior commit

* fix(test): added missing replacements to safe stringify and code removal

* fix(json): fixed deserialisation function

* feat(socket): using string to decode socket response from server

* fix: add parsing/stringify for bigint in fastify

* fix: customSerializerPlugin + custom parse for `getJson` and `postJson`

* fix(server): serialisation hook simplification

* fix(logs): simplified logs and removed unused

* Update @shardus/types to version 1.2.13

* fix(rename): crypto util stringify indentifier rename

* 3.4.18-0

* 3.4.18

---------

Co-authored-by: Arham Jain <[email protected]>
Co-authored-by: CombatPug <[email protected]>
Co-authored-by: Jintu Das <[email protected]>

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shardus/archiver",
-  "version": "3.4.17",
+  "version": "3.4.18",
   "engines": {
     "node": "18.16.1"
   },
@@ -78,7 +78,7 @@
     "@fastify/rate-limit": "^7.6.0",
     "@shardus/archiver-discovery": "^1.1.0",
     "@shardus/crypto-utils": "4.1.3",
-    "@shardus/types": "1.2.10",
+    "@shardus/types": "1.2.13",
     "deepmerge": "^4.2.2",
     "fastify": "4.12.0",
     "log4js": "^6.3.0",

scripts/api_tester.ts

@@ -2,6 +2,7 @@ import * as crypto from '@shardus/crypto-utils'
 import fetch from 'node-fetch'
 import { join } from 'path'
 import { config, overrideDefaultConfig } from '../src/Config'
+import { Utils as StringUtils } from '@shardus/types'
 
 const configFile = join(process.cwd(), 'archiver-config.json')
 overrideDefaultConfig(configFile)
@@ -27,7 +28,7 @@ fetch(`${ARCHIVER_URL}/totalData`, {
   // fetch(`${ARCHIVER_URL}/receipt`, {
   // fetch(`${ARCHIVER_URL}/account`, {
   method: 'post',
-  body: JSON.stringify(data),
+  body: StringUtils.safeStringify(data),
   headers: { 'Content-Type': 'application/json' },
   timeout: 2000,
 })

scripts/archiver_data_patcher.ts

@@ -15,6 +15,7 @@ import * as Crypto from '../src/Crypto'
 import { join } from 'path'
 import * as Logger from '../src/Logger'
 import { startSaving } from '../src/saveConsoleOutput'
+import { Utils as StringUtils } from '@shardus/types'
 const {
   MAX_RECEIPTS_PER_REQUEST,
   MAX_BETWEEN_CYCLES_PER_REQUEST,
@@ -55,7 +56,7 @@ const runProgram = async (): Promise<void> => {
   Crypto.setCryptoHashKey(hashKey)
   let logsConfig
   try {
-    logsConfig = JSON.parse(readFileSync(resolve(__dirname, '../archiver-log.json'), 'utf8'))
+    logsConfig = StringUtils.safeJsonParse(readFileSync(resolve(__dirname, '../archiver-log.json'), 'utf8'))
   } catch (err) {
     console.log('Failed to parse archiver log file:', err)
   }
@@ -93,7 +94,7 @@ const runProgram = async (): Promise<void> => {
         const downloadedReceipts = downloadedReceiptCountByCycles.receipts.filter((d) => d.cycle === j)
         const existingReceipts = receiptsCountByCycles.filter((d) => d.cycle === j)
         // console.log(j, downloadedReceipts, existingReceipts)
-        if (JSON.stringify(downloadedReceipts) !== JSON.stringify(existingReceipts)) {
+        if (StringUtils.safeStringify(downloadedReceipts) !== StringUtils.safeStringify(existingReceipts)) {
           console.log('Unmatched', j, downloadedReceipts, existingReceipts)
           const receipts = await fetchDataForCycle(archiver, DataType.RECEIPT, j)
           console.log('Downloaded receipts for cycle', j, ' -> ', receipts.length)
@@ -129,7 +130,7 @@ const runProgram = async (): Promise<void> => {
         )
         const existingOriginalTxsData = originalTxsDataCountByCycles.filter((d) => d.cycle === j)
         // console.log(j, downloadedOriginalTxsData, existingOriginalTxsData)
-        if (JSON.stringify(downloadedOriginalTxsData) !== JSON.stringify(existingOriginalTxsData)) {
+        if (StringUtils.safeStringify(downloadedOriginalTxsData) !== StringUtils.safeStringify(existingOriginalTxsData)) {
           console.log('Unmatched', j, downloadedOriginalTxsData, existingOriginalTxsData)
           const originalTxsData = await fetchDataForCycle(archiver, DataType.ORIGINAL_TX_DATA, j)
           if (originalTxsData) {

scripts/archiver_data_sync_check.ts

@@ -67,7 +67,10 @@ const runProgram = async (): Promise<void> => {
     }
     // console.dir(responses, { depth: null })
     // save to file
-    writeFileSync(`archiver_${archiverInfo}_${startCycle}_${endCycle}_${URL}.json`, JSON.stringify(responses))
+    writeFileSync(
+      `archiver_${archiverInfo}_${startCycle}_${endCycle}_${URL}.json`,
+      StringUtils.safeStringify(responses)
+    )
   }
 }
 

scripts/create_shut_down_cycle.ts

@@ -41,7 +41,7 @@ const runProgram = async (): Promise<void> => {
   Crypto.setCryptoHashKey(hashKey)
   let logsConfig
   try {
-    logsConfig = JSON.parse(readFileSync(resolve(__dirname, '../archiver-log.json'), 'utf8'))
+    logsConfig = StringUtils.safeJsonParse(readFileSync(resolve(__dirname, '../archiver-log.json'), 'utf8'))
   } catch (err) {
     console.log('Failed to parse archiver log file:', err)
   }

scripts/repair_missing_cycle.ts

@@ -20,7 +20,7 @@ const start = async (): Promise<void> => {
   Crypto.setCryptoHashKey(hashKey)
   let logsConfig
   try {
-    logsConfig = JSON.parse(readFileSync(resolve(__dirname, '../archiver-log.json'), 'utf8'))
+    logsConfig = StringUtils.safeJsonParse(readFileSync(resolve(__dirname, '../archiver-log.json'), 'utf8'))
   } catch (err) {
     console.log('Failed to parse archiver log file:', err)
   }

scripts/update_network_account.ts

@@ -24,7 +24,7 @@ const runProgram = async (): Promise<void> => {
   Crypto.setCryptoHashKey(hashKey)
   let logsConfig
   try {
-    logsConfig = JSON.parse(readFileSync(resolve(__dirname, '../archiver-log.json'), 'utf8'))
+    logsConfig = StringUtils.safeJsonParse(readFileSync(resolve(__dirname, '../archiver-log.json'), 'utf8'))
   } catch (err) {
     console.log('Failed to parse archiver log file:', err)
   }

scripts/verify_account_hash.ts

@@ -20,7 +20,7 @@ const runProgram = async (): Promise<void> => {
   Crypto.setCryptoHashKey(hashKey)
   let logsConfig
   try {
-    logsConfig = JSON.parse(readFileSync(resolve(__dirname, '../archiver-log.json'), 'utf8'))
+    logsConfig = StringUtils.safeJsonParse(readFileSync(resolve(__dirname, '../archiver-log.json'), 'utf8'))
   } catch (err) {
     console.log('Failed to parse archiver log file:', err)
   }

src/API.ts

@@ -27,6 +27,7 @@ import * as AccountDataProvider from './Data/AccountDataProvider'
 import { getGlobalNetworkAccount } from './GlobalAccount'
 import { cycleRecordWithShutDownMode } from './Data/Cycles'
 import { isDebugMiddleware } from './DebugMode'
+import { Utils as StringUtils } from '@shardus/types'
 const { version } = require('../package.json') // eslint-disable-line @typescript-eslint/no-var-requires
 
 const TXID_LENGTH = 64
@@ -299,7 +300,7 @@ export function registerRoutes(server: FastifyInstance<Server, IncomingMessage,
       }
       cycleInfo = await CycleDB.queryCycleRecordsBetween(from, to)
       if (isDownload) {
-        const dataInBuffer = Buffer.from(JSON.stringify(cycleInfo), 'utf-8')
+        const dataInBuffer = Buffer.from(StringUtils.safeStringify(cycleInfo), 'utf-8')
         const dataInStream = Readable.from(dataInBuffer)
         const filename = `cycle_records_from_${from}_to_${to}`
 
@@ -836,7 +837,8 @@ export function registerRoutes(server: FastifyInstance<Server, IncomingMessage,
 
   server.post('/gossip-data', async (_request: GossipDataRequest, reply) => {
     const gossipPayload = _request.body
-    if (config.VERBOSE) Logger.mainLogger.debug('Gossip Data received', JSON.stringify(gossipPayload))
+    if (config.VERBOSE)
+      Logger.mainLogger.debug('Gossip Data received', StringUtils.safeStringify(gossipPayload))
     const result = Collector.validateGossipData(gossipPayload)
     if (!result.success) {
       reply.send({ success: false, error: result.error })
@@ -855,7 +857,7 @@ export function registerRoutes(server: FastifyInstance<Server, IncomingMessage,
 
   server.post('/get_account_data_archiver', async (_request: AccountDataRequest, reply) => {
     const payload = _request.body as AccountDataProvider.AccountDataRequestSchema
-    if (config.VERBOSE) Logger.mainLogger.debug('Account Data received', JSON.stringify(payload))
+    if (config.VERBOSE) Logger.mainLogger.debug('Account Data received', StringUtils.safeStringify(payload))
     const result = AccountDataProvider.validateAccountDataRequest(payload)
     // Logger.mainLogger.debug('Account Data validation result', result)
     if (!result.success) {
@@ -873,7 +875,7 @@ export function registerRoutes(server: FastifyInstance<Server, IncomingMessage,
 
   server.post('/get_account_data_by_list_archiver', async (_request: AccountDataRequest, reply) => {
     const payload = _request.body as AccountDataProvider.AccountDataByListRequestSchema
-    if (config.VERBOSE) Logger.mainLogger.debug('Account Data By List received', JSON.stringify(payload))
+    if (config.VERBOSE) Logger.mainLogger.debug('Account Data By List received', StringUtils.safeStringify(payload))
     const result = AccountDataProvider.validateAccountDataByListRequest(payload)
     // Logger.mainLogger.debug('Account Data By List validation result', result)
     if (!result.success) {
@@ -891,7 +893,7 @@ export function registerRoutes(server: FastifyInstance<Server, IncomingMessage,
 
   server.post('/get_globalaccountreport_archiver', async (_request: AccountDataRequest, reply) => {
     const payload = _request.body as AccountDataProvider.GlobalAccountReportRequestSchema
-    if (config.VERBOSE) Logger.mainLogger.debug('Global Account Report received', JSON.stringify(payload))
+    if (config.VERBOSE) Logger.mainLogger.debug('Global Account Report received', StringUtils.safeStringify(payload))
     const result = AccountDataProvider.validateGlobalAccountReportRequest(payload)
     // Logger.mainLogger.debug('Global Account Report validation result', result)
     if (!result.success) {
@@ -1094,7 +1096,7 @@ export function registerRoutes(server: FastifyInstance<Server, IncomingMessage,
 
     server.post('/gossip-hashes', async (_request: GossipHashesRequest, reply) => {
       const gossipMessage = _request.body
-      Logger.mainLogger.debug('Gossip received', JSON.stringify(gossipMessage))
+      Logger.mainLogger.debug('Gossip received', StringUtils.safeStringify(gossipMessage))
       addHashesGossip(gossipMessage.sender, gossipMessage.data)
       const res = Crypto.sign({
         success: true,

src/Config.ts

@@ -3,6 +3,7 @@ import * as Logger from './Logger'
 import * as merge from 'deepmerge'
 import * as minimist from 'minimist'
 import { join } from 'path'
+import { Utils as StringUtils } from '@shardus/types'
 
 export interface Config {
   [index: string]: object | string | number | boolean
@@ -106,7 +107,7 @@ let config: Config = {
   },
   cycleRecordsCache: {
     enabled: false,
-  }
+  },
 }
 // Override default config params from config file, env vars, and cli args
 export async function overrideDefaultConfig(file: string): Promise<void> {
@@ -116,7 +117,7 @@ export async function overrideDefaultConfig(file: string): Promise<void> {
   // Override config from config file
   try {
     // eslint-disable-next-line security/detect-non-literal-fs-filename
-    const fileConfig = JSON.parse(fs.readFileSync(file, { encoding: 'utf8' }))
+    const fileConfig = StringUtils.safeJsonParse(fs.readFileSync(file, { encoding: 'utf8' }))
     const overwriteMerge = (target: [], source: []): [] => source
     config = merge(config, fileConfig, { arrayMerge: overwriteMerge })
   } catch (err) {
@@ -142,7 +143,7 @@ export async function overrideDefaultConfig(file: string): Promise<void> {
           try {
             const parameterStr = env[param]
             if (parameterStr) {
-              const parameterObj = JSON.parse(parameterStr)
+              const parameterObj = StringUtils.safeJsonParse(parameterStr)
               config[param] = parameterObj
             }
           } catch (e) {

src/Crypto.ts

@@ -1,13 +1,13 @@
 import * as core from '@shardus/crypto-utils'
 import { SignedObject, TaggedObject, publicKey, curvePublicKey, sharedKey } from '@shardus/crypto-utils'
+import { Utils as StringUtils } from '@shardus/types'
 import * as State from './State'
-import { cryptoStringify } from './utils/stringify'
 
 // Crypto initialization fns
 
 export function setCryptoHashKey(hashkey: string): void {
   core.init(hashkey)
-  core.setCustomStringifier(cryptoStringify, 'shardus_crypto_stringify')
+  core.setCustomStringifier(StringUtils.safeStringify, 'shardus_safeStringify')
 }
 
 export const hashObj = core.hashObj
@@ -16,7 +16,7 @@ export const hashObj = core.hashObj
 export type SignedMessage = SignedObject
 
 export function sign<T>(obj: T): T & SignedObject {
-  const objCopy = JSON.parse(core.stringify(obj))
+  const objCopy = StringUtils.safeJsonParse(core.stringify(obj))
   core.signObj(objCopy, State.getSecretKey(), State.getNodeInfo().publicKey)
   return objCopy
 }
@@ -55,7 +55,7 @@ export function getOrCreateSharedKey(pk: publicKey): sharedKey {
 
 export function tag<T>(obj: T, recipientPk: publicKey): T & TaggedMessage {
   const sharedKey = getOrCreateSharedKey(recipientPk)
-  const objCopy = JSON.parse(core.stringify(obj))
+  const objCopy = StringUtils.safeJsonParse(core.stringify(obj))
   objCopy.publicKey = State.getNodeInfo().publicKey
   core.tagObj(objCopy, sharedKey)
   return objCopy

src/Data/AccountDataProvider.ts

@@ -231,7 +231,7 @@ export const provideAccountDataRequest = async (
       }
     }
     delta = tsEnd - highestTs
-    // Logger.mainLogger.debug('Account Data received', JSON.stringify(payload))
+    // Logger.mainLogger.debug('Account Data received', StringUtils.safeStringify(payload))
     // Logger.mainLogger.debug(
     //   'delta ' + delta,
     //   'tsEnd ' + tsEnd,

src/Data/Collector.ts

@@ -29,6 +29,7 @@ import { ConsensusNodeInfo } from '../NodeList'
 import { verifyAccountHash } from '../shardeum/calculateAccountHash'
 import { verifyAppReceiptData } from '../shardeum/verifyAppReceiptData'
 import { Cycle as DbCycle } from '../dbstore/types'
+import { Utils as StringUtils } from '@shardus/types'
 
 export let storingAccountData = false
 const processedReceiptsMap: Map<string, number> = new Map()
@@ -227,7 +228,7 @@ const isReceiptRobust = async (
         Logger.mainLogger.error(
           `The receipt validation failed from robustQuery nodes ${receipt.tx.txId} , ${receipt.cycle}, ${receipt.tx.timestamp}`
         )
-        Logger.mainLogger.error(JSON.stringify(robustQueryReceipt), JSON.stringify(fullReceipt))
+        Logger.mainLogger.error(StringUtils.safeStringify(robustQueryReceipt), StringUtils.safeStringify(fullReceipt))
       }
     }
     Logger.mainLogger.error(
@@ -611,7 +612,7 @@ export const storeReceiptData = async (
     })
     if (config.dataLogWrite && ReceiptLogWriter)
       ReceiptLogWriter.writeToLog(
-        `${JSON.stringify({
+        `${StringUtils.safeStringify({
           ...receipt,
           receiptId: tx.txId,
           timestamp: tx.timestamp,
@@ -775,10 +776,10 @@ export const storeCycleData = async (cycles: P2PTypes.CycleCreatorTypes.CycleDat
       cycleMarker: cycleRecord.marker,
       cycleRecord,
     }
-    if (config.dataLogWrite && CycleLogWriter) CycleLogWriter.writeToLog(`${JSON.stringify(cycleObj)}\n`)
+    if (config.dataLogWrite && CycleLogWriter) CycleLogWriter.writeToLog(`${StringUtils.safeStringify(cycleObj)}\n`)
     const cycleExist = await queryCycleByMarker(cycleObj.cycleMarker)
     if (cycleExist) {
-      if (JSON.stringify(cycleObj) !== JSON.stringify(cycleExist))
+      if (StringUtils.safeStringify(cycleObj) !== StringUtils.safeStringify(cycleExist))
         await updateCycle(cycleObj.cycleMarker, cycleObj)
     } else {
       // await Cycle.insertCycle(cycleObj)
@@ -885,7 +886,7 @@ export const storeOriginalTxData = async (
     if (missingOriginalTxsMap.has(txId)) missingOriginalTxsMap.delete(txId)
 
     if (config.dataLogWrite && OriginalTxDataLogWriter)
-      OriginalTxDataLogWriter.writeToLog(`${JSON.stringify(originalTxData)}\n`)
+      OriginalTxDataLogWriter.writeToLog(`${StringUtils.safeStringify(originalTxData)}\n`)
     combineOriginalTxsData.push(originalTxData)
     txDataList.push({ txId, timestamp })
     if (combineOriginalTxsData.length >= bucketSize) {