From edc0074f5d366b562334047df488c056572dc87f Mon Sep 17 00:00:00 2001
From: Caralee Jackson <tiedyedsheepengineering@gmail.com>
Date: Tue, 9 Jul 2024 18:20:21 -0500
Subject: [PATCH 01/20] fixing issue comment Pr review stuff

---
 .github/workflows/ci.yml | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bd81bdf2..49ff74bb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,6 +12,16 @@ on:
     branches:
       - dev
       - main
+  issue_comment:
+    inputs:
+      workflowBranch:
+        description: 'Branch of the reusable workflow. Defaults to main, select dev for testing only.'
+        required: true
+        default: 'main'
+        type: choice
+        options:
+          - dev
+          - main
   workflow_dispatch:
     inputs:
       workflowBranch:
@@ -22,11 +32,6 @@ on:
         options:
           - dev
           - main
-
-permissions:
-  issues: write
-  pull-requests: write
-
 jobs:
   echo-inputs:
     name: Repo Workflow Debugging
@@ -44,6 +49,10 @@ jobs:
   ci-dev:
     if: ${{ github.event.inputs.workflowBranch == 'dev' }}
     uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@dev
+    permissions:
+      issues: write
+      pull-requests: write
+      contents: write
     with:
       node-version: ${{ vars.NODE_VERSION }}
       lint-required: ${{ vars.IS_LINT_REQUIRED == 'true' }}
@@ -55,6 +64,10 @@ jobs:
   ci-main:
     if: ${{ github.event.inputs.workflowBranch == 'main' || !github.event.inputs.workflowBranch }}
     uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@main
+    permissions:
+      issues: write
+      pull-requests: write
+      contents: write
     with:
       node-version: ${{ vars.NODE_VERSION }}
       lint-required: ${{ vars.IS_LINT_REQUIRED == 'true' }}

From b8f3dcfe458a7384ad8d7bd96d3c0e693971c2d3 Mon Sep 17 00:00:00 2001
From: Caralee Jackson <tiedyedsheepengineering@gmail.com>
Date: Fri, 19 Jul 2024 12:13:53 -0500
Subject: [PATCH 02/20] add codeowners and sync ci.yml

---
 .github/workflows/ci.yml |  4 ++--
 CODEOWNERS               | 13 +++++++++++++
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 CODEOWNERS

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 49ff74bb..4597d442 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -46,7 +46,7 @@ jobs:
           echo "Apply Patches Required: ${{ vars.IS_APPLY_PATCHES_REQUIRED }}"
           echo "Unit Tests Required: ${{ vars.IS_UNIT_TESTS_REQUIRED }}"
           echo "*** End - Check inputs in repo workflow ***"
-  ci-dev:
+  ci-test-only:
     if: ${{ github.event.inputs.workflowBranch == 'dev' }}
     uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@dev
     permissions:
@@ -61,7 +61,7 @@ jobs:
       unit-tests-required: ${{ vars.IS_UNIT_TESTS_REQUIRED == 'true' }}
     secrets: inherit
 
-  ci-main:
+  ci:
     if: ${{ github.event.inputs.workflowBranch == 'main' || !github.event.inputs.workflowBranch }}
     uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@main
     permissions:
diff --git a/CODEOWNERS b/CODEOWNERS
new file mode 100644
index 00000000..200b7372
--- /dev/null
+++ b/CODEOWNERS
@@ -0,0 +1,13 @@
+# CODEOWNERS file
+# To add additional teams to any approval, include them on the same line separated by spaces
+# It is best practice to assign a team as a code owner and not an invidual.
+# Please submit requests for new teams to Systems and Automation
+
+# Global approval (all files)
+# * @shardeum/team-name
+
+# Directory-level approval
+/.github/ @shardeum/systems-and-automation
+
+# Specific file rules
+# README.md @shardeum/team-name

From 318f403b3c16d911fb413626664977755322971b Mon Sep 17 00:00:00 2001
From: Caralee Jackson <tiedyedsheepengineering@gmail.com>
Date: Fri, 19 Jul 2024 12:39:16 -0500
Subject: [PATCH 03/20] fix lint and add test scripts

---
 package.json | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/package.json b/package.json
index a3bf835b..049da770 100644
--- a/package.json
+++ b/package.json
@@ -17,12 +17,11 @@
   "scripts": {
     "start": "npm run prepare && node build/server.js",
     "release": "npm run prepare && np --no-cleanup --no-tests --no-yarn --any-branch",
-    "test": "echo \"Error: no test specified\" && exit 1",
     "check": "gts check",
     "clean": "npm-run-all clean:*",
     "clean:typescript": "gts clean",
-    "lint": "eslint './src/**/*.ts'",
-    "lint-windows": "eslint ./src/**/*.ts",
+    "lint": "eslint \"./src/**/*.ts\"",
+    "test": "echo \"Error: no test specified\" && exit 1",
     "format-check": "prettier --check './src/**/*.ts'",
     "format-fix": "prettier --write './src/**/*.ts'",
     "clean:artifacts": "shx rm -rf archiver-logs/ archiver-db/ data-logs/",

From 776a128a3af48f0537ba4138c536294aa14d9a9d Mon Sep 17 00:00:00 2001
From: Caralee Jackson <tiedyedsheepengineering@gmail.com>
Date: Fri, 19 Jul 2024 12:50:30 -0500
Subject: [PATCH 04/20] prettier fixes

---
 src/dbstore/types.ts            | 12 ++++++------
 src/profiler/StringifyReduce.ts |  2 +-
 src/sync-v2/queries.ts          |  4 +++-
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/dbstore/types.ts b/src/dbstore/types.ts
index 179bc525..fabc7ef0 100644
--- a/src/dbstore/types.ts
+++ b/src/dbstore/types.ts
@@ -1,11 +1,11 @@
-import { P2P, StateManager } from "@shardus/types"
+import { P2P, StateManager } from '@shardus/types'
 
 export interface Cycle {
-    counter: P2P.CycleCreatorTypes.CycleData['counter']
-    cycleRecord: P2P.CycleCreatorTypes.CycleData
-    cycleMarker: StateManager.StateMetaDataTypes.CycleMarker
+  counter: P2P.CycleCreatorTypes.CycleData['counter']
+  cycleRecord: P2P.CycleCreatorTypes.CycleData
+  cycleMarker: StateManager.StateMetaDataTypes.CycleMarker
 }
 
 export type DbCycle = Cycle & {
-    cycleRecord: string
-}
\ No newline at end of file
+  cycleRecord: string
+}
diff --git a/src/profiler/StringifyReduce.ts b/src/profiler/StringifyReduce.ts
index f0092ee8..6f24eb0a 100644
--- a/src/profiler/StringifyReduce.ts
+++ b/src/profiler/StringifyReduce.ts
@@ -1,4 +1,4 @@
-import { Utils as StringUtils } from "@shardus/types"
+import { Utils as StringUtils } from '@shardus/types'
 
 export const makeShortHash = (x: string, n = 4): string => {
   if (!x) {
diff --git a/src/sync-v2/queries.ts b/src/sync-v2/queries.ts
index 46e1b96b..82731e86 100644
--- a/src/sync-v2/queries.ts
+++ b/src/sync-v2/queries.ts
@@ -119,7 +119,9 @@ function attemptSimpleFetch<T>(
 }
 
 /** Executes a robust query to retrieve the cycle marker from the network. */
-export function robustQueryForCycleRecordHash(nodes: ActiveNode[]): RobustQueryResultAsync<{ currentCycleHash: hexstring }> {
+export function robustQueryForCycleRecordHash(
+  nodes: ActiveNode[]
+): RobustQueryResultAsync<{ currentCycleHash: hexstring }> {
   return makeRobustQueryCall(nodes, 'current-cycle-hash')
 }
 

From 01faa5e87e0792afb1a97b8345b5b15eddf9dbb7 Mon Sep 17 00:00:00 2001
From: Caralee Jackson <tiedyedsheepengineering@gmail.com>
Date: Fri, 19 Jul 2024 19:08:54 -0500
Subject: [PATCH 05/20] cleanup gitlab.ci

---
 .gitlab-ci.yml | 69 --------------------------------------------------
 1 file changed, 69 deletions(-)
 delete mode 100644 .gitlab-ci.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
deleted file mode 100644
index 1de80e8e..00000000
--- a/.gitlab-ci.yml
+++ /dev/null
@@ -1,69 +0,0 @@
-# You can override the included template(s) by including variable overrides
-# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
-# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
-# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
-# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings
-# Note that environment variables can be set in several places
-# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
-
-# Include security-related templates
-include:
-  - template: Security/Dependency-Scanning.gitlab-ci.yml
-  - template: Security/Secret-Detection.gitlab-ci.yml
-  - template: Security/SAST.gitlab-ci.yml
-  - remote: 'https://gitlab.com/pod_security/shared-ci/-/raw/main/security.yml'
-
-# Define the default Docker image for all jobs
-image: registry.gitlab.com/shardus/dev-container
-
-# Define global cache settings for all jobs
-cache:
-  key: '$CI_COMMIT_REF_SLUG-node-modules'
-  paths:
-    - node_modules/
-
-# Define the stages for the pipeline
-stages:
-  - prepare
-  - build
-  - appsec
-  - test
-
-# Prepare Job: Install Node.js dependencies
-prepare-job:
-  stage: prepare
-  script:
-    - npm ci
-
-# Build Job: Compiles the code
-compile-job:
-  stage: build
-  needs: ['prepare-job']
-  script:
-    - echo "Running Compiler..."
-    - npm run compile
-    - echo "Compilation complete."
-
-# Format Checker Job: Runs Prettier for code formatting
-format-checker-job:
-  stage: build
-  needs: ['prepare-job']
-  script:
-    - echo "Running Prettier..."
-    - npm run format-check
-    - echo "Running Prettier complete."
-
-# Lint Checker Job: Runs ESlint for code linting
-lint-checker-job:
-  stage: build
-  needs: ['prepare-job']
-  script:
-    - echo "Running ESlint..."
-    - npm run lint
-    - echo "Running ESlint complete."
-
-# SAST Job: Performs static application security testing
-sast:
-  variables:
-    SAST_EXCLUDED_ANALYZERS: bandit, brakeman, flawfinder, gosec, kubesec, phpcs-security-audit, pmd-apex, security-code-scan, semgrep, sobelow, spotbugs
-  stage: test

From 957856d71b12ad19a2e0aa9d8321d157f78a227f Mon Sep 17 00:00:00 2001
From: Caralee Jackson <tiedyedsheepengineering@gmail.com>
Date: Mon, 22 Jul 2024 12:19:25 -0500
Subject: [PATCH 06/20] Revert "prettier fixes"

This reverts commit 776a128a3af48f0537ba4138c536294aa14d9a9d.
---
 src/dbstore/types.ts            | 12 ++++++------
 src/profiler/StringifyReduce.ts |  2 +-
 src/sync-v2/queries.ts          |  4 +---
 3 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/dbstore/types.ts b/src/dbstore/types.ts
index fabc7ef0..179bc525 100644
--- a/src/dbstore/types.ts
+++ b/src/dbstore/types.ts
@@ -1,11 +1,11 @@
-import { P2P, StateManager } from '@shardus/types'
+import { P2P, StateManager } from "@shardus/types"
 
 export interface Cycle {
-  counter: P2P.CycleCreatorTypes.CycleData['counter']
-  cycleRecord: P2P.CycleCreatorTypes.CycleData
-  cycleMarker: StateManager.StateMetaDataTypes.CycleMarker
+    counter: P2P.CycleCreatorTypes.CycleData['counter']
+    cycleRecord: P2P.CycleCreatorTypes.CycleData
+    cycleMarker: StateManager.StateMetaDataTypes.CycleMarker
 }
 
 export type DbCycle = Cycle & {
-  cycleRecord: string
-}
+    cycleRecord: string
+}
\ No newline at end of file
diff --git a/src/profiler/StringifyReduce.ts b/src/profiler/StringifyReduce.ts
index 6f24eb0a..f0092ee8 100644
--- a/src/profiler/StringifyReduce.ts
+++ b/src/profiler/StringifyReduce.ts
@@ -1,4 +1,4 @@
-import { Utils as StringUtils } from '@shardus/types'
+import { Utils as StringUtils } from "@shardus/types"
 
 export const makeShortHash = (x: string, n = 4): string => {
   if (!x) {
diff --git a/src/sync-v2/queries.ts b/src/sync-v2/queries.ts
index 82731e86..46e1b96b 100644
--- a/src/sync-v2/queries.ts
+++ b/src/sync-v2/queries.ts
@@ -119,9 +119,7 @@ function attemptSimpleFetch<T>(
 }
 
 /** Executes a robust query to retrieve the cycle marker from the network. */
-export function robustQueryForCycleRecordHash(
-  nodes: ActiveNode[]
-): RobustQueryResultAsync<{ currentCycleHash: hexstring }> {
+export function robustQueryForCycleRecordHash(nodes: ActiveNode[]): RobustQueryResultAsync<{ currentCycleHash: hexstring }> {
   return makeRobustQueryCall(nodes, 'current-cycle-hash')
 }
 

From 5f61735113b436c4bfcbc9a4ea651f2c0e2e98f8 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Thu, 4 Jul 2024 20:02:35 +0545
Subject: [PATCH 07/20] Added the voteHashCalculation for new POQ-O receipt

---
 src/Config.ts         |  2 ++
 src/Data/Collector.ts | 27 ++++++++++++++++++++++++++-
 src/Data/Cycles.ts    |  6 +++---
 3 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/src/Config.ts b/src/Config.ts
index a8120034..9fcd6049 100644
--- a/src/Config.ts
+++ b/src/Config.ts
@@ -65,6 +65,7 @@ export interface Config {
   saveOnlyGossipData: boolean
   // For debugging purpose, set this to true to stop gossiping tx data
   stopGossipTxData: boolean
+  usePOQo: boolean
 }
 
 let config: Config = {
@@ -126,6 +127,7 @@ let config: Config = {
   extraConsensorsToSubscribe: 1,
   saveOnlyGossipData: false,
   stopGossipTxData: false,
+  usePOQo: true,
 }
 // Override default config params from config file, env vars, and cli args
 export async function overrideDefaultConfig(file: string): Promise<void> {
diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts
index 33956898..f45dce5d 100644
--- a/src/Data/Collector.ts
+++ b/src/Data/Collector.ts
@@ -581,7 +581,32 @@ export const verifyReceiptData = async (
 }
 
 const calculateVoteHash = (vote: Receipt.AppliedVote): string => {
-  return Crypto.hashObj({ ...vote, node_id: '' })
+  try {
+    if (config.usePOQo === true) {
+      const appliedHash = {
+        applied: vote.transaction_result,
+        cantApply: vote.cant_apply,
+      }
+      const stateHash = {
+        account_id: vote.account_id,
+        account_state_hash_after: vote.account_state_hash_after,
+        account_state_hash_before: vote.account_state_hash_before,
+      }
+      const appDataHash = {
+        app_data_hash: vote.app_data_hash,
+      }
+      const voteToHash = {
+        appliedHash: Crypto.hashObj(appliedHash),
+        stateHash: Crypto.hashObj(stateHash),
+        appDataHash: Crypto.hashObj(appDataHash),
+      }
+      return Crypto.hashObj(voteToHash)
+    }
+    return Crypto.hashObj({ ...vote, node_id: '' })
+  } catch {
+    Logger.mainLogger.error('Error in calculateVoteHash', vote)
+    return ''
+  }
 }
 
 export const storeReceiptData = async (
diff --git a/src/Data/Cycles.ts b/src/Data/Cycles.ts
index 005eb41f..25748446 100644
--- a/src/Data/Cycles.ts
+++ b/src/Data/Cycles.ts
@@ -52,7 +52,7 @@ export let cycleRecordWithShutDownMode = null as P2PTypes.CycleCreatorTypes.Cycl
 export let currentNetworkMode: P2PTypes.ModesTypes.Record['mode'] = 'forming'
 export const shardValuesByCycle = new Map<number, StateManager.shardFunctionTypes.CycleShardData>()
 
-const CYCLE_SHARD_STORAGE_LIMIT = 3
+const CYCLE_SHARD_STORAGE_LIMIT = 20
 
 export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData[]): Promise<void> {
   if (profilerInstance) profilerInstance.profileSectionStart('process_cycle', false)
@@ -93,8 +93,8 @@ export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData
         setShutdownCycleRecord(cycle)
         NodeList.toggleFirstNode()
       }
-      // Clean receipts/originalTxs cache that are older than 5 minutes
-      const cleanupTimestamp = Date.now() - 5 * 60 * 1000
+      // Clean receipts/originalTxs cache that are older than 20 minutes to match with CYCLE_SHARD_STORAGE_LIMIT ( actual fix is on another branch )
+      const cleanupTimestamp = Date.now() - 20 * 60 * 1000
       cleanOldOriginalTxsMap(cleanupTimestamp)
       cleanOldReceiptsMap(cleanupTimestamp)
     }

From 4bd1610a4781b35af00f7412a1bd84b25e7a97aa Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Tue, 9 Jul 2024 15:21:48 +0545
Subject: [PATCH 08/20] Added requiredVotesPercentage config

---
 src/Config.ts         | 3 +++
 src/Data/Collector.ts | 5 ++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/Config.ts b/src/Config.ts
index 9fcd6049..5641109d 100644
--- a/src/Config.ts
+++ b/src/Config.ts
@@ -66,6 +66,8 @@ export interface Config {
   // For debugging purpose, set this to true to stop gossiping tx data
   stopGossipTxData: boolean
   usePOQo: boolean
+  // The percentage of votes required to confirm transaction
+  requiredVotesPercentage: number
 }
 
 let config: Config = {
@@ -128,6 +130,7 @@ let config: Config = {
   saveOnlyGossipData: false,
   stopGossipTxData: false,
   usePOQo: true,
+  requiredVotesPercentage: 2 / 3,
 }
 // Override default config params from config file, env vars, and cli args
 export async function overrideDefaultConfig(file: string): Promise<void> {
diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts
index f45dce5d..6f7b4f9c 100644
--- a/src/Data/Collector.ts
+++ b/src/Data/Collector.ts
@@ -405,7 +405,10 @@ export const verifyReceiptData = async (
     if (votingGroupCount > cycleShardData.nodes.length) {
       votingGroupCount = cycleShardData.nodes.length
     }
-    const requiredSignatures = Math.round(votingGroupCount * (2 / 3))
+    const requiredSignatures =
+      config.usePOQo === true
+        ? Math.ceil(votingGroupCount * config.requiredVotesPercentage)
+        : Math.round(votingGroupCount * config.requiredVotesPercentage)
     if (signatures.length < requiredSignatures) {
       Logger.mainLogger.error(
         `Invalid receipt appliedReceipt signatures count is less than requiredSignatures, ${signatures.length}, ${requiredSignatures}`

From 1feccc0364c3df94edf8187bbab2e33643dc6245 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Fri, 19 Jul 2024 18:20:42 +0545
Subject: [PATCH 09/20] Updated to use single index for each field

---
 src/dbstore/index.ts | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/dbstore/index.ts b/src/dbstore/index.ts
index 9d5f8977..bed6256d 100644
--- a/src/dbstore/index.ts
+++ b/src/dbstore/index.ts
@@ -7,7 +7,10 @@ export const initializeDB = async (config: Config): Promise<void> => {
     'CREATE TABLE if not exists `transactions` (`txId` TEXT NOT NULL UNIQUE PRIMARY KEY, `appReceiptId` TEXT, `timestamp` BIGINT NOT NULL, `cycleNumber` NUMBER NOT NULL, `data` JSON NOT NULL, `originalTxData` JSON NOT NULL)'
   )
   await db.runCreate(
-    'CREATE INDEX if not exists `transactions_idx` ON `transactions` (`cycleNumber` DESC, `timestamp` DESC)'
+    'CREATE INDEX if not exists `transactions_cycleNumber` ON `transactions` (`cycleNumber` DESC)'
+  )
+  await db.runCreate(
+    'CREATE INDEX if not exists `transactions_timestamp` ON `transactions` (`timestamp` DESC)'
   )
   await db.runCreate(
     'CREATE INDEX if not exists `transactions_appReceiptId_idx` ON `transactions` (`appReceiptId`)'
@@ -19,19 +22,20 @@ export const initializeDB = async (config: Config): Promise<void> => {
   await db.runCreate(
     'CREATE TABLE if not exists `accounts` (`accountId` TEXT NOT NULL UNIQUE PRIMARY KEY, `data` JSON NOT NULL, `timestamp` BIGINT NOT NULL, `hash` TEXT NOT NULL, `cycleNumber` NUMBER NOT NULL, `isGlobal` BOOLEAN NOT NULL)'
   )
-  await db.runCreate(
-    'CREATE INDEX if not exists `accounts_idx` ON `accounts` (`cycleNumber` DESC, `timestamp` DESC)'
-  )
+  await db.runCreate('CREATE INDEX if not exists `accounts_cycleNumber` ON `accounts` (`cycleNumber` DESC)')
+  await db.runCreate('CREATE INDEX if not exists `accounts_timestamp` ON `accounts` (`timestamp` DESC)')
   await db.runCreate(
     'CREATE TABLE if not exists `receipts` (`receiptId` TEXT NOT NULL UNIQUE PRIMARY KEY, `tx` JSON NOT NULL, `cycle` NUMBER NOT NULL, `timestamp` BIGINT NOT NULL, `beforeStateAccounts` JSON, `accounts` JSON NOT NULL, `appliedReceipt` JSON NOT NULL, `appReceiptData` JSON, `executionShardKey` TEXT NOT NULL, `globalModification` BOOLEAN NOT NULL)'
   )
-  await db.runCreate('CREATE INDEX if not exists `receipts_idx` ON `receipts` (`cycle` ASC, `timestamp` ASC)')
+  await db.runCreate('CREATE INDEX if not exists `receipts_cycle` ON `receipts` (`cycle` ASC)')
+  await db.runCreate('CREATE INDEX if not exists `receipts_timestamp` ON `receipts` (`timestamp` ASC)')
+
   await db.runCreate(
     'CREATE TABLE if not exists `originalTxsData` (`txId` TEXT NOT NULL, `timestamp` BIGINT NOT NULL, `cycle` NUMBER NOT NULL, `originalTxData` JSON NOT NULL, PRIMARY KEY (`txId`, `timestamp`))'
   )
-  // await db.runCreate('Drop INDEX if exists `originalTxData_idx`');
+  await db.runCreate('CREATE INDEX if not exists `originalTxsData_cycle` ON `originalTxsData` (`cycle` ASC)')
   await db.runCreate(
-    'CREATE INDEX if not exists `originalTxsData_idx` ON `originalTxsData` (`cycle` ASC, `timestamp` ASC)'
+    'CREATE INDEX if not exists `originalTxsData_timestamp` ON `originalTxsData` (`timestamp` ASC)'
   )
   await db.runCreate('CREATE INDEX if not exists `originalTxsData_txId_idx` ON `originalTxsData` (`txId`)')
 }

From 9916d909f6bfe59c1672a839644943020cac3ab0 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Fri, 19 Jul 2024 19:51:55 +0545
Subject: [PATCH 10/20] rename originalTxsData_txId index

---
 src/dbstore/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/dbstore/index.ts b/src/dbstore/index.ts
index bed6256d..1bb31108 100644
--- a/src/dbstore/index.ts
+++ b/src/dbstore/index.ts
@@ -37,7 +37,7 @@ export const initializeDB = async (config: Config): Promise<void> => {
   await db.runCreate(
     'CREATE INDEX if not exists `originalTxsData_timestamp` ON `originalTxsData` (`timestamp` ASC)'
   )
-  await db.runCreate('CREATE INDEX if not exists `originalTxsData_txId_idx` ON `originalTxsData` (`txId`)')
+  await db.runCreate('CREATE INDEX if not exists `originalTxsData_txId` ON `originalTxsData` (`txId`)')
 }
 
 export const closeDatabase = async (): Promise<void> => {

From 0512864e597827f54bdb47fc0a7bbda48286f317 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Fri, 19 Jul 2024 20:11:18 +0545
Subject: [PATCH 11/20] Updated to use ascending order index

---
 src/dbstore/index.ts | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/dbstore/index.ts b/src/dbstore/index.ts
index 1bb31108..0e5ae727 100644
--- a/src/dbstore/index.ts
+++ b/src/dbstore/index.ts
@@ -7,10 +7,10 @@ export const initializeDB = async (config: Config): Promise<void> => {
     'CREATE TABLE if not exists `transactions` (`txId` TEXT NOT NULL UNIQUE PRIMARY KEY, `appReceiptId` TEXT, `timestamp` BIGINT NOT NULL, `cycleNumber` NUMBER NOT NULL, `data` JSON NOT NULL, `originalTxData` JSON NOT NULL)'
   )
   await db.runCreate(
-    'CREATE INDEX if not exists `transactions_cycleNumber` ON `transactions` (`cycleNumber` DESC)'
+    'CREATE INDEX if not exists `transactions_cycleNumber` ON `transactions` (`cycleNumber` ASC)'
   )
   await db.runCreate(
-    'CREATE INDEX if not exists `transactions_timestamp` ON `transactions` (`timestamp` DESC)'
+    'CREATE INDEX if not exists `transactions_timestamp` ON `transactions` (`timestamp` ASC)'
   )
   await db.runCreate(
     'CREATE INDEX if not exists `transactions_appReceiptId_idx` ON `transactions` (`appReceiptId`)'
@@ -18,12 +18,12 @@ export const initializeDB = async (config: Config): Promise<void> => {
   await db.runCreate(
     'CREATE TABLE if not exists `cycles` (`cycleMarker` TEXT NOT NULL UNIQUE PRIMARY KEY, `counter` NUMBER NOT NULL, `cycleRecord` JSON NOT NULL)'
   )
-  await db.runCreate('CREATE INDEX if not exists `cycles_idx` ON `cycles` (`counter` DESC)')
+  await db.runCreate('CREATE INDEX if not exists `cycles_idx` ON `cycles` (`counter` ASC)')
   await db.runCreate(
     'CREATE TABLE if not exists `accounts` (`accountId` TEXT NOT NULL UNIQUE PRIMARY KEY, `data` JSON NOT NULL, `timestamp` BIGINT NOT NULL, `hash` TEXT NOT NULL, `cycleNumber` NUMBER NOT NULL, `isGlobal` BOOLEAN NOT NULL)'
   )
-  await db.runCreate('CREATE INDEX if not exists `accounts_cycleNumber` ON `accounts` (`cycleNumber` DESC)')
-  await db.runCreate('CREATE INDEX if not exists `accounts_timestamp` ON `accounts` (`timestamp` DESC)')
+  await db.runCreate('CREATE INDEX if not exists `accounts_cycleNumber` ON `accounts` (`cycleNumber` ASC)')
+  await db.runCreate('CREATE INDEX if not exists `accounts_timestamp` ON `accounts` (`timestamp` ASC)')
   await db.runCreate(
     'CREATE TABLE if not exists `receipts` (`receiptId` TEXT NOT NULL UNIQUE PRIMARY KEY, `tx` JSON NOT NULL, `cycle` NUMBER NOT NULL, `timestamp` BIGINT NOT NULL, `beforeStateAccounts` JSON, `accounts` JSON NOT NULL, `appliedReceipt` JSON NOT NULL, `appReceiptData` JSON, `executionShardKey` TEXT NOT NULL, `globalModification` BOOLEAN NOT NULL)'
   )

From 732ab1e94d046605e4f1d0377b1bb57f9e3feef4 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Fri, 19 Jul 2024 20:22:28 +0545
Subject: [PATCH 12/20] Added db profiler

---
 src/dbstore/sqlite3storage.ts | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/dbstore/sqlite3storage.ts b/src/dbstore/sqlite3storage.ts
index c5302e4b..667ff841 100644
--- a/src/dbstore/sqlite3storage.ts
+++ b/src/dbstore/sqlite3storage.ts
@@ -22,6 +22,13 @@ export async function init(config: Config): Promise<void> {
     }
   })
   await run('PRAGMA journal_mode=WAL')
+  db.on('profile', (sql, time) => {
+    if (time > 500 && time < 1000) {
+      console.log('SLOW QUERY', sql, time)
+    } else if (time > 1000) {
+      console.log('VERY SLOW QUERY', sql, time)
+    }
+  })
   console.log('Database initialized.')
 }
 

From c1c4201e175386105839c10b0403758e03be6b1b Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Thu, 18 Jul 2024 23:41:31 +0545
Subject: [PATCH 13/20] Fix StringUtils issue in the update-network-account
 script

---
 scripts/update_network_account.ts | 1 +
 src/Data/Collector.ts             | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/update_network_account.ts b/scripts/update_network_account.ts
index b72c5974..90cf1671 100644
--- a/scripts/update_network_account.ts
+++ b/scripts/update_network_account.ts
@@ -9,6 +9,7 @@ import { startSaving } from '../src/saveConsoleOutput'
 import * as Logger from '../src/Logger'
 import { accountSpecificHash } from '../src/shardeum/calculateAccountHash'
 import { addSigListeners } from '../src/State'
+import { Utils as StringUtils } from '@shardus/types'
 
 const activeVersion = '1.9.0'
 const latestVersion = '1.9.0'
diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts
index 6f7b4f9c..ec7ee857 100644
--- a/src/Data/Collector.ts
+++ b/src/Data/Collector.ts
@@ -447,7 +447,7 @@ export const verifyReceiptData = async (
       // Check if the node is in the execution group
       if (!cycleShardData.parititionShardDataMap.get(homePartition).coveredBy[node.id]) {
         Logger.mainLogger.error(
-          `The node with public key ${nodePubKey} of the receipt ${txId}} is not in the execution group of the tx`
+          `The node with public key ${nodePubKey} of the receipt ${txId} is not in the execution group of the tx`
         )
         if (nestedCountersInstance)
           nestedCountersInstance.countEvent(

From 7175d87d053c1bb5446ea43e54769e7fac19c3f6 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Mon, 22 Jul 2024 11:55:15 +0545
Subject: [PATCH 14/20] Fix StringUtils missing issue in scripts

---
 scripts/archiver_data_sync_check.ts | 1 +
 scripts/create_shut_down_cycle.ts   | 1 +
 scripts/repair_missing_cycle.ts     | 4 ++--
 scripts/verify_account_hash.ts      | 1 +
 4 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/archiver_data_sync_check.ts b/scripts/archiver_data_sync_check.ts
index 0d5555a4..704bf35d 100644
--- a/scripts/archiver_data_sync_check.ts
+++ b/scripts/archiver_data_sync_check.ts
@@ -4,6 +4,7 @@ import { join } from 'path'
 import { postJson } from '../src/P2P'
 import { config, overrideDefaultConfig } from '../src/Config'
 import { ArchiverNodeInfo } from '../src/State'
+import { Utils as StringUtils } from '@shardus/types'
 
 const configFile = join(process.cwd(), 'archiver-config.json')
 overrideDefaultConfig(configFile)
diff --git a/scripts/create_shut_down_cycle.ts b/scripts/create_shut_down_cycle.ts
index ed793ee6..cf575733 100644
--- a/scripts/create_shut_down_cycle.ts
+++ b/scripts/create_shut_down_cycle.ts
@@ -10,6 +10,7 @@ import * as Logger from '../src/Logger'
 import { P2P } from '@shardus/types'
 import { addSigListeners } from '../src/State'
 import { computeCycleMarker } from '../src/Data/Cycles'
+import { Utils as StringUtils } from '@shardus/types'
 
 const archiversAtShutdown = [
   {
diff --git a/scripts/repair_missing_cycle.ts b/scripts/repair_missing_cycle.ts
index 43188aea..a35b657d 100644
--- a/scripts/repair_missing_cycle.ts
+++ b/scripts/repair_missing_cycle.ts
@@ -1,6 +1,5 @@
 import { readFileSync } from 'fs'
-import { resolve } from 'path'
-import { join } from 'path'
+import { resolve, join } from 'path'
 import { overrideDefaultConfig, config } from '../src/Config'
 import * as Crypto from '../src/Crypto'
 import * as db from '../src/dbstore/sqlite3storage'
@@ -8,6 +7,7 @@ import * as dbstore from '../src/dbstore'
 import * as CycleDB from '../src/dbstore/cycles'
 import { startSaving } from '../src/saveConsoleOutput'
 import * as Logger from '../src/Logger'
+import { Utils as StringUtils } from '@shardus/types'
 
 const patchCycleData = false
 
diff --git a/scripts/verify_account_hash.ts b/scripts/verify_account_hash.ts
index 5eaf8686..edd760d3 100644
--- a/scripts/verify_account_hash.ts
+++ b/scripts/verify_account_hash.ts
@@ -9,6 +9,7 @@ import { startSaving } from '../src/saveConsoleOutput'
 import * as Logger from '../src/Logger'
 import { AccountType, fixAccountUint8Arrays, accountSpecificHash } from '../src/shardeum/calculateAccountHash'
 import { addSigListeners } from '../src/State'
+import { Utils as StringUtils } from '@shardus/types'
 
 const updateHash = false
 const runProgram = async (): Promise<void> => {

From 1821e3fe5fffdfedfa67088a7cf6a2bda1969971 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Wed, 26 Jun 2024 22:56:57 +0545
Subject: [PATCH 15/20] - Restricted to only the first validator can forward
 the accounts data to the archiver during early cycles - Remove accounts
 transformation used in account verification

---
 src/Data/Collector.ts                | 39 ++++++++++++----
 src/Data/Data.ts                     | 70 +++++++++++++++-------------
 src/shardeum/calculateAccountHash.ts | 21 ---------
 3 files changed, 67 insertions(+), 63 deletions(-)

diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts
index ec7ee857..a26d233d 100644
--- a/src/Data/Collector.ts
+++ b/src/Data/Collector.ts
@@ -26,7 +26,7 @@ import { CycleLogWriter, ReceiptLogWriter, OriginalTxDataLogWriter } from '../Da
 import * as OriginalTxDB from '../dbstore/originalTxsData'
 import ShardFunction from '../ShardFunctions'
 import { ConsensusNodeInfo } from '../NodeList'
-import { verifyAccountHash } from '../shardeum/calculateAccountHash'
+import { accountSpecificHash, verifyAccountHash } from '../shardeum/calculateAccountHash'
 import { verifyAppReceiptData } from '../shardeum/verifyAppReceiptData'
 import { Cycle as DbCycle } from '../dbstore/types'
 import { Utils as StringUtils } from '@shardus/types'
@@ -927,12 +927,11 @@ interface StoreAccountParam {
 }
 
 export const storeAccountData = async (restoreData: StoreAccountParam = {}): Promise<void> => {
-  console.log(
-    'RestoreData',
-    'accounts',
-    restoreData.accounts ? restoreData.accounts.length : 0,
-    'receipts',
-    restoreData.receipts ? restoreData.receipts.length : 0
+  Logger.mainLogger.debug(
+    `storeAccountData: ${restoreData.accounts ? restoreData.accounts.length : 0} accounts`
+  )
+  Logger.mainLogger.debug(
+    `storeAccountData: ${restoreData.receipts ? restoreData.receipts.length : 0} receipts`
   )
   const { accounts, receipts } = restoreData
   if (profilerInstance) profilerInstance.profileSectionStart('store_account_data')
@@ -953,7 +952,28 @@ export const storeAccountData = async (restoreData: StoreAccountParam = {}): Pro
   //   //   await Account.insertAccount(account)
   //   // }
   // }
-  if (accounts && accounts.length > 0) await Account.bulkInsertAccounts(accounts)
+  //
+  if (accounts && accounts.length > 0) {
+    const combineAccounts = []
+    for (const account of accounts) {
+      try {
+        const calculatedAccountHash = accountSpecificHash(account.data)
+        if (calculatedAccountHash !== account.hash) {
+          Logger.mainLogger.error(
+            'Invalid account hash',
+            account.accountId,
+            account.hash,
+            calculatedAccountHash
+          )
+          continue
+        }
+        combineAccounts.push(account)
+      } catch (error) {
+        Logger.mainLogger.error('Error in calculating genesis account hash', error)
+      }
+    }
+    if (combineAccounts.length > 0) await Account.bulkInsertAccounts(accounts)
+  }
   if (receipts && receipts.length > 0) {
     Logger.mainLogger.debug('Received receipts Size', receipts.length)
     const combineTransactions = []
@@ -971,10 +991,9 @@ export const storeAccountData = async (restoreData: StoreAccountParam = {}): Pro
     await Transaction.bulkInsertTransactions(combineTransactions)
   }
   if (profilerInstance) profilerInstance.profileSectionEnd('store_account_data')
-  console.log('Combined Accounts Data', combineAccountsData.accounts.length)
   Logger.mainLogger.debug('Combined Accounts Data', combineAccountsData.accounts.length)
   if (combineAccountsData.accounts.length > 0 || combineAccountsData.receipts.length > 0) {
-    console.log('Found combine accountsData')
+    Logger.mainLogger.debug('Found combine accountsData', combineAccountsData.accounts.length)
     const accountData = { ...combineAccountsData }
     clearCombinedAccountsData()
     storeAccountData(accountData)
diff --git a/src/Data/Data.ts b/src/Data/Data.ts
index f74bd1f5..a3de8cef 100644
--- a/src/Data/Data.ts
+++ b/src/Data/Data.ts
@@ -63,6 +63,11 @@ const {
   MAX_BETWEEN_CYCLES_PER_REQUEST,
 } = config.REQUEST_LIMIT
 
+const GENESIS_ACCOUNTS_CYCLE_RANGE = {
+  startCycle: 0,
+  endCycle: 5,
+}
+
 export enum DataRequestTypes {
   SUBSCRIBE = 'SUBSCRIBE',
   UNSUBSCRIBE = 'UNSUBSCRIBE',
@@ -280,26 +285,30 @@ export function initSocketClient(node: NodeList.ConsensusNodeInfo): void {
         collectCycleData(newData.responses.CYCLE, sender.nodeInfo.ip + ':' + sender.nodeInfo.port)
       }
       if (newData.responses && newData.responses.ACCOUNT) {
-        console.log(
-          'RECEIVED ACCOUNTS DATA',
-          sender.nodeInfo.publicKey,
-          sender.nodeInfo.ip,
-          sender.nodeInfo.port
-        )
-        Logger.mainLogger.debug(
-          'RECEIVED ACCOUNTS DATA',
-          sender.nodeInfo.publicKey,
-          sender.nodeInfo.ip,
-          sender.nodeInfo.port
-        )
+        if (getCurrentCycleCounter() > GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle) {
+          Logger.mainLogger.error(
+            'Account data is not meant to be received after the genesis cycle',
+            getCurrentCycleCounter()
+          )
+          unsubscribeDataSender(sender.nodeInfo.publicKey)
+          return
+        }
+        if (NodeList.byPublicKey.size > 1 || !NodeList.byPublicKey.has(sender.nodeInfo.publicKey)) {
+          Logger.mainLogger.error(
+            'Account data is not meant to be received by the first validator',
+            `Number of nodes in the network ${NodeList.byPublicKey.size}`
+          )
+          unsubscribeDataSender(sender.nodeInfo.publicKey)
+          return
+        }
+        Logger.mainLogger.debug(`RECEIVED ACCOUNTS DATA FROM ${sender.nodeInfo.ip}:${sender.nodeInfo.port}`)
         nestedCountersInstance.countEvent('genesis', 'accounts', 1)
         if (!forwardGenesisAccounts) {
-          console.log('Genesis Accounts To Sycn', newData.responses.ACCOUNT)
           Logger.mainLogger.debug('Genesis Accounts To Sycn', newData.responses.ACCOUNT)
           syncGenesisAccountsFromConsensor(newData.responses.ACCOUNT, sender.nodeInfo)
         } else {
           if (storingAccountData) {
-            console.log('Storing Data')
+            Logger.mainLogger.debug('Storing Account Data')
             let newCombineAccountsData = { ...combineAccountsData }
             if (newData.responses.ACCOUNT.accounts)
               newCombineAccountsData.accounts = [
@@ -321,17 +330,14 @@ export function initSocketClient(node: NodeList.ConsensusNodeInfo): void {
       }
 
       // Set new contactTimeout for sender. Postpone sender removal because data is still received from consensor
-      if (currentCycleDuration > 0) {
-        nestedCountersInstance.countEvent('archiver', 'postpone_contact_timeout')
-        // To make sure that the sender is still in the subscribed list
-        sender = dataSenders.get(newData.publicKey)
-        if (sender)
-          sender.contactTimeout = createContactTimeout(
-            sender.nodeInfo.publicKey,
-            'This timeout is created after processing data'
-          )
-      }
-      return
+      nestedCountersInstance.countEvent('archiver', 'postpone_contact_timeout')
+      // To make sure that the sender is still in the subscribed list
+      sender = dataSenders.get(newData.publicKey)
+      if (sender)
+        sender.contactTimeout = createContactTimeout(
+          sender.nodeInfo.publicKey,
+          'This timeout is created after processing data'
+        )
     }
   })
 }
@@ -1009,7 +1015,7 @@ export async function syncGenesisAccountsFromArchiver(): Promise<void> {
   // }
   const res = (await queryFromArchivers(
     RequestDataType.ACCOUNT,
-    { startCycle: 0, endCycle: 5 },
+    { startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle, endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle },
     QUERY_TIMEOUT_MAX
   )) as ArchiverAccountResponse
   if (res && (res.totalAccounts || res.totalAccounts === 0)) {
@@ -1026,8 +1032,8 @@ export async function syncGenesisAccountsFromArchiver(): Promise<void> {
     const response = (await queryFromArchivers(
       RequestDataType.ACCOUNT,
       {
-        startCycle: 0,
-        endCycle: 5,
+        startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle,
+        endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle,
         page,
       },
       QUERY_TIMEOUT_MAX
@@ -1059,8 +1065,8 @@ export async function syncGenesisTransactionsFromArchiver(): Promise<void> {
   const res = (await queryFromArchivers(
     RequestDataType.TRANSACTION,
     {
-      startCycle: 0,
-      endCycle: 5,
+      startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle,
+      endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle,
     },
     QUERY_TIMEOUT_MAX
   )) as ArchiverTransactionResponse
@@ -1078,8 +1084,8 @@ export async function syncGenesisTransactionsFromArchiver(): Promise<void> {
     const response = (await queryFromArchivers(
       RequestDataType.TRANSACTION,
       {
-        startCycle: 0,
-        endCycle: 5,
+        startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle,
+        endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle,
         page,
       },
       QUERY_TIMEOUT_MAX
diff --git a/src/shardeum/calculateAccountHash.ts b/src/shardeum/calculateAccountHash.ts
index 61e6853e..326d08f9 100644
--- a/src/shardeum/calculateAccountHash.ts
+++ b/src/shardeum/calculateAccountHash.ts
@@ -57,17 +57,6 @@ export const accountSpecificHash = (account: any): string => {
   return hash
 }
 
-// Converting the correct account data format to get the correct hash
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export const fixAccountUint8Arrays = (account: any): void => {
-  if (!account) return // if account is null, return
-  if (account.storageRoot) account.storageRoot = Uint8Array.from(Object.values(account.storageRoot)) // Account
-  if (account.codeHash) account.codeHash = Uint8Array.from(Object.values(account.codeHash)) //
-  //Account and ContractCode
-  if (account.codeByte) account.codeByte = Uint8Array.from(Object.values(account.codeByte)) // ContractCode
-  if (account.value) account.value = Uint8Array.from(Object.values(account.value)) // ContractByte
-}
-
 export const verifyAccountHash = (receipt: ArchiverReceipt): boolean => {
   try {
     if (receipt.globalModification && config.skipGlobalTxReceiptVerification) return true // return true if global modification
@@ -87,16 +76,6 @@ export const verifyAccountHash = (receipt: ArchiverReceipt): boolean => {
       return false
     }
     for (const account of receipt.accounts) {
-      if (account.data.accountType === AccountType.Account) {
-        fixAccountUint8Arrays(account.data.account)
-        // console.dir(acc, { depth: null })
-      } else if (
-        account.data.accountType === AccountType.ContractCode ||
-        account.data.accountType === AccountType.ContractStorage
-      ) {
-        fixAccountUint8Arrays(account.data)
-        // console.dir(acc, { depth: null })
-      }
       const calculatedAccountHash = accountSpecificHash(account.data)
       const indexOfAccount = receipt.appliedReceipt.appliedVote.account_id.indexOf(account.accountId)
       if (indexOfAccount === -1) {

From 7c545c91e22246219ad29dfd5e6ffc4c82586b59 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Mon, 22 Jul 2024 12:11:30 +0545
Subject: [PATCH 16/20] Remove fixAccountUint8Arrays from the scripts

---
 scripts/validate_archiver_receipt.ts | 12 +-----------
 scripts/verify_account_hash.ts       | 12 +-----------
 src/Data/Data.ts                     |  8 ++++++--
 3 files changed, 8 insertions(+), 24 deletions(-)

diff --git a/scripts/validate_archiver_receipt.ts b/scripts/validate_archiver_receipt.ts
index a7682f02..1785fef3 100644
--- a/scripts/validate_archiver_receipt.ts
+++ b/scripts/validate_archiver_receipt.ts
@@ -3,7 +3,7 @@ import { overrideDefaultConfig, config } from '../src/Config'
 import * as Crypto from '../src/Crypto'
 import * as Utils from '../src/Utils'
 import * as Receipt from '../src/dbstore/receipts'
-import { AccountType, accountSpecificHash, fixAccountUint8Arrays } from '../src/shardeum/calculateAccountHash'
+import { AccountType, accountSpecificHash } from '../src/shardeum/calculateAccountHash'
 import { ShardeumReceipt } from '../src/shardeum/verifyAppReceiptData'
 
 // Add the full receipt data here
@@ -243,16 +243,6 @@ export const verifyAccountHash = (receipt: Receipt.ArchiverReceipt): boolean =>
   try {
     if (receipt.globalModification && config.skipGlobalTxReceiptVerification) return true // return true if global modification
     for (const account of receipt.accounts) {
-      if (account.data.accountType === AccountType.Account) {
-        fixAccountUint8Arrays(account.data.account)
-        // console.dir(acc, { depth: null })
-      } else if (
-        account.data.accountType === AccountType.ContractCode ||
-        account.data.accountType === AccountType.ContractStorage
-      ) {
-        fixAccountUint8Arrays(account.data)
-        // console.dir(acc, { depth: null })
-      }
       const calculatedAccountHash = accountSpecificHash(account.data)
       const indexOfAccount = receipt.appliedReceipt.appliedVote.account_id.indexOf(account.accountId)
       if (indexOfAccount === -1) {
diff --git a/scripts/verify_account_hash.ts b/scripts/verify_account_hash.ts
index edd760d3..e0be9b99 100644
--- a/scripts/verify_account_hash.ts
+++ b/scripts/verify_account_hash.ts
@@ -7,7 +7,7 @@ import * as dbstore from '../src/dbstore'
 import * as AccountDB from '../src/dbstore/accounts'
 import { startSaving } from '../src/saveConsoleOutput'
 import * as Logger from '../src/Logger'
-import { AccountType, fixAccountUint8Arrays, accountSpecificHash } from '../src/shardeum/calculateAccountHash'
+import { AccountType, accountSpecificHash } from '../src/shardeum/calculateAccountHash'
 import { addSigListeners } from '../src/State'
 import { Utils as StringUtils } from '@shardus/types'
 
@@ -48,16 +48,6 @@ const runProgram = async (): Promise<void> => {
       if (accountHash1 !== accountHash2) {
         console.log(account.accountId, 'accountHash', accountHash1, 'accountHash2', accountHash2)
       }
-      if (account.data.accountType === AccountType.Account) {
-        fixAccountUint8Arrays(account.data.account)
-        // console.dir(acc, { depth: null })
-      } else if (
-        account.data.accountType === AccountType.ContractCode ||
-        account.data.accountType === AccountType.ContractStorage
-      ) {
-        fixAccountUint8Arrays(account.data)
-        // console.dir(acc, { depth: null })
-      }
       const calculatedAccountHash = accountSpecificHash(account.data)
 
       if (accountHash1 !== calculatedAccountHash) {
diff --git a/src/Data/Data.ts b/src/Data/Data.ts
index a3de8cef..ab89bdcf 100644
--- a/src/Data/Data.ts
+++ b/src/Data/Data.ts
@@ -287,13 +287,17 @@ export function initSocketClient(node: NodeList.ConsensusNodeInfo): void {
       if (newData.responses && newData.responses.ACCOUNT) {
         if (getCurrentCycleCounter() > GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle) {
           Logger.mainLogger.error(
-            'Account data is not meant to be received after the genesis cycle',
+            'Account data is not meant to be received after the genesis accounts cycle range',
             getCurrentCycleCounter()
           )
           unsubscribeDataSender(sender.nodeInfo.publicKey)
           return
         }
-        if (NodeList.byPublicKey.size > 1 || !NodeList.byPublicKey.has(sender.nodeInfo.publicKey)) {
+        if (
+          Cycles.currentNetworkMode !== 'forming' ||
+          NodeList.byPublicKey.size > 1 ||
+          !NodeList.byPublicKey.has(sender.nodeInfo.publicKey)
+        ) {
           Logger.mainLogger.error(
             'Account data is not meant to be received by the first validator',
             `Number of nodes in the network ${NodeList.byPublicKey.size}`

From 8ec0f5bff98aeb50d540f936eb92dfb38663d45a Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Tue, 2 Jul 2024 23:00:13 +0545
Subject: [PATCH 17/20] Created config for the number of cycles to keep shard
 calculations data

---
 src/Config.ts      | 3 +++
 src/Data/Cycles.ts | 4 +---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Config.ts b/src/Config.ts
index 5641109d..2ec120cf 100644
--- a/src/Config.ts
+++ b/src/Config.ts
@@ -68,6 +68,8 @@ export interface Config {
   usePOQo: boolean
   // The percentage of votes required to confirm transaction
   requiredVotesPercentage: number
+  // number of recent cycles of shard data to keep
+  CYCLE_SHARD_STORAGE_LIMIT: number
 }
 
 let config: Config = {
@@ -131,6 +133,7 @@ let config: Config = {
   stopGossipTxData: false,
   usePOQo: true,
   requiredVotesPercentage: 2 / 3,
+  CYCLE_SHARD_STORAGE_LIMIT: 3,
 }
 // Override default config params from config file, env vars, and cli args
 export async function overrideDefaultConfig(file: string): Promise<void> {
diff --git a/src/Data/Cycles.ts b/src/Data/Cycles.ts
index 25748446..139fe7f6 100644
--- a/src/Data/Cycles.ts
+++ b/src/Data/Cycles.ts
@@ -52,8 +52,6 @@ export let cycleRecordWithShutDownMode = null as P2PTypes.CycleCreatorTypes.Cycl
 export let currentNetworkMode: P2PTypes.ModesTypes.Record['mode'] = 'forming'
 export const shardValuesByCycle = new Map<number, StateManager.shardFunctionTypes.CycleShardData>()
 
-const CYCLE_SHARD_STORAGE_LIMIT = 20
-
 export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData[]): Promise<void> {
   if (profilerInstance) profilerInstance.profileSectionStart('process_cycle', false)
   try {
@@ -518,7 +516,7 @@ function updateShardValues(cycle: P2PTypes.CycleCreatorTypes.CycleData): void {
   const list = cycleShardData.nodes.map((n) => n['ip'] + ':' + n['port'])
   Logger.mainLogger.debug('cycleShardData', cycleShardData.cycleNumber, list.length, stringifyReduce(list))
   shardValuesByCycle.set(cycleShardData.cycleNumber, cycleShardData)
-  if (shardValuesByCycle.size > CYCLE_SHARD_STORAGE_LIMIT) {
+  if (shardValuesByCycle.size > config.CYCLE_SHARD_STORAGE_LIMIT) {
     shardValuesByCycle.delete(shardValuesByCycle.keys().next().value)
   }
 }

From 333754952e8f74594aefe6b9aa731606d6e3ffa3 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Wed, 3 Jul 2024 13:21:09 +0545
Subject: [PATCH 18/20] Added debug logs for receipt processing time taken

---
 src/Data/Collector.ts  | 17 ++++++++++++++---
 src/Data/GossipData.ts |  4 ++--
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts
index a26d233d..cfb2a4ac 100644
--- a/src/Data/Collector.ts
+++ b/src/Data/Collector.ts
@@ -390,7 +390,18 @@ export const verifyReceiptData = async (
   const { executionShardKey, cycle, appliedReceipt, globalModification } = receipt
   if (globalModification && config.skipGlobalTxReceiptVerification) return { success: true }
   const { appliedVote, signatures } = appliedReceipt
-  const txId = receipt.tx.txId
+  const { txId, timestamp } = receipt.tx
+  const currentTimestamp = Date.now()
+  // Console log the timetaken between the receipt timestamp and the current time ( both in ms and s)
+  console.log(
+    `Time taken between receipt timestamp and current time: ${txId}`,
+    `${currentTimestamp - timestamp} ms`,
+    `${currentTimestamp - timestamp / 1000} s`
+  )
+  if (getCurrentCycleCounter() - cycle > 2) {
+    Logger.mainLogger.error(`Found receipt with cycle older than 2 cycles ${txId}, ${cycle}, ${timestamp}`)
+    console.dir(receipt, { depth: null })
+  }
   const cycleShardData = shardValuesByCycle.get(cycle)
   if (!cycleShardData) {
     Logger.mainLogger.error('Cycle shard data not found')
@@ -435,7 +446,7 @@ export const verifyReceiptData = async (
       const node = cycleShardData.nodes.find((node) => node.publicKey === nodePubKey)
       if (node == null) {
         Logger.mainLogger.error(
-          `The node with public key ${nodePubKey} of the receipt ${txId}} is not in the active nodesList of cycle ${cycle}`
+          `The node with public key ${nodePubKey} of the receipt ${txId}} with ${timestamp} is not in the active nodesList of cycle ${cycle}`
         )
         if (nestedCountersInstance)
           nestedCountersInstance.countEvent(
@@ -447,7 +458,7 @@ export const verifyReceiptData = async (
       // Check if the node is in the execution group
       if (!cycleShardData.parititionShardDataMap.get(homePartition).coveredBy[node.id]) {
         Logger.mainLogger.error(
-          `The node with public key ${nodePubKey} of the receipt ${txId} is not in the execution group of the tx`
+          `The node with public key ${nodePubKey} of the receipt ${txId} with ${timestamp} is not in the execution group of the tx`
         )
         if (nestedCountersInstance)
           nestedCountersInstance.countEvent(
diff --git a/src/Data/GossipData.ts b/src/Data/GossipData.ts
index 0f761e58..d2414e6d 100644
--- a/src/Data/GossipData.ts
+++ b/src/Data/GossipData.ts
@@ -98,13 +98,13 @@ export async function sendDataToAdjacentArchivers(
         })
         promises.push(promise)
       } catch (e) {
-        Logger.mainLogger.error('Error', e)
+        Logger.mainLogger.error(`Gossip Error to archiver ${archiver.ip}: ${archiver.port}`, e)
       }
     }
     try {
       await Promise.allSettled(promises)
     } catch (err) {
-      Logger.mainLogger.error('Network: ' + err)
+      Logger.mainLogger.error('Gossip Error: ' + err)
     }
   } catch (ex) {
     Logger.mainLogger.debug(ex)

From ab04c94be03ae754e8519254ae3294b5ba44ce02 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Wed, 3 Jul 2024 14:34:30 +0545
Subject: [PATCH 19/20] Updated to keep 10 CYCLE_SHARD_STORAGE_LIMIT

---
 src/Config.ts         | 2 +-
 src/Data/Collector.ts | 9 ++++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/Config.ts b/src/Config.ts
index 2ec120cf..b4abb154 100644
--- a/src/Config.ts
+++ b/src/Config.ts
@@ -133,7 +133,7 @@ let config: Config = {
   stopGossipTxData: false,
   usePOQo: true,
   requiredVotesPercentage: 2 / 3,
-  CYCLE_SHARD_STORAGE_LIMIT: 3,
+  CYCLE_SHARD_STORAGE_LIMIT: 10,
 }
 // Override default config params from config file, env vars, and cli args
 export async function overrideDefaultConfig(file: string): Promise<void> {
diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts
index cfb2a4ac..a668a916 100644
--- a/src/Data/Collector.ts
+++ b/src/Data/Collector.ts
@@ -396,10 +396,13 @@ export const verifyReceiptData = async (
   console.log(
     `Time taken between receipt timestamp and current time: ${txId}`,
     `${currentTimestamp - timestamp} ms`,
-    `${currentTimestamp - timestamp / 1000} s`
+    `${(currentTimestamp - timestamp) / 1000} s`
   )
-  if (getCurrentCycleCounter() - cycle > 2) {
-    Logger.mainLogger.error(`Found receipt with cycle older than 2 cycles ${txId}, ${cycle}, ${timestamp}`)
+  const currentCycle = getCurrentCycleCounter()
+  if (currentCycle - cycle > 2) {
+    Logger.mainLogger.error(
+      `Found receipt with cycle older than 2 cycles ${txId}, ${cycle}, ${timestamp}, ${currentCycle}`
+    )
     console.dir(receipt, { depth: null })
   }
   const cycleShardData = shardValuesByCycle.get(cycle)

From dddd88a67774c5aa72c378f11c033f8cf1277e97 Mon Sep 17 00:00:00 2001
From: jairajdev <jairajranamagar.dev@gmail.com>
Date: Mon, 22 Jul 2024 13:06:44 +0545
Subject: [PATCH 20/20] Put the console logs behind the verbose flag

---
 src/Data/Collector.ts | 17 +++++++++--------
 src/Data/Cycles.ts    |  4 ++--
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts
index a668a916..3335af16 100644
--- a/src/Data/Collector.ts
+++ b/src/Data/Collector.ts
@@ -391,19 +391,20 @@ export const verifyReceiptData = async (
   if (globalModification && config.skipGlobalTxReceiptVerification) return { success: true }
   const { appliedVote, signatures } = appliedReceipt
   const { txId, timestamp } = receipt.tx
-  const currentTimestamp = Date.now()
-  // Console log the timetaken between the receipt timestamp and the current time ( both in ms and s)
-  console.log(
-    `Time taken between receipt timestamp and current time: ${txId}`,
-    `${currentTimestamp - timestamp} ms`,
-    `${(currentTimestamp - timestamp) / 1000} s`
-  )
+  if (config.VERBOSE) {
+    const currentTimestamp = Date.now()
+    // Console log the timetaken between the receipt timestamp and the current time ( both in ms and s)
+    console.log(
+      `Time taken between receipt timestamp and current time: ${txId}`,
+      `${currentTimestamp - timestamp} ms`,
+      `${(currentTimestamp - timestamp) / 1000} s`
+    )
+  }
   const currentCycle = getCurrentCycleCounter()
   if (currentCycle - cycle > 2) {
     Logger.mainLogger.error(
       `Found receipt with cycle older than 2 cycles ${txId}, ${cycle}, ${timestamp}, ${currentCycle}`
     )
-    console.dir(receipt, { depth: null })
   }
   const cycleShardData = shardValuesByCycle.get(cycle)
   if (!cycleShardData) {
diff --git a/src/Data/Cycles.ts b/src/Data/Cycles.ts
index 139fe7f6..88f9a0a0 100644
--- a/src/Data/Cycles.ts
+++ b/src/Data/Cycles.ts
@@ -91,8 +91,8 @@ export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData
         setShutdownCycleRecord(cycle)
         NodeList.toggleFirstNode()
       }
-      // Clean receipts/originalTxs cache that are older than 20 minutes to match with CYCLE_SHARD_STORAGE_LIMIT ( actual fix is on another branch )
-      const cleanupTimestamp = Date.now() - 20 * 60 * 1000
+      // Clean receipts/originalTxs cache that are older than <CYCLE_SHARD_STORAGE_LIMIT> minutes
+      const cleanupTimestamp = (cycle.start - config.CYCLE_SHARD_STORAGE_LIMIT * 60) * 1000
       cleanOldOriginalTxsMap(cleanupTimestamp)
       cleanOldReceiptsMap(cleanupTimestamp)
     }