BLUE-150 Created config for the number of cycles to keep shard calculations data

.github/workflows/ci.yml

@@ -12,6 +12,16 @@ on:
     branches:
       - dev
       - main
+  issue_comment:
+    inputs:
+      workflowBranch:
+        description: 'Branch of the reusable workflow. Defaults to main, select dev for testing only.'
+        required: true
+        default: 'main'
+        type: choice
+        options:
+          - dev
+          - main
   workflow_dispatch:
     inputs:
       workflowBranch:
@@ -22,11 +32,6 @@ on:
         options:
           - dev
           - main
-
-permissions:
-  issues: write
-  pull-requests: write
-
 jobs:
   echo-inputs:
     name: Repo Workflow Debugging
@@ -41,9 +46,13 @@ jobs:
           echo "Apply Patches Required: ${{ vars.IS_APPLY_PATCHES_REQUIRED }}"
           echo "Unit Tests Required: ${{ vars.IS_UNIT_TESTS_REQUIRED }}"
           echo "*** End - Check inputs in repo workflow ***"
-  ci-dev:
+  ci-test-only:
     if: ${{ github.event.inputs.workflowBranch == 'dev' }}
     uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@dev
+    permissions:
+      issues: write
+      pull-requests: write
+      contents: write
     with:
       node-version: ${{ vars.NODE_VERSION }}
       lint-required: ${{ vars.IS_LINT_REQUIRED == 'true' }}
@@ -52,9 +61,13 @@ jobs:
       unit-tests-required: ${{ vars.IS_UNIT_TESTS_REQUIRED == 'true' }}
     secrets: inherit
 
-  ci-main:
+  ci:
     if: ${{ github.event.inputs.workflowBranch == 'main' || !github.event.inputs.workflowBranch }}
     uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@main
+    permissions:
+      issues: write
+      pull-requests: write
+      contents: write
     with:
       node-version: ${{ vars.NODE_VERSION }}
       lint-required: ${{ vars.IS_LINT_REQUIRED == 'true' }}

CODEOWNERS

@@ -0,0 +1,13 @@
+# CODEOWNERS file
+# To add additional teams to any approval, include them on the same line separated by spaces
+# It is best practice to assign a team as a code owner and not an invidual.
+# Please submit requests for new teams to Systems and Automation
+
+# Global approval (all files)
+# * @shardeum/team-name
+
+# Directory-level approval
+/.github/ @shardeum/systems-and-automation
+
+# Specific file rules
+# README.md @shardeum/team-name

package.json

@@ -17,12 +17,11 @@
   "scripts": {
     "start": "npm run prepare && node build/server.js",
     "release": "npm run prepare && np --no-cleanup --no-tests --no-yarn --any-branch",
-    "test": "echo \"Error: no test specified\" && exit 1",
     "check": "gts check",
     "clean": "npm-run-all clean:*",
     "clean:typescript": "gts clean",
-    "lint": "eslint './src/**/*.ts'",
-    "lint-windows": "eslint ./src/**/*.ts",
+    "lint": "eslint \"./src/**/*.ts\"",
+    "test": "echo \"Error: no test specified\" && exit 1",
     "format-check": "prettier --check './src/**/*.ts'",
     "format-fix": "prettier --write './src/**/*.ts'",
     "clean:artifacts": "shx rm -rf archiver-logs/ archiver-db/ data-logs/",

scripts/archiver_data_sync_check.ts

@@ -4,6 +4,7 @@ import { join } from 'path'
 import { postJson } from '../src/P2P'
 import { config, overrideDefaultConfig } from '../src/Config'
 import { ArchiverNodeInfo } from '../src/State'
+import { Utils as StringUtils } from '@shardus/types'
 
 const configFile = join(process.cwd(), 'archiver-config.json')
 overrideDefaultConfig(configFile)

scripts/create_shut_down_cycle.ts

@@ -10,6 +10,7 @@ import * as Logger from '../src/Logger'
 import { P2P } from '@shardus/types'
 import { addSigListeners } from '../src/State'
 import { computeCycleMarker } from '../src/Data/Cycles'
+import { Utils as StringUtils } from '@shardus/types'
 
 const archiversAtShutdown = [
   {

scripts/repair_missing_cycle.ts

@@ -1,13 +1,13 @@
 import { readFileSync } from 'fs'
-import { resolve } from 'path'
-import { join } from 'path'
+import { resolve, join } from 'path'
 import { overrideDefaultConfig, config } from '../src/Config'
 import * as Crypto from '../src/Crypto'
 import * as db from '../src/dbstore/sqlite3storage'
 import * as dbstore from '../src/dbstore'
 import * as CycleDB from '../src/dbstore/cycles'
 import { startSaving } from '../src/saveConsoleOutput'
 import * as Logger from '../src/Logger'
+import { Utils as StringUtils } from '@shardus/types'
 
 const patchCycleData = false
 

scripts/update_network_account.ts

@@ -9,6 +9,7 @@ import { startSaving } from '../src/saveConsoleOutput'
 import * as Logger from '../src/Logger'
 import { accountSpecificHash } from '../src/shardeum/calculateAccountHash'
 import { addSigListeners } from '../src/State'
+import { Utils as StringUtils } from '@shardus/types'
 
 const activeVersion = '1.9.0'
 const latestVersion = '1.9.0'

scripts/validate_archiver_receipt.ts

@@ -3,7 +3,7 @@ import { overrideDefaultConfig, config } from '../src/Config'
 import * as Crypto from '../src/Crypto'
 import * as Utils from '../src/Utils'
 import * as Receipt from '../src/dbstore/receipts'
-import { AccountType, accountSpecificHash, fixAccountUint8Arrays } from '../src/shardeum/calculateAccountHash'
+import { AccountType, accountSpecificHash } from '../src/shardeum/calculateAccountHash'
 import { ShardeumReceipt } from '../src/shardeum/verifyAppReceiptData'
 
 // Add the full receipt data here
@@ -243,16 +243,6 @@ export const verifyAccountHash = (receipt: Receipt.ArchiverReceipt): boolean =>
   try {
     if (receipt.globalModification && config.skipGlobalTxReceiptVerification) return true // return true if global modification
     for (const account of receipt.accounts) {
-      if (account.data.accountType === AccountType.Account) {
-        fixAccountUint8Arrays(account.data.account)
-        // console.dir(acc, { depth: null })
-      } else if (
-        account.data.accountType === AccountType.ContractCode ||
-        account.data.accountType === AccountType.ContractStorage
-      ) {
-        fixAccountUint8Arrays(account.data)
-        // console.dir(acc, { depth: null })
-      }
       const calculatedAccountHash = accountSpecificHash(account.data)
       const indexOfAccount = receipt.appliedReceipt.appliedVote.account_id.indexOf(account.accountId)
       if (indexOfAccount === -1) {

scripts/verify_account_hash.ts

@@ -7,8 +7,9 @@ import * as dbstore from '../src/dbstore'
 import * as AccountDB from '../src/dbstore/accounts'
 import { startSaving } from '../src/saveConsoleOutput'
 import * as Logger from '../src/Logger'
-import { AccountType, fixAccountUint8Arrays, accountSpecificHash } from '../src/shardeum/calculateAccountHash'
+import { AccountType, accountSpecificHash } from '../src/shardeum/calculateAccountHash'
 import { addSigListeners } from '../src/State'
+import { Utils as StringUtils } from '@shardus/types'
 
 const updateHash = false
 const runProgram = async (): Promise<void> => {
@@ -47,16 +48,6 @@ const runProgram = async (): Promise<void> => {
       if (accountHash1 !== accountHash2) {
         console.log(account.accountId, 'accountHash', accountHash1, 'accountHash2', accountHash2)
       }
-      if (account.data.accountType === AccountType.Account) {
-        fixAccountUint8Arrays(account.data.account)
-        // console.dir(acc, { depth: null })
-      } else if (
-        account.data.accountType === AccountType.ContractCode ||
-        account.data.accountType === AccountType.ContractStorage
-      ) {
-        fixAccountUint8Arrays(account.data)
-        // console.dir(acc, { depth: null })
-      }
       const calculatedAccountHash = accountSpecificHash(account.data)
 
       if (accountHash1 !== calculatedAccountHash) {

src/Config.ts

@@ -65,6 +65,11 @@ export interface Config {
   saveOnlyGossipData: boolean
   // For debugging purpose, set this to true to stop gossiping tx data
   stopGossipTxData: boolean
+  usePOQo: boolean
+  // The percentage of votes required to confirm transaction
+  requiredVotesPercentage: number
+  // number of recent cycles of shard data to keep
+  CYCLE_SHARD_STORAGE_LIMIT: number
 }
 
 let config: Config = {
@@ -126,6 +131,9 @@ let config: Config = {
   extraConsensorsToSubscribe: 1,
   saveOnlyGossipData: false,
   stopGossipTxData: false,
+  usePOQo: true,
+  requiredVotesPercentage: 2 / 3,
+  CYCLE_SHARD_STORAGE_LIMIT: 10,
 }
 // Override default config params from config file, env vars, and cli args
 export async function overrideDefaultConfig(file: string): Promise<void> {