Merge branch 'dev' into GOLD-131

.github/workflows/ci.yml

@@ -0,0 +1,77 @@
+name: Node CI Workflow
+#
+#  The parameters are defaulted at the org level but can be overridden on the repository.
+#  See the github-automation repo for more documentation
+#
+on:
+  push:
+    branches:
+      - dev
+      - main
+  pull_request:
+    branches:
+      - dev
+      - main
+  issue_comment:
+    inputs:
+      workflowBranch:
+        description: 'Branch of the reusable workflow. Defaults to main, select dev for testing only.'
+        required: true
+        default: 'main'
+        type: choice
+        options:
+          - dev
+          - main
+  workflow_dispatch:
+    inputs:
+      workflowBranch:
+        description: 'Branch of the reusable workflow. Defaults to main, select dev for testing only.'
+        required: true
+        default: 'main'
+        type: choice
+        options:
+          - dev
+          - main
+jobs:
+  echo-inputs:
+    name: Repo Workflow Debugging
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check Repo Vars
+        run: |
+          echo "*** Start - Check inputs in repo workflow ***"
+          echo "Node Version: ${{ vars.NODE_VERSION }}"
+          echo "Lint Required: ${{ vars.IS_LINT_REQUIRED }}"
+          echo "Format Check Required: ${{ vars.IS_FORMAT_CHECK_REQUIRED }}"
+          echo "Apply Patches Required: ${{ vars.IS_APPLY_PATCHES_REQUIRED }}"
+          echo "Unit Tests Required: ${{ vars.IS_UNIT_TESTS_REQUIRED }}"
+          echo "*** End - Check inputs in repo workflow ***"
+  ci-test-only:
+    if: ${{ github.event.inputs.workflowBranch == 'dev' }}
+    uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@dev
+    permissions:
+      issues: write
+      pull-requests: write
+      contents: write
+    with:
+      node-version: ${{ vars.NODE_VERSION }}
+      lint-required: ${{ vars.IS_LINT_REQUIRED == 'true' }}
+      format-check-required: ${{ vars.IS_FORMAT_CHECK_REQUIRED == 'true' }}
+      apply-patches-required: ${{ vars.IS_APPLY_PATCHES_REQUIRED == 'true' }}
+      unit-tests-required: ${{ vars.IS_UNIT_TESTS_REQUIRED == 'true' }}
+    secrets: inherit
+
+  ci:
+    if: ${{ github.event.inputs.workflowBranch == 'main' || !github.event.inputs.workflowBranch }}
+    uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@main
+    permissions:
+      issues: write
+      pull-requests: write
+      contents: write
+    with:
+      node-version: ${{ vars.NODE_VERSION }}
+      lint-required: ${{ vars.IS_LINT_REQUIRED == 'true' }}
+      format-check-required: ${{ vars.IS_FORMAT_CHECK_REQUIRED == 'true' }}
+      apply-patches-required: ${{ vars.IS_APPLY_PATCHES_REQUIRED == 'true' }}
+      unit-tests-required: ${{ vars.IS_UNIT_TESTS_REQUIRED == 'true' }}
+    secrets: inherit

CODEOWNERS

@@ -0,0 +1,13 @@
+# CODEOWNERS file
+# To add additional teams to any approval, include them on the same line separated by spaces
+# It is best practice to assign a team as a code owner and not an invidual.
+# Please submit requests for new teams to Systems and Automation
+
+# Global approval (all files)
+# * @shardeum/team-name
+
+# Directory-level approval
+/.github/ @shardeum/systems-and-automation
+
+# Specific file rules
+# README.md @shardeum/team-name

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Shardeum
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -6,6 +6,11 @@ This is a node that runs as part of the shardus network, with the function of re
 
 To release, just run `npm run release`
 
+## Health Check
+
+GET `/is-alive` this endpoint returns 200 if the server is running.
+GET `/is-healthy` currently the same as `/is-alive` but will be expanded.
+
 ## Contributing
 
 Contributions are very welcome! Everyone interacting in our codebases, issue trackers, and any other form of communication, including chat rooms and mailing lists, is expected to follow our [code of conduct](./CODE_OF_CONDUCT.md) so we can all enjoy the effort we put into this project.

archiver-config.json

@@ -56,7 +56,7 @@
       "publicKey": "aec5d2b663869d9c22ba99d8de76f3bff0f54fa5e39d2899ec1f3f4543422ec7"
     }
   ],
-  "ARCHIVER_MODE": "debug",
+  "ARCHIVER_MODE": "release",
   "DevPublicKey": "",
   "EXISTING_ARCHIVER_DB_PATH": ""
 }

debug_mode.patch

@@ -0,0 +1,27 @@
+diff --git a/archiver-config.json b/archiver-config.json
+index a892baf..ffac25a 100644
+--- a/archiver-config.json
++++ b/archiver-config.json
+@@ -56,7 +56,7 @@
+       "publicKey": "aec5d2b663869d9c22ba99d8de76f3bff0f54fa5e39d2899ec1f3f4543422ec7"
+     }
+   ],
+-  "ARCHIVER_MODE": "release",
++  "ARCHIVER_MODE": "debug",
+   "DevPublicKey": "",
+   "EXISTING_ARCHIVER_DB_PATH": ""
+ }
+\ No newline at end of file
+diff --git a/src/Config.ts b/src/Config.ts
+index 6b41ee4..a812003 100644
+--- a/src/Config.ts
++++ b/src/Config.ts
+@@ -86,7 +86,7 @@ let config: Config = {
+     save: true,
+     interval: 1,
+   },
+-  ARCHIVER_MODE: 'release', // 'debug'/'release'
++  ARCHIVER_MODE: 'debug', // 'debug'/'release'
+   DevPublicKey: '',
+   dataLogWrite: true,
+   dataLogWriter: {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shardus/archiver",
-  "version": "3.4.21",
+  "version": "3.4.23",
   "engines": {
     "node": "18.16.1"
   },
@@ -17,12 +17,11 @@
   "scripts": {
     "start": "npm run prepare && node build/server.js",
     "release": "npm run prepare && np --no-cleanup --no-tests --no-yarn --any-branch",
-    "test": "echo \"Error: no test specified\" && exit 1",
     "check": "gts check",
     "clean": "npm-run-all clean:*",
     "clean:typescript": "gts clean",
-    "lint": "eslint './src/**/*.ts'",
-    "lint-windows": "eslint ./src/**/*.ts",
+    "lint": "eslint \"./src/**/*.ts\"",
+    "test": "echo \"Error: no test specified\" && exit 1",
     "format-check": "prettier --check './src/**/*.ts'",
     "format-fix": "prettier --write './src/**/*.ts'",
     "clean:artifacts": "shx rm -rf archiver-logs/ archiver-db/ data-logs/",

scripts/archiver_data_sync_check.ts

@@ -4,6 +4,7 @@ import { join } from 'path'
 import { postJson } from '../src/P2P'
 import { config, overrideDefaultConfig } from '../src/Config'
 import { ArchiverNodeInfo } from '../src/State'
+import { Utils as StringUtils } from '@shardus/types'
 
 const configFile = join(process.cwd(), 'archiver-config.json')
 overrideDefaultConfig(configFile)

scripts/create_shut_down_cycle.ts

@@ -10,6 +10,7 @@ import * as Logger from '../src/Logger'
 import { P2P } from '@shardus/types'
 import { addSigListeners } from '../src/State'
 import { computeCycleMarker } from '../src/Data/Cycles'
+import { Utils as StringUtils } from '@shardus/types'
 
 const archiversAtShutdown = [
   {

scripts/repair_missing_cycle.ts

@@ -1,13 +1,13 @@
 import { readFileSync } from 'fs'
-import { resolve } from 'path'
-import { join } from 'path'
+import { resolve, join } from 'path'
 import { overrideDefaultConfig, config } from '../src/Config'
 import * as Crypto from '../src/Crypto'
 import * as db from '../src/dbstore/sqlite3storage'
 import * as dbstore from '../src/dbstore'
 import * as CycleDB from '../src/dbstore/cycles'
 import { startSaving } from '../src/saveConsoleOutput'
 import * as Logger from '../src/Logger'
+import { Utils as StringUtils } from '@shardus/types'
 
 const patchCycleData = false
 

scripts/update_config.ts

@@ -0,0 +1,50 @@
+import axios from 'axios'
+import { join } from 'path'
+import { Utils } from '@shardus/types'
+import * as crypto from '@shardus/crypto-utils'
+import { config, overrideDefaultConfig } from '../src/Config'
+
+const configFile = join(process.cwd(), 'archiver-config.json')
+overrideDefaultConfig(configFile)
+
+crypto.init(config.ARCHIVER_HASH_KEY)
+
+const DEV_KEYS = {
+  pk: config.ARCHIVER_PUBLIC_KEY,
+  sk: config.ARCHIVER_SECRET_KEY,
+}
+
+function sign<T>(obj: T, sk: string, pk: string): T & any {
+  const objCopy = JSON.parse(crypto.stringify(obj))
+  crypto.signObj(objCopy, sk, pk)
+  return objCopy
+}
+
+function createSignature(data: any, pk: string, sk: string): any {
+  return sign({ ...data }, sk, pk)
+}
+
+const UPDATE_CONFIG = {
+  /* Add Config properties that need to be updated here */
+  VERBOSE: true,
+  RATE_LIMIT: 200,
+}
+
+const INPUT = Utils.safeStringify(createSignature(UPDATE_CONFIG, DEV_KEYS.pk, DEV_KEYS.sk))
+
+axios
+  .patch('http://127.0.0.1:4000/set-config', INPUT, {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  })
+  .then((response) => {
+    console.log(response.data)
+  })
+  .catch((error) => {
+    if (error.response) {
+      console.error(error.response)
+    } else {
+      console.error(error.message)
+    }
+  })

scripts/update_network_account.ts

@@ -9,6 +9,7 @@ import { startSaving } from '../src/saveConsoleOutput'
 import * as Logger from '../src/Logger'
 import { accountSpecificHash } from '../src/shardeum/calculateAccountHash'
 import { addSigListeners } from '../src/State'
+import { Utils as StringUtils } from '@shardus/types'
 
 const activeVersion = '1.9.0'
 const latestVersion = '1.9.0'

scripts/validate_archiver_receipt.ts

@@ -3,7 +3,7 @@ import { overrideDefaultConfig, config } from '../src/Config'
 import * as Crypto from '../src/Crypto'
 import * as Utils from '../src/Utils'
 import * as Receipt from '../src/dbstore/receipts'
-import { AccountType, accountSpecificHash, fixAccountUint8Arrays } from '../src/shardeum/calculateAccountHash'
+import { AccountType, accountSpecificHash } from '../src/shardeum/calculateAccountHash'
 import { ShardeumReceipt } from '../src/shardeum/verifyAppReceiptData'
 
 // Add the full receipt data here
@@ -243,16 +243,6 @@ export const verifyAccountHash = (receipt: Receipt.ArchiverReceipt): boolean =>
   try {
     if (receipt.globalModification && config.skipGlobalTxReceiptVerification) return true // return true if global modification
     for (const account of receipt.accounts) {
-      if (account.data.accountType === AccountType.Account) {
-        fixAccountUint8Arrays(account.data.account)
-        // console.dir(acc, { depth: null })
-      } else if (
-        account.data.accountType === AccountType.ContractCode ||
-        account.data.accountType === AccountType.ContractStorage
-      ) {
-        fixAccountUint8Arrays(account.data)
-        // console.dir(acc, { depth: null })
-      }
       const calculatedAccountHash = accountSpecificHash(account.data)
       const indexOfAccount = receipt.appliedReceipt.appliedVote.account_id.indexOf(account.accountId)
       if (indexOfAccount === -1) {