SHARD-1166: Archiver whitelisting (#123)

* Added allowed archiver signed list and endpoint to fetch it

* Move the allowed archiver list verification to archiver

* Add script to sign the archiver config file

* Add allowed-archivers endpoint unit test

* Add more unit tests

* Moved the verification and config loading to AllowedArchiversManager class

* Use archiverWhitelistMinSigRequired config

* Get and apply latest changes from global account

* Code refactor and bug fixes

* Fix: reload config on global account updates

* Remove counter

* Remove allowed signers and minSigRequired from the config

* Update unit test

allowed-archivers.json

@@ -0,0 +1,20 @@
+{
+    "allowedArchivers": [
+        {
+            "ip": "127.0.0.1",
+            "port": 4000,
+            "publicKey": "758b1c119412298802cd28dbfa394cdfeecc4074492d60844cc192d632d84de3"
+        },
+        {
+            "ip": "127.0.0.1",
+            "port": 4001,
+            "publicKey": "e8a5c26b9e2c3c31eb7c7d73eaed9484374c16d983ce95f3ab18a62521964a94"
+        }
+    ],
+    "signatures": [
+        {
+            "owner": "0x002D3a2BfE09E3E29b6d38d58CaaD16EEe4C9BC5",
+            "sig": "0x53535921e57d0796b0dbf1451ff6a9ff535bbf2f51cdcc1a5ddd618aa62ca0f36a98c5e2fcecaa7217626735fc3bb5f47acc5e7e60284bb20fb157b50390d45f1b"
+        }
+    ]
+}

scripts/archiver_config_sign.ts

@@ -0,0 +1,52 @@
+import { ethers } from 'ethers';
+import * as fs from 'fs';
+import { Utils as StringUtils } from '@shardeum-foundation/lib-types';
+
+interface ConfigData {
+    allowedArchivers: string[];
+}
+
+interface SignaturePayload {
+    allowedArchivers: string[];
+}
+
+async function generateSignature(): Promise<void> {
+    try {
+
+        // Get private key from env or command line
+        const privateKey = process.env.PRIVATE_KEY || process.argv[2];
+        if (!privateKey) {
+            throw new Error('Private key not provided. Set PRIVATE_KEY in .env or provide as command line argument');
+        }
+
+        // Read and parse config file
+        const configData: ConfigData = StringUtils.safeJsonParse(
+            fs.readFileSync('./allowed-archivers.json', 'utf8')
+        );
+
+        // Create payload
+        const rawPayload: SignaturePayload = {
+            allowedArchivers: configData.allowedArchivers
+        };
+
+        // Generate hash of payload
+        const payloadHash = ethers.keccak256(
+            ethers.toUtf8Bytes(StringUtils.safeStringify(rawPayload))
+        );
+
+        console.log('Payload hash:', payloadHash);
+
+        // Initialize wallet and sign
+        const wallet = new ethers.Wallet(privateKey);
+        const signature = await wallet.signMessage(payloadHash);
+        console.log('Signature:', signature);
+    } catch (error) {
+        console.error('Error generating signature:', error);
+        process.exit(1);
+    }
+}
+
+// Execute if running directly
+if (require.main === module) {
+    generateSignature();
+}

src/API.ts

@@ -35,11 +35,10 @@ import {
   failureReceiptCount,
 } from './primary-process'
 import * as ServiceQueue from './ServiceQueue'
-import { readFileSync } from 'fs'
-import { join } from 'path'
 import ticketRoutes from './routes/tickets'
-const { version } = require('../package.json') // eslint-disable-line @typescript-eslint/no-var-requires
+import { allowedArchiversManager } from './shardeum/allowedArchiversManager'
 
+const { version } = require('../package.json') // eslint-disable-line @typescript-eslint/no-var-requires
 const TXID_LENGTH = 64
 const {
   MAX_CYCLES_PER_REQUEST,
@@ -281,6 +280,23 @@ export function registerRoutes(server: FastifyInstance<Server, IncomingMessage,
     reply.send(res)
   })
 
+  server.get('/allowed-archivers', async (_request, reply) => {
+    try {
+      const config = allowedArchiversManager.getCurrentConfig()
+      if (!config) {
+        return reply.status(500).send({
+          error: 'Internal server error'
+        })
+      }
+      return reply.send(config)
+    } catch (error) {
+      Logger.mainLogger.error('Error serving allowed-archivers:', error)
+      return reply.status(500).send({
+        error: 'Internal server error'
+      })
+    }
+  })
+
   server.get('/nodeInfo', (request, reply) => {
     if (reachabilityAllowed) {
       reply.send({
@@ -1296,19 +1312,32 @@ export const validateRequestData = (
       Logger.mainLogger.error('Data sender publicKey and sign owner key does not match')
       return { success: false, error: 'Data sender publicKey and sign owner key does not match' }
     }
+    if (!Crypto.verify(data)) {
+      Logger.mainLogger.error('Invalid signature', data)
+      return { success: false, error: 'Invalid signature' }
+    }
     if (!skipArchiverCheck && config.limitToArchiversOnly) {
-      // Check if the sender is in the archiver list or is the devPublicKey
+      // Check if the sender is in the allowed archivers list
+      const isAllowedArchiver = allowedArchiversManager.isArchiverAllowed(data.sender)
+
+      // Check if the sender is in the active archiver list or is the devPublicKey
+      const isActiveArchiver = State.activeArchivers.some(
+        (archiver) => archiver.publicKey === data.sender
+      )
+
       const approvedSender =
-        State.activeArchivers.some((archiver) => archiver.publicKey === data.sender) ||
+        (isAllowedArchiver && isActiveArchiver) ||
         config.DevPublicKey === data.sender
+
       if (!approvedSender) {
-        return { success: false, error: 'Data request sender is not an archiver' }
+        return {
+          success: false,
+          error: isAllowedArchiver
+            ? 'Archiver is not active'
+            : 'Data request sender is not an authorized archiver'
+        }
       }
     }
-    if (!Crypto.verify(data)) {
-      Logger.mainLogger.error('Invalid signature', data)
-      return { success: false, error: 'Invalid signature' }
-    }
     return { success: true }
   } catch (e) {
     Logger.mainLogger.error('Error validating request data', e)

src/GlobalAccount.ts

@@ -8,6 +8,7 @@ import { postJson, getJson } from './P2P'
 import { robustQuery, deepCopy } from './Utils'
 import { isDeepStrictEqual } from 'util'
 import { accountSpecificHash } from './shardeum/calculateAccountHash'
+import { allowedArchiversManager } from './shardeum/allowedArchiversManager'
 
 let cachedGlobalNetworkAccount: AccountDB.AccountsCopy
 let cachedGlobalNetworkAccountHash: string
@@ -35,6 +36,11 @@ export function getGlobalNetworkAccount(hash: boolean): object | string {
 export function setGlobalNetworkAccount(account: AccountDB.AccountsCopy): void {
   cachedGlobalNetworkAccount = rfdc()(account)
   cachedGlobalNetworkAccountHash = account.hash
+  // Get the latest change if any
+  allowedArchiversManager.setGlobalAccountConfig(
+    account.data?.listOfChanges?.[account.data?.listOfChanges?.length - 1]?.change?.debug?.multisigKeys,
+    account.data?.listOfChanges?.[account.data?.listOfChanges?.length - 1]?.change?.debug?.minSigRequiredForArchiverWhitelist
+  )
 }
 
 interface NetworkConfigChanges {

src/LostArchivers.ts

@@ -8,6 +8,7 @@ import { calcIncomingTimes } from './Data/Data'
 import { postJson } from './P2P'
 import { sign } from './Crypto'
 import { SignedObject } from '@shardeum-foundation/lib-types/build/src/p2p/P2PTypes'
+import { allowedArchiversManager } from './shardeum/allowedArchiversManager'
 
 let shouldSendRefutes = false
 
@@ -101,5 +102,6 @@ function die(): void {
   Logger.mainLogger.debug(
     'Archiver was found in `removedArchivers` and will exit now without sending a leave request'
   )
+  allowedArchiversManager.stopWatching()
   process.exit(2)
 }

src/State.ts

@@ -11,6 +11,7 @@ import { publicKey, secretKey, curvePublicKey, curveSecretKey } from '@shardeum-
 import fetch from 'node-fetch'
 import { getAdjacentLeftAndRightArchivers } from './Data/GossipData'
 import { closeDatabase } from './dbstore'
+import { allowedArchiversManager } from './shardeum/allowedArchiversManager'
 
 export interface ArchiverNodeState {
   ip: string
@@ -136,6 +137,7 @@ export async function exitArchiver(): Promise<void> {
     }
     Logger.mainLogger.debug('Archiver will exit in 3 seconds.')
     setTimeout(() => {
+      allowedArchiversManager.stopWatching()
       process.exit()
     }, 3000)
   } catch (e) {
@@ -148,6 +150,7 @@ export function addSigListeners(sigint = true, sigterm = true): void {
     process.on('SIGINT', async () => {
       Logger.mainLogger.debug('Exiting on SIGINT', process.pid)
       await closeDatabase()
+      allowedArchiversManager.stopWatching()
       if (isActive) exitArchiver()
       else process.exit(0)
     })
@@ -156,11 +159,12 @@ export function addSigListeners(sigint = true, sigterm = true): void {
     process.on('SIGTERM', async () => {
       Logger.mainLogger.debug('Exiting on SIGTERM', process.pid)
       await closeDatabase()
+      allowedArchiversManager.stopWatching()
       if (isActive) exitArchiver()
       else process.exit(0)
     })
   }
-  Logger.mainLogger.debug('Registerd exit signal listeners.')
+  Logger.mainLogger.debug('Registered exit signal listeners.')
 }
 
 export function addArchiver(archiver: ArchiverNodeInfo): void {

src/Utils.ts

@@ -62,8 +62,8 @@ export interface SequentialQueryResult<Node> {
 export function shuffleArray<T>(array: T[]): void {
   for (let i = array.length - 1; i > 0; i--) {
     const j = Math.floor(Math.random() * (i + 1))
-    // eslint-disable-next-line security/detect-object-injection
-    ;[array[i], array[j]] = [array[j], array[i]]
+      // eslint-disable-next-line security/detect-object-injection
+      ;[array[i], array[j]] = [array[j], array[i]]
   }
 }
 
@@ -275,8 +275,7 @@ export async function robustQuery<Node = unknown, Response = unknown>(
     //        This change would require also changing all the places it is called.
     if (!disableFailLog)
       Logger.mainLogger.error(
-        `Could not get ${redundancy} ${
-          redundancy > 1 ? 'redundant responses' : 'response'
+        `Could not get ${redundancy} ${redundancy > 1 ? 'redundant responses' : 'response'
         } from ${nodeCount} ${nodeCount !== 1 ? 'nodes' : 'node'}. Encountered ${errors} query errors.`
       )
     console.trace()

src/server.ts

@@ -45,8 +45,10 @@ import { healthCheckRouter } from './routes/healthCheck'
 import { initializeTickets } from './routes/tickets';
 import { initAjvSchemas } from './types/ajv/Helpers'
 import { initializeSerialization } from './utils/serialization/SchemaHelpers'
+import { allowedArchiversManager } from './shardeum/allowedArchiversManager'
 
 const configFile = join(process.cwd(), 'archiver-config.json')
+const allowedArchiversConfigPath = join(__dirname, '../allowed-archivers.json')
 let logDir: string
 const cluster = clusterModule as unknown as clusterModule.Cluster
 
@@ -70,14 +72,15 @@ async function start(): Promise<void> {
   if (logsConfig.saveConsoleOutput) {
     startSaving(join(baseDir, logsConfig.dir))
   }
-
+  // Initialize allowed archivers manager
+  allowedArchiversManager.initialize(allowedArchiversConfigPath)
   // Global error handling
   process.on('uncaughtException', (error) => {
     Logger.mainLogger.error('Uncaught Exception - Global:', error);
   });
 
   process.on('unhandledRejection', (reason, promise) => {
-      Logger.mainLogger.error('Unhandled Rejection - Global:', promise, 'reason:', reason);
+    Logger.mainLogger.error('Unhandled Rejection - Global:', promise, 'reason:', reason);
   });
 
   // Initialize storage
@@ -502,10 +505,10 @@ async function startServer(): Promise<void> {
 
 // Add this before starting the server
 try {
-    initializeTickets();
+  initializeTickets();
 } catch (err) {
-    console.error('Failed to initialize tickets. Server startup aborted:', err);
-    process.exit(1);
+  console.error('Failed to initialize tickets. Server startup aborted:', err);
+  process.exit(1);
 }
 
 start()

src/services/ticketVerification.ts

@@ -31,6 +31,8 @@ function validateVerificationConfig(config: VerificationConfig): void {
         throw new Error('Invalid requiredSecurityLevel');
     }
 }
+
+// TODO: consider moving it to Utils.ts
 export function verifyMultiSigs(
     rawPayload: object,
     sigs: Sign[],