Merge pull request #20 from orbs-network/vc-stuck

vc stuck logic to rely on topology together with last commit

e2e/driver.ts

@@ -37,7 +37,7 @@ export class TestEnvironment {
       VchainUptimeRequiredSeconds: 2,
       VchainSyncThresholdSeconds: 5 * 60,
       VchainOutOfSyncThresholdSeconds: 60 * 60,
-      VchainStuckThresholdSeconds: 60 * 60,
+      VchainStuckThresholdSeconds: 2 * 60 * 60,
       EthereumSyncRequirementSeconds: 20 * 60,
       FailToSyncVcsTimeoutSeconds: 24 * 60 * 60,
       ElectionsRefreshWindowSeconds: 2 * 60 * 60,

e2e/index.test.ts

@@ -85,6 +85,7 @@ test.serial('[E2E] launches with one vchain out of sync -> sends ready-to-sync',
     },
     TimeEnteredStandbyWithoutVcSync: 0,
     TimeEnteredBadReputation: {},
+    TimeEnteredTopology: -1,
   });
   t.deepEqual(errors, []);
 
@@ -161,6 +162,7 @@ test.serial('[E2E] all vchains synced -> sends ready-for-committee', async (t) =
     },
     TimeEnteredStandbyWithoutVcSync: 0,
     TimeEnteredBadReputation: {},
+    TimeEnteredTopology: -1,
   });
   t.deepEqual(errors, []);
 
@@ -269,6 +271,7 @@ test.serial('[E2E] enter committee -> sends vote unready for bad rep', async (t)
         '43': 0,
       },
     },
+    TimeEnteredTopology: -1,
   });
   t.deepEqual(errors, []);
 

src/config.example.ts

@@ -15,7 +15,7 @@ export const exampleConfig: Configuration = {
   VchainUptimeRequiredSeconds: 2,
   VchainSyncThresholdSeconds: 5 * 60,
   VchainOutOfSyncThresholdSeconds: 60 * 60,
-  VchainStuckThresholdSeconds: 60 * 60,
+  VchainStuckThresholdSeconds: 2 * 60 * 60,
   EthereumSyncRequirementSeconds: 20 * 60,
   FailToSyncVcsTimeoutSeconds: 24 * 60 * 60,
   ElectionsRefreshWindowSeconds: 2 * 60 * 60,

src/config.ts

@@ -36,7 +36,7 @@ export const defaultConfiguration = {
   VchainUptimeRequiredSeconds: 5,
   VchainSyncThresholdSeconds: 5 * 60,
   VchainOutOfSyncThresholdSeconds: 60 * 60,
-  VchainStuckThresholdSeconds: 60 * 60,
+  VchainStuckThresholdSeconds: 2 * 60 * 60,
   EthereumBalancePollTimeSeconds: 4 * 60 * 60,
   EthereumCanJoinCommitteePollTimeSeconds: 10 * 60,
   EthereumSyncRequirementSeconds: 20 * 60,

src/index.ts

@@ -12,6 +12,7 @@ import {
   shouldNotifyReadyForCommittee,
   shouldNotifyReadyToSync,
   shouldCheckCanJoinCommittee,
+  calcTimeEnteredTopology,
 } from './model/logic-elections';
 import { getAllGuardiansToVoteUnready } from './model/logic-voteunready';
 import Signer from 'orbs-signer-client';
@@ -87,6 +88,14 @@ async function runLoopTick(config: Configuration, state: State) {
 
   // STEP 2: update all state machine logic (compute)
 
+  // time entered topology
+  const newTimeEnteredTopology = calcTimeEnteredTopology(state, config);
+  if (newTimeEnteredTopology != state.TimeEnteredTopology) {
+    const logMessage = state.TimeEnteredTopology == -1 ? `Exited topology` : `Entered topology`;
+    Logger.log(logMessage);
+    state.TimeEnteredTopology = newTimeEnteredTopology;
+  }
+
   // vchain sync status state machine
   const newVchainSyncStatus = calcVchainSyncStatus(state, config);
   if (newVchainSyncStatus != state.VchainSyncStatus) {

src/model/logic-elections.ts

@@ -1,6 +1,8 @@
 import { State } from './state';
 import * as Logger from '../logger';
-import { getToday } from '../helpers';
+import { getToday, getCurrentClockTime } from '../helpers';
+import { findEthFromOrbsAddress } from './helpers';
+import { Configuration } from '../config';
 
 const MAX_STANDBYS = 5; // in future, can be taken from the MaxStandbysChanged event
 
@@ -77,6 +79,16 @@ export function shouldCheckCanJoinCommittee(state: State, config: EthereumElecti
   return true;
 }
 
+// lower bound on time duration -> not opting for vc stuck and resulting in lazy "vcs in sync"
+export function calcTimeEnteredTopology(state: State, config: Configuration): number {
+  const myEthAddress = findEthFromOrbsAddress(config.NodeOrbsAddress, state);
+  if (state.ManagementCurrentTopology.some((node) => node.EthAddress == myEthAddress)) {
+    if (state.TimeEnteredTopology == -1) return getCurrentClockTime();
+    else return state.TimeEnteredTopology;
+  }
+  return -1;
+}
+
 // helpers
 
 export interface EthereumElectionsParams {

src/model/logic-vcsync.test.ts

@@ -8,6 +8,7 @@ import { exampleConfig } from '../config.example';
 function getExampleState() {
   const exampleState = new State();
   exampleState.ManagementRefTime = getCurrentClockTime();
+  exampleState.TimeEnteredTopology = getCurrentClockTime() - 24 * 60 * 60;
   exampleState.VchainMetrics['1000'] = {
     LastBlockHeight: 5000,
     UptimeSeconds: 3000,
@@ -54,11 +55,19 @@ test('in sync becomes out of sync and returns to sync', (t) => {
   state.VchainSyncStatus = calcVchainSyncStatus(state, exampleConfig);
   t.is(state.VchainSyncStatus, 'exist-not-in-sync');
 
-  state.VchainMetrics['1001'].LastCommitTime = getCurrentClockTime() - 24 * 60 * 60; // becomes VCStuck
+  state.VchainMetrics['1001'].LastCommitTime = getCurrentClockTime() - 24 * 60 * 60; // together with TimeEnteredTopology in setup becomes VCStuck
   state.VchainSyncStatus = calcVchainSyncStatus(state, exampleConfig);
   t.is(state.VchainSyncStatus, 'in-sync');
 
-  state.VchainMetrics['1001'].LastCommitTime = getCurrentClockTime() - 38; // no longer VCStuck
+  state.TimeEnteredTopology = getCurrentClockTime() - 1 * 60 * 60; // no longer VCStuck 1
+  state.VchainSyncStatus = calcVchainSyncStatus(state, exampleConfig);
+  t.is(state.VchainSyncStatus, 'exist-not-in-sync');
+
+  state.TimeEnteredTopology = getCurrentClockTime() - 24 * 60 * 60; // together with old LastCommitTime return to VCStuck
+  state.VchainSyncStatus = calcVchainSyncStatus(state, exampleConfig);
+  t.is(state.VchainSyncStatus, 'in-sync');
+
+  state.VchainMetrics['1001'].LastCommitTime = getCurrentClockTime() - 38; // no longer VCStuck 2
   state.VchainSyncStatus = calcVchainSyncStatus(state, exampleConfig);
   t.is(state.VchainSyncStatus, 'exist-not-in-sync');
 

src/model/logic-vcsync.ts

@@ -55,8 +55,14 @@ function isVcLive(vcId: string, state: State, config: VchainSyncStatusParams): b
 function isVcStuck(vcId: string, state: State, config: VchainSyncStatusParams): boolean {
   if (!state.VchainMetrics[vcId]) return false;
   const lastCommitTime = state.VchainMetrics[vcId].LastCommitTime;
+  const timeEnteredTopology = state.TimeEnteredTopology;
   if (lastCommitTime == -1) return false;
+  if (timeEnteredTopology == -1) return false;
   const now = getCurrentClockTime();
-  if (now - lastCommitTime > config.VchainStuckThresholdSeconds) return true;
+  if (
+    now - lastCommitTime > config.VchainStuckThresholdSeconds &&
+    now - timeEnteredTopology > config.VchainStuckThresholdSeconds
+  )
+    return true;
   return false;
 }

src/model/state.ts

@@ -21,6 +21,7 @@ export class State {
   ManagementOthersElectionsStatus: { [EthAddress: string]: ManagementElectionsStatus | undefined } = {};
   ManagementCurrentCommittee: CommitteeMember[] = [];
   ManagementCurrentStandbys: { EthAddress: string }[] = [];
+  ManagementCurrentTopology: { EthAddress: string }[] = [];
 
   // updated by read/vchain-metrics.ts
   VchainMetricsLastPollTime = 0; // UTC seconds
@@ -42,6 +43,7 @@ export class State {
   EthereumFeesStats: { [month: string]: number } = {}; // number in eth
 
   // updated by index.ts
+  TimeEnteredTopology = -1; // UTC seconds
   VchainSyncStatus: VchainSyncStatusEnum = 'not-exist';
   EthereumSyncStatus: EthereumSyncStatusEnum = 'out-of-sync';
 

src/read/management.test.ts

@@ -42,6 +42,43 @@ const validManagementStatusResponse = {
         Name: 'Guardian0',
       },
     ],
+    CurrentTopology: [
+      {
+        EthAddress: '29ce860a2247d97160d6dfc087a15f41e2349087',
+        OrbsAddress: '16fcf728f8dc3f687132f2157d8379c021a08c12',
+        Ip: '41.206.134.10',
+        Port: 0,
+        Name: 'Guardian0',
+      },
+      {
+        EthAddress: 'e16e965a4cc3fcd597ecdb9cd9ab8f3e6a750ac9',
+        OrbsAddress: '86544bdd6c8b957cd198252c45fa215fc3892126',
+        Ip: '225.110.150.90',
+        Port: 0,
+        Name: 'Guardian1',
+      },
+      {
+        EthAddress: '51baa09f2f7dfc7a0f65886b68720958d389cac7',
+        OrbsAddress: '174dc3b45bdbbc32aa0b95e64d0247ce99b08f69',
+        Ip: '81.186.160.159',
+        Port: 0,
+        Name: 'Guardian2',
+      },
+      {
+        EthAddress: '8a670ddc1910c27278ab7db2a148a0dccc6bf0f5',
+        OrbsAddress: '7c2300d32ebf4a6ae9edf95f4f57ab5a07488c2e',
+        Ip: '138.103.13.220',
+        Port: 0,
+        Name: 'Guardian3',
+      },
+      {
+        EthAddress: 'cb6642be414696f77336dae06fed3775f08de0ea',
+        OrbsAddress: '33546759bdcfb5c753a4102b86b3e73e714d5213',
+        Ip: '203.102.66.190',
+        Port: 0,
+        Name: 'Guardian4',
+      },
+    ],
     Guardians: {
       '29ce860a2247d97160d6dfc087a15f41e2349087': {
         EthAddress: '29ce860a2247d97160d6dfc087a15f41e2349087',
@@ -223,6 +260,25 @@ test.serial('reads data from valid ManagementStatus', async (t) => {
       },
     ])
   );
+  t.assert(
+    _.isMatch(state.ManagementCurrentTopology, [
+      {
+        EthAddress: '29ce860a2247d97160d6dfc087a15f41e2349087',
+      },
+      {
+        EthAddress: 'e16e965a4cc3fcd597ecdb9cd9ab8f3e6a750ac9',
+      },
+      {
+        EthAddress: '51baa09f2f7dfc7a0f65886b68720958d389cac7',
+      },
+      {
+        EthAddress: '8a670ddc1910c27278ab7db2a148a0dccc6bf0f5',
+      },
+      {
+        EthAddress: 'cb6642be414696f77336dae06fed3775f08de0ea',
+      },
+    ])
+  );
 });
 
 test.serial('my orbsAddress not found in ManagementStatus', async (t) => {

src/read/management.ts

@@ -15,6 +15,7 @@ export async function readManagementStatus(endpoint: string, myOrbsAddress: stri
   state.ManagementVirtualChains = response.Payload.CurrentVirtualChains;
   state.ManagementCurrentCommittee = response.Payload.CurrentCommittee;
   state.ManagementCurrentStandbys = _.filter(response.Payload.CurrentCandidates, (node) => node.IsStandby);
+  state.ManagementCurrentTopology = response.Payload.CurrentTopology;
   state.ManagementEthToOrbsAddress = _.mapValues(response.Payload.Guardians, (node) => node.OrbsAddress);
 
   const myEthAddress = findEthFromOrbsAddress(myOrbsAddress, state);
@@ -50,6 +51,7 @@ interface ManagementStatusResponse {
     CurrentRefBlock: number;
     CurrentCommittee: { EthAddress: string; Weight: number }[];
     CurrentCandidates: { EthAddress: string; IsStandby: boolean }[];
+    CurrentTopology: { EthAddress: string }[];
     Guardians: {
       [EthAddress: string]: {
         OrbsAddress: string;
@@ -89,6 +91,11 @@ const managementStatusResponseDecoder: Decoder<ManagementStatusResponse> = objec
         IsStandby: bool,
       })
     ),
+    CurrentTopology: array(
+      object({
+        EthAddress: str,
+      })
+    ),
     Guardians: record(
       object({
         OrbsAddress: str,

src/write/status.test.ts

@@ -67,6 +67,7 @@ test.serial('contains all payload fields', (t) => {
     ManagementIsStandby: false,
     TimeEnteredStandbyWithoutVcSync: 0,
     TimeEnteredBadReputation: {},
+    TimeEnteredTopology: -1,
     Config: exampleConfig,
   });
 });

src/write/status.ts

@@ -42,6 +42,7 @@ export function writeStatusToDisk(filePath: string, state: State, config: Config
       ManagementMyElectionStatus: state.ManagementMyElectionsStatus,
       TimeEnteredStandbyWithoutVcSync: state.TimeEnteredStandbyWithoutVcSync,
       TimeEnteredBadReputation: state.TimeEnteredBadReputation,
+      TimeEnteredTopology: state.TimeEnteredTopology,
       Config: config,
     },
   };