Finish epoch processing split (#11243)

* added `getEpochInfoOfEpoch(uint256)` to get epoch info of specific epochs * updated natspec * PR feedback * Use constant * breakup test; added more * ++ registry to precompile contract * added elected to epoch info * numberValidatorsInSet backwards compatibility * validator address & validatorSigner address from set * support `getEpochNumber()` backwards compatibility in UsingRegistry * using `getEpochByBlockNumber` instead of `getEpochByNumber` * limit span of loop only to L2 blocks * use binary search instead of blind loop * ++ natspec * added getEpochNumberOfBlock() backwards compatibility support * ++ electedSigners * make usingPrecompile onlyL1 contract * ++ precompile override contract * fix epoch manager bug and mock * using precompileOverride in validators contract * wording * using indexed function when querying specific validator accounts or signers on L2 * fixed failing test * using independent mapping for elected accounts and signers * removed old elected list * support 0.5.13 contracts using precompiles * -- registry import * ++ registry in PrecompilesOverride * ++ more tests * increase gaslimit * moved election history update to epoch processing start. * unused var * clean up * fixed e2e test * simplify code * Finish epoch processing split * FinishEpochProcessing split e2e test * setToProcessGroups unit tests * process group unit tests * Removed support for election historical data after that it's unlikely to be in use by external parties * PR feedback * -- spacing * reorder state var * stat var reorder * small fix * delete electedSigners at end of epochprocessing * CI fix * added comment * prettify * Pr comments --------- Co-authored-by: soloseng <[email protected]> Co-authored-by: Martín Volpe <[email protected]>
celo-org · Oct 15, 2024 · 0ae4b8c · 0ae4b8c
1 parent 7566b18
commit 0ae4b8c
Show file tree

Hide file tree

Showing 5 changed files with 494 additions and 30 deletions.
diff --git a/packages/protocol/contracts-0.8/common/EpochManager.sol b/packages/protocol/contracts-0.8/common/EpochManager.sol
@@ -33,7 +33,8 @@ contract EpochManager is
 
   enum EpochProcessStatus {
     NotStarted,
-    Started
+    Started,
+    IndivudualGroupsProcessing
   }
 
   struct EpochProcessState {
@@ -62,6 +63,8 @@ contract EpochManager is
   // so we keep a copy
   address[] public electedSigners;
 
+  uint256 public toProcessGroups = 0;
+
   /**
    * @notice Event emited when epochProcessing has begun.
    * @param epochNumber The epoch number that is being processed.
@@ -205,6 +208,86 @@ contract EpochManager is
     emit EpochProcessingStarted(currentEpochNumber);
   }
 
+  /**
+   * @notice Starts individual processing of the elected groups.
+   * As second step it is necessary to call processGroup
+   */
+  function setToProcessGroups() external {
+    require(isOnEpochProcess(), "Epoch process is not started");
+
+    EpochProcessState storage _epochProcessing = epochProcessing;
+    _epochProcessing.status = EpochProcessStatus.IndivudualGroupsProcessing;
+
+    IValidators validators = getValidators();
+    IElection election = getElection();
+    IScoreReader scoreReader = getScoreReader();
+    require(
+      electedAccounts.length == electedSigners.length,
+      "Elected accounts and signers of different lengths."
+    );
+    for (uint i = 0; i < electedAccounts.length; i++) {
+      address group = validators.getValidatorsGroup(electedAccounts[i]);
+      if (processedGroups[group] == 0) {
+        toProcessGroups++;
+        uint256 groupScore = scoreReader.getGroupScore(group);
+        // We need to precompute epoch rewards for each group since computation depends on total active votes for all groups.
+        uint256 epochRewards = election.getGroupEpochRewardsBasedOnScore(
+          group,
+          _epochProcessing.totalRewardsVoter,
+          groupScore
+        );
+        processedGroups[group] = epochRewards == 0 ? type(uint256).max : epochRewards;
+      }
+    }
+  }
+
+  /**
+   * @notice Processes the rewards for a list of groups. For last group it will also finalize the epoch.
+   * @param groups List of validator groups to be processed.
+   * @param lessers List of validator groups that hold less votes that indexed group.
+   * @param greaters List of validator groups that hold more votes that indexed group.
+   */
+  function processGroups(
+    address[] calldata groups,
+    address[] calldata lessers,
+    address[] calldata greaters
+  ) external {
+    for (uint i = 0; i < groups.length; i++) {
+      processGroup(groups[i], lessers[i], greaters[i]);
+    }
+  }
+
+  /**
+   * @notice Processes the rewards for a group. For last group it will also finalize the epoch.
+   * @param group The group to process.
+   * @param lesser The group with less votes than the indexed group.
+   * @param greater The group with more votes than the indexed group.
+   */
+  function processGroup(address group, address lesser, address greater) public {
+    EpochProcessState storage _epochProcessing = epochProcessing;
+    require(
+      _epochProcessing.status == EpochProcessStatus.IndivudualGroupsProcessing,
+      "Indivudual epoch process is not started"
+    );
+    require(toProcessGroups > 0, "no more groups to process");
+
+    uint256 epochRewards = processedGroups[group];
+    // checks that group is actually from elected group
+    require(epochRewards > 0, "group not from current elected set");
+    IElection election = getElection();
+
+    if (epochRewards != type(uint256).max) {
+      election.distributeEpochRewards(group, epochRewards, lesser, greater);
+    }
+
+    delete processedGroups[group];
+    toProcessGroups--;
+
+    if (toProcessGroups == 0) {
+      _finishEpochHelper(_epochProcessing, election);
+    }
+  }
+
   /**
    * @notice Finishes processing an epoch and releasing funds to the beneficiaries.
    * @param groups List of validator groups to be processed.
@@ -217,17 +300,11 @@ contract EpochManager is
     address[] calldata greaters
   ) external virtual nonReentrant {
     require(isOnEpochProcess(), "Epoch process is not started");
-    // finalize epoch
-    // last block should be the block before and timestamp from previous block
-    epochs[currentEpochNumber].lastBlock = block.number - 1;
-    // start new epoch
-    currentEpochNumber++;
-    epochs[currentEpochNumber].firstBlock = block.number;
-    epochs[currentEpochNumber].startTimestamp = block.timestamp;
+    require(toProcessGroups == 0, "Can't finish epoch while individual groups are being processed");
 
     EpochProcessState storage _epochProcessing = epochProcessing;
 
-    uint256 toProcessGroups = 0;
+    uint256 _toProcessGroups = 0;
     IValidators validators = getValidators();
     IElection election = getElection();
     IScoreReader scoreReader = getScoreReader();
@@ -238,7 +315,7 @@ contract EpochManager is
     for (uint i = 0; i < electedAccounts.length; i++) {
       address group = validators.getValidatorsGroup(electedAccounts[i]);
       if (processedGroups[group] == 0) {
-        toProcessGroups++;
+        _toProcessGroups++;
         uint256 groupScore = scoreReader.getGroupScore(group);
         // We need to precompute epoch rewards for each group since computation depends on total active votes for all groups.
         uint256 epochRewards = election.getGroupEpochRewardsBasedOnScore(
@@ -252,7 +329,7 @@ contract EpochManager is
       delete electedSigners[i];
     }
 
-    require(toProcessGroups == groups.length, "number of groups does not match");
+    require(_toProcessGroups == groups.length, "number of groups does not match");
 
     for (uint i = 0; i < groups.length; i++) {
       uint256 epochRewards = processedGroups[groups[i]];
@@ -264,26 +341,8 @@ contract EpochManager is
 
       delete processedGroups[groups[i]];
     }
-    getCeloUnreleasedTreasury().release(
-      registry.getAddressForOrDie(GOVERNANCE_REGISTRY_ID),
-      epochProcessing.totalRewardsCommunity
-    );
-    getCeloUnreleasedTreasury().release(
-      getEpochRewards().carbonOffsettingPartner(),
-      epochProcessing.totalRewardsCarbonFund
-    );
-    // run elections
-
-    address[] memory _newlyElected = election.electValidatorAccounts();
 
-    electedAccounts = _newlyElected;
-
-    _setElectedSigners(_newlyElected);
-
-    EpochProcessState memory _epochProcessingEmpty;
-    epochProcessing = _epochProcessingEmpty;
-
-    emit EpochProcessingEnded(currentEpochNumber - 1);
+    _finishEpochHelper(_epochProcessing, election);
   }
 
   /**
@@ -614,6 +673,47 @@ contract EpochManager is
     }
   }
 
+  /**
+   * @notice Finishes processing an epoch and releasing funds to the beneficiaries.
+   * @param _epochProcessing The current epoch processing state.
+   * @param election The Election contract.
+   */
+  function _finishEpochHelper(
+    EpochProcessState storage _epochProcessing,
+    IElection election
+  ) internal {
+    // finalize epoch
+    // last block should be the block before and timestamp from previous block
+    epochs[currentEpochNumber].lastBlock = block.number - 1;
+    currentEpochNumber++;
+    // start new epoch
+    epochs[currentEpochNumber].firstBlock = block.number;
+    epochs[currentEpochNumber].startTimestamp = block.timestamp;
+
+    // run elections
+    address[] memory _newlyElected = election.electValidatorAccounts();
+    electedAccounts = _newlyElected;
+    _setElectedSigners(_newlyElected);
+
+    ICeloUnreleasedTreasury celoUnreleasedTreasury = getCeloUnreleasedTreasury();
+    celoUnreleasedTreasury.release(
+      registry.getAddressForOrDie(GOVERNANCE_REGISTRY_ID),
+      _epochProcessing.totalRewardsCommunity
+    );
+    celoUnreleasedTreasury.release(
+      getEpochRewards().carbonOffsettingPartner(),
+      _epochProcessing.totalRewardsCarbonFund
+    );
+
+    _epochProcessing.status = EpochProcessStatus.NotStarted;
+    _epochProcessing.perValidatorReward = 0;
+    _epochProcessing.totalRewardsVoter = 0;
+    _epochProcessing.totalRewardsCommunity = 0;
+    _epochProcessing.totalRewardsCarbonFund = 0;
+
+    emit EpochProcessingEnded(currentEpochNumber - 1);
+  }
+
   /**
    * @notice Returns the epoch info of a specified blockNumber.
    * @dev This function is here for backward compatibility. It is rather gas heavy and can run out of gas.

diff --git a/packages/protocol/contracts-0.8/common/test/MockEpochManager.sol b/packages/protocol/contracts-0.8/common/test/MockEpochManager.sol
@@ -85,6 +85,9 @@ contract MockEpochManager is IEpochManager {
     epochProcessing = _epochProcessingEmpty;
   }
 
+  function setToProcessGroups() external {}
+  function processGroup(address group, address lesser, address greater) external {}
+
   function setIsTimeForNextEpoch(bool _isTime) external {
     _isTimeForNextEpoch = _isTime;
   }

diff --git a/packages/protocol/contracts/common/interfaces/IEpochManager.sol b/packages/protocol/contracts/common/interfaces/IEpochManager.sol
@@ -13,6 +13,8 @@ interface IEpochManager {
     address[] calldata lessers,
     address[] calldata greaters
   ) external;
+  function setToProcessGroups() external;
+  function processGroup(address group, address lesser, address greater) external;
   function sendValidatorPayment(address) external;
   function getCurrentEpoch() external view returns (uint256, uint256, uint256, uint256);
   function getEpochByNumber(

diff --git a/packages/protocol/test-sol/devchain/e2e/common/EpochManager.t.sol b/packages/protocol/test-sol/devchain/e2e/common/EpochManager.t.sol
@@ -746,3 +746,143 @@ contract E2E_GasTest2_FinishNextEpochProcess is E2E_GasTest_Setup {
     console.log("elected count2: ", epochManager.getElectedAccounts().length);
   }
 }
+
+contract E2E_FinishNextEpochProcess_Split is E2E_GasTest_Setup {
+  using EnumerableSet for EnumerableSet.AddressSet;
+
+  function setUp() public override {
+    super.setUp();
+
+    activateValidators();
+    whenL2(vm);
+
+    vm.prank(epochManagerEnabler);
+    epochManager.initializeSystem(1, 1, firstElected);
+
+    validatorsArray = getValidators().getRegisteredValidators();
+    groups = getValidators().getRegisteredValidatorGroups();
+
+    vm.startPrank(scoreManager.owner());
+    scoreManager.setGroupScore(groups[0], groupScore[0]);
+    scoreManager.setGroupScore(groups[1], groupScore[1]);
+    scoreManager.setGroupScore(groups[2], groupScore[2]);
+
+    scoreManager.setValidatorScore(validatorsArray[0], validatorScore[0]);
+    scoreManager.setValidatorScore(validatorsArray[1], validatorScore[1]);
+    scoreManager.setValidatorScore(validatorsArray[2], validatorScore[2]);
+    scoreManager.setValidatorScore(validatorsArray[3], validatorScore[3]);
+    scoreManager.setValidatorScore(validatorsArray[4], validatorScore[4]);
+    scoreManager.setValidatorScore(validatorsArray[5], validatorScore[5]);
+
+    vm.stopPrank();
+
+    timeTravel(vm, epochDuration + 1);
+    epochManager.startNextEpochProcess();
+
+    address[] memory lessers;
+    address[] memory greaters;
+    address[] memory groupsEligible;
+    GroupWithVotes[] memory groupWithVotes;
+    uint256[] memory groupActiveBalances;
+    (lessers, greaters, groupWithVotes) = getLessersAndGreaters(groups);
+
+    uint256 currentEpoch = epochManager.getCurrentEpochNumber();
+    address[] memory currentlyElected = epochManager.getElectedAccounts();
+    for (uint256 i = 0; i < currentlyElected.length; i++) {
+      originalyElected.add(currentlyElected[i]);
+    }
+
+    // wait some time before finishing
+    timeTravel(vm, epochDuration / 2);
+    blockTravel(vm, 100);
+
+    epochManager.setToProcessGroups();
+    for (uint256 i = 0; i < groups.length; i++) {
+      epochManager.processGroup(groups[i], lessers[i], greaters[i]);
+    }
+
+    assertEq(currentEpoch + 1, epochManager.getCurrentEpochNumber());
+
+    address[] memory newlyElected = epochManager.getElectedAccounts();
+
+    for (uint256 i = 0; i < currentlyElected.length; i++) {
+      assertEq(originalyElected.contains(currentlyElected[i]), true);
+    }
+
+    timeTravel(vm, epochDuration + 1);
+    epochManager.startNextEpochProcess();
+
+    // wait some time before finishing
+    timeTravel(vm, epochDuration / 2);
+    blockTravel(vm, 100);
+
+    (lessers, greaters, groupWithVotes) = getLessersAndGreaters(groups);
+    epochManager.setToProcessGroups();
+    for (uint256 i = 0; i < groups.length; i++) {
+      epochManager.processGroup(groups[i], lessers[i], greaters[i]);
+    }
+    // epochManager.finishNextEpochProcess(groups, lessers, greaters);
+    assertGroupWithVotes(groupWithVotes);
+
+    assertEq(currentEpoch + 2, epochManager.getCurrentEpochNumber());
+
+    address[] memory newlyElected2 = epochManager.getElectedAccounts();
+
+    for (uint256 i = 0; i < currentlyElected.length; i++) {
+      assertEq(originalyElected.contains(newlyElected2[i]), true);
+    }
+    uint256 validatorGroupCount = 60;
+    uint256 validatorPerGroupCount = 2;
+
+    for (uint256 i = 0; i < validatorGroupCount; i++) {
+      (address newValidatorGroup, address newValidator) = registerNewValidatorGroupWithValidator(
+        i,
+        validatorPerGroupCount
+      );
+    }
+
+    timeTravel(vm, epochDuration + 1);
+    epochManager.startNextEpochProcess();
+
+    timeTravel(vm, epochDuration / 2);
+    blockTravel(vm, 100);
+
+    (lessers, greaters, groupWithVotes) = getLessersAndGreaters(groups);
+
+    epochManager.setToProcessGroups();
+    for (uint256 i = 0; i < groups.length; i++) {
+      epochManager.processGroup(groups[i], lessers[i], greaters[i]);
+    }
+
+    activateValidators();
+
+    timeTravel(vm, epochDuration + 1);
+    epochManager.startNextEpochProcess();
+
+    groups = getCurrentlyElectedGroups();
+
+    timeTravel(vm, epochDuration / 2);
+    blockTravel(vm, 100);
+  }
+
+  /**
+    * @notice Test the gas used by finishNextEpochProcess
+    This test is trying to measure gas used by finishNextEpochProcess in a real life worst case. We have 126 validators and 123 groups.
+    There are two main loops in the function, one for calculating rewards and the other for updating the elected validators.
+    FinishNextEpochProcess is called twice, first time with going from 6 -> 110 validators which consumes approx. 6M gas and the second time with going from 110 -> 110 validators which consumes approx. 19M gas. 
+     */
+  function test_shouldFinishNextEpochProcessing_GasTest_Split() public {
+    address[] memory lessers;
+    address[] memory greaters;
+    GroupWithVotes[] memory groupWithVotes;
+    (lessers, greaters, groupWithVotes) = getLessersAndGreaters(groups);
+    epochManager.setToProcessGroups();
+
+    for (uint256 i = 0; i < groups.length; i++) {
+      uint256 gasLeftBefore1 = gasleft();
+      epochManager.processGroup(groups[i], lessers[i], greaters[i]);
+      uint256 gasLeftAfter1 = gasleft();
+      console.log("processGroup gas used: ", gasLeftBefore1 - gasLeftAfter1);
+    }
+  }
+}