From 0ae4b8c3b64a75b14f3b2332c89ce3d7e2cd8ec7 Mon Sep 17 00:00:00 2001 From: pahor167 <47992132+pahor167@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:16:15 +0200 Subject: [PATCH] Finish epoch processing split (#11243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * added `getEpochInfoOfEpoch(uint256)` to get epoch info of specific epochs * updated natspec * PR feedback * Use constant * breakup test; added more * ++ registry to precompile contract * added elected to epoch info * numberValidatorsInSet backwards compatibility * validator address & validatorSigner address from set * support `getEpochNumber()` backwards compatibility in UsingRegistry * using `getEpochByBlockNumber` instead of `getEpochByNumber` * limit span of loop only to L2 blocks * use binary search instead of blind loop * ++ natspec * added getEpochNumberOfBlock() backwards compatibility support * ++ electedSigners * make usingPrecompile onlyL1 contract * ++ precompile override contract * fix epoch manager bug and mock * using precompileOverride in validators contract * wording * using indexed function when querying specific validator accounts or signers on L2 * fixed failing test * using independent mapping for elected accounts and signers * removed old elected list * support 0.5.13 contracts using precompiles * -- registry import * ++ registry in PrecompilesOverride * ++ more tests * increase gaslimit * moved election history update to epoch processing start. * unused var * clean up * fixed e2e test * simplify code * Finish epoch processing split * FinishEpochProcessing split e2e test * setToProcessGroups unit tests * process group unit tests * Removed support for election historical data after that it's unlikely to be in use by external parties * PR feedback * -- spacing * reorder state var * stat var reorder * small fix * delete electedSigners at end of epochprocessing * CI fix * added comment * prettify * Pr comments --------- Co-authored-by: soloseng <102702451+soloseng@users.noreply.github.com> Co-authored-by: Martín Volpe --- .../contracts-0.8/common/EpochManager.sol | 160 ++++++++++--- .../common/test/MockEpochManager.sol | 3 + .../common/interfaces/IEpochManager.sol | 2 + .../devchain/e2e/common/EpochManager.t.sol | 140 +++++++++++ .../test-sol/unit/common/EpochManager.t.sol | 219 ++++++++++++++++++ 5 files changed, 494 insertions(+), 30 deletions(-) diff --git a/packages/protocol/contracts-0.8/common/EpochManager.sol b/packages/protocol/contracts-0.8/common/EpochManager.sol index 9e07b80017f..966540c90d3 100644 --- a/packages/protocol/contracts-0.8/common/EpochManager.sol +++ b/packages/protocol/contracts-0.8/common/EpochManager.sol @@ -33,7 +33,8 @@ contract EpochManager is enum EpochProcessStatus { NotStarted, - Started + Started, + IndivudualGroupsProcessing } struct EpochProcessState { @@ -62,6 +63,8 @@ contract EpochManager is // so we keep a copy address[] public electedSigners; + uint256 public toProcessGroups = 0; + /** * @notice Event emited when epochProcessing has begun. * @param epochNumber The epoch number that is being processed. @@ -205,6 +208,86 @@ contract EpochManager is emit EpochProcessingStarted(currentEpochNumber); } + /** + * @notice Starts individual processing of the elected groups. + * As second step it is necessary to call processGroup + */ + function setToProcessGroups() external { + require(isOnEpochProcess(), "Epoch process is not started"); + + EpochProcessState storage _epochProcessing = epochProcessing; + _epochProcessing.status = EpochProcessStatus.IndivudualGroupsProcessing; + + IValidators validators = getValidators(); + IElection election = getElection(); + IScoreReader scoreReader = getScoreReader(); + require( + electedAccounts.length == electedSigners.length, + "Elected accounts and signers of different lengths." + ); + for (uint i = 0; i < electedAccounts.length; i++) { + address group = validators.getValidatorsGroup(electedAccounts[i]); + if (processedGroups[group] == 0) { + toProcessGroups++; + uint256 groupScore = scoreReader.getGroupScore(group); + // We need to precompute epoch rewards for each group since computation depends on total active votes for all groups. + uint256 epochRewards = election.getGroupEpochRewardsBasedOnScore( + group, + _epochProcessing.totalRewardsVoter, + groupScore + ); + processedGroups[group] = epochRewards == 0 ? type(uint256).max : epochRewards; + } + } + } + + /** + * @notice Processes the rewards for a list of groups. For last group it will also finalize the epoch. + * @param groups List of validator groups to be processed. + * @param lessers List of validator groups that hold less votes that indexed group. + * @param greaters List of validator groups that hold more votes that indexed group. + */ + function processGroups( + address[] calldata groups, + address[] calldata lessers, + address[] calldata greaters + ) external { + for (uint i = 0; i < groups.length; i++) { + processGroup(groups[i], lessers[i], greaters[i]); + } + } + + /** + * @notice Processes the rewards for a group. For last group it will also finalize the epoch. + * @param group The group to process. + * @param lesser The group with less votes than the indexed group. + * @param greater The group with more votes than the indexed group. + */ + function processGroup(address group, address lesser, address greater) public { + EpochProcessState storage _epochProcessing = epochProcessing; + require( + _epochProcessing.status == EpochProcessStatus.IndivudualGroupsProcessing, + "Indivudual epoch process is not started" + ); + require(toProcessGroups > 0, "no more groups to process"); + + uint256 epochRewards = processedGroups[group]; + // checks that group is actually from elected group + require(epochRewards > 0, "group not from current elected set"); + IElection election = getElection(); + + if (epochRewards != type(uint256).max) { + election.distributeEpochRewards(group, epochRewards, lesser, greater); + } + + delete processedGroups[group]; + toProcessGroups--; + + if (toProcessGroups == 0) { + _finishEpochHelper(_epochProcessing, election); + } + } + /** * @notice Finishes processing an epoch and releasing funds to the beneficiaries. * @param groups List of validator groups to be processed. @@ -217,17 +300,11 @@ contract EpochManager is address[] calldata greaters ) external virtual nonReentrant { require(isOnEpochProcess(), "Epoch process is not started"); - // finalize epoch - // last block should be the block before and timestamp from previous block - epochs[currentEpochNumber].lastBlock = block.number - 1; - // start new epoch - currentEpochNumber++; - epochs[currentEpochNumber].firstBlock = block.number; - epochs[currentEpochNumber].startTimestamp = block.timestamp; + require(toProcessGroups == 0, "Can't finish epoch while individual groups are being processed"); EpochProcessState storage _epochProcessing = epochProcessing; - uint256 toProcessGroups = 0; + uint256 _toProcessGroups = 0; IValidators validators = getValidators(); IElection election = getElection(); IScoreReader scoreReader = getScoreReader(); @@ -238,7 +315,7 @@ contract EpochManager is for (uint i = 0; i < electedAccounts.length; i++) { address group = validators.getValidatorsGroup(electedAccounts[i]); if (processedGroups[group] == 0) { - toProcessGroups++; + _toProcessGroups++; uint256 groupScore = scoreReader.getGroupScore(group); // We need to precompute epoch rewards for each group since computation depends on total active votes for all groups. uint256 epochRewards = election.getGroupEpochRewardsBasedOnScore( @@ -252,7 +329,7 @@ contract EpochManager is delete electedSigners[i]; } - require(toProcessGroups == groups.length, "number of groups does not match"); + require(_toProcessGroups == groups.length, "number of groups does not match"); for (uint i = 0; i < groups.length; i++) { uint256 epochRewards = processedGroups[groups[i]]; @@ -264,26 +341,8 @@ contract EpochManager is delete processedGroups[groups[i]]; } - getCeloUnreleasedTreasury().release( - registry.getAddressForOrDie(GOVERNANCE_REGISTRY_ID), - epochProcessing.totalRewardsCommunity - ); - getCeloUnreleasedTreasury().release( - getEpochRewards().carbonOffsettingPartner(), - epochProcessing.totalRewardsCarbonFund - ); - // run elections - - address[] memory _newlyElected = election.electValidatorAccounts(); - electedAccounts = _newlyElected; - - _setElectedSigners(_newlyElected); - - EpochProcessState memory _epochProcessingEmpty; - epochProcessing = _epochProcessingEmpty; - - emit EpochProcessingEnded(currentEpochNumber - 1); + _finishEpochHelper(_epochProcessing, election); } /** @@ -614,6 +673,47 @@ contract EpochManager is } } + /** + * @notice Finishes processing an epoch and releasing funds to the beneficiaries. + * @param _epochProcessing The current epoch processing state. + * @param election The Election contract. + */ + function _finishEpochHelper( + EpochProcessState storage _epochProcessing, + IElection election + ) internal { + // finalize epoch + // last block should be the block before and timestamp from previous block + epochs[currentEpochNumber].lastBlock = block.number - 1; + currentEpochNumber++; + // start new epoch + epochs[currentEpochNumber].firstBlock = block.number; + epochs[currentEpochNumber].startTimestamp = block.timestamp; + + // run elections + address[] memory _newlyElected = election.electValidatorAccounts(); + electedAccounts = _newlyElected; + _setElectedSigners(_newlyElected); + + ICeloUnreleasedTreasury celoUnreleasedTreasury = getCeloUnreleasedTreasury(); + celoUnreleasedTreasury.release( + registry.getAddressForOrDie(GOVERNANCE_REGISTRY_ID), + _epochProcessing.totalRewardsCommunity + ); + celoUnreleasedTreasury.release( + getEpochRewards().carbonOffsettingPartner(), + _epochProcessing.totalRewardsCarbonFund + ); + + _epochProcessing.status = EpochProcessStatus.NotStarted; + _epochProcessing.perValidatorReward = 0; + _epochProcessing.totalRewardsVoter = 0; + _epochProcessing.totalRewardsCommunity = 0; + _epochProcessing.totalRewardsCarbonFund = 0; + + emit EpochProcessingEnded(currentEpochNumber - 1); + } + /** * @notice Returns the epoch info of a specified blockNumber. * @dev This function is here for backward compatibility. It is rather gas heavy and can run out of gas. diff --git a/packages/protocol/contracts-0.8/common/test/MockEpochManager.sol b/packages/protocol/contracts-0.8/common/test/MockEpochManager.sol index a42bbaaa30a..27168f0f2ca 100644 --- a/packages/protocol/contracts-0.8/common/test/MockEpochManager.sol +++ b/packages/protocol/contracts-0.8/common/test/MockEpochManager.sol @@ -85,6 +85,9 @@ contract MockEpochManager is IEpochManager { epochProcessing = _epochProcessingEmpty; } + function setToProcessGroups() external {} + function processGroup(address group, address lesser, address greater) external {} + function setIsTimeForNextEpoch(bool _isTime) external { _isTimeForNextEpoch = _isTime; } diff --git a/packages/protocol/contracts/common/interfaces/IEpochManager.sol b/packages/protocol/contracts/common/interfaces/IEpochManager.sol index ccad6d3a1c5..72c54f71814 100644 --- a/packages/protocol/contracts/common/interfaces/IEpochManager.sol +++ b/packages/protocol/contracts/common/interfaces/IEpochManager.sol @@ -13,6 +13,8 @@ interface IEpochManager { address[] calldata lessers, address[] calldata greaters ) external; + function setToProcessGroups() external; + function processGroup(address group, address lesser, address greater) external; function sendValidatorPayment(address) external; function getCurrentEpoch() external view returns (uint256, uint256, uint256, uint256); function getEpochByNumber( diff --git a/packages/protocol/test-sol/devchain/e2e/common/EpochManager.t.sol b/packages/protocol/test-sol/devchain/e2e/common/EpochManager.t.sol index ba0fa13dd7f..4ffc7ba037b 100644 --- a/packages/protocol/test-sol/devchain/e2e/common/EpochManager.t.sol +++ b/packages/protocol/test-sol/devchain/e2e/common/EpochManager.t.sol @@ -746,3 +746,143 @@ contract E2E_GasTest2_FinishNextEpochProcess is E2E_GasTest_Setup { console.log("elected count2: ", epochManager.getElectedAccounts().length); } } + +contract E2E_FinishNextEpochProcess_Split is E2E_GasTest_Setup { + using EnumerableSet for EnumerableSet.AddressSet; + + function setUp() public override { + super.setUp(); + + activateValidators(); + whenL2(vm); + + vm.prank(epochManagerEnabler); + epochManager.initializeSystem(1, 1, firstElected); + + validatorsArray = getValidators().getRegisteredValidators(); + groups = getValidators().getRegisteredValidatorGroups(); + + vm.startPrank(scoreManager.owner()); + scoreManager.setGroupScore(groups[0], groupScore[0]); + scoreManager.setGroupScore(groups[1], groupScore[1]); + scoreManager.setGroupScore(groups[2], groupScore[2]); + + scoreManager.setValidatorScore(validatorsArray[0], validatorScore[0]); + scoreManager.setValidatorScore(validatorsArray[1], validatorScore[1]); + scoreManager.setValidatorScore(validatorsArray[2], validatorScore[2]); + scoreManager.setValidatorScore(validatorsArray[3], validatorScore[3]); + scoreManager.setValidatorScore(validatorsArray[4], validatorScore[4]); + scoreManager.setValidatorScore(validatorsArray[5], validatorScore[5]); + + vm.stopPrank(); + + timeTravel(vm, epochDuration + 1); + epochManager.startNextEpochProcess(); + + address[] memory lessers; + address[] memory greaters; + address[] memory groupsEligible; + GroupWithVotes[] memory groupWithVotes; + uint256[] memory groupActiveBalances; + (lessers, greaters, groupWithVotes) = getLessersAndGreaters(groups); + + uint256 currentEpoch = epochManager.getCurrentEpochNumber(); + address[] memory currentlyElected = epochManager.getElectedAccounts(); + for (uint256 i = 0; i < currentlyElected.length; i++) { + originalyElected.add(currentlyElected[i]); + } + + // wait some time before finishing + timeTravel(vm, epochDuration / 2); + blockTravel(vm, 100); + + epochManager.setToProcessGroups(); + for (uint256 i = 0; i < groups.length; i++) { + epochManager.processGroup(groups[i], lessers[i], greaters[i]); + } + + assertEq(currentEpoch + 1, epochManager.getCurrentEpochNumber()); + + address[] memory newlyElected = epochManager.getElectedAccounts(); + + for (uint256 i = 0; i < currentlyElected.length; i++) { + assertEq(originalyElected.contains(currentlyElected[i]), true); + } + + timeTravel(vm, epochDuration + 1); + epochManager.startNextEpochProcess(); + + // wait some time before finishing + timeTravel(vm, epochDuration / 2); + blockTravel(vm, 100); + + (lessers, greaters, groupWithVotes) = getLessersAndGreaters(groups); + epochManager.setToProcessGroups(); + for (uint256 i = 0; i < groups.length; i++) { + epochManager.processGroup(groups[i], lessers[i], greaters[i]); + } + // epochManager.finishNextEpochProcess(groups, lessers, greaters); + assertGroupWithVotes(groupWithVotes); + + assertEq(currentEpoch + 2, epochManager.getCurrentEpochNumber()); + + address[] memory newlyElected2 = epochManager.getElectedAccounts(); + + for (uint256 i = 0; i < currentlyElected.length; i++) { + assertEq(originalyElected.contains(newlyElected2[i]), true); + } + uint256 validatorGroupCount = 60; + uint256 validatorPerGroupCount = 2; + + for (uint256 i = 0; i < validatorGroupCount; i++) { + (address newValidatorGroup, address newValidator) = registerNewValidatorGroupWithValidator( + i, + validatorPerGroupCount + ); + } + + timeTravel(vm, epochDuration + 1); + epochManager.startNextEpochProcess(); + + timeTravel(vm, epochDuration / 2); + blockTravel(vm, 100); + + (lessers, greaters, groupWithVotes) = getLessersAndGreaters(groups); + + epochManager.setToProcessGroups(); + for (uint256 i = 0; i < groups.length; i++) { + epochManager.processGroup(groups[i], lessers[i], greaters[i]); + } + + activateValidators(); + + timeTravel(vm, epochDuration + 1); + epochManager.startNextEpochProcess(); + + groups = getCurrentlyElectedGroups(); + + timeTravel(vm, epochDuration / 2); + blockTravel(vm, 100); + } + + /** + * @notice Test the gas used by finishNextEpochProcess + This test is trying to measure gas used by finishNextEpochProcess in a real life worst case. We have 126 validators and 123 groups. + There are two main loops in the function, one for calculating rewards and the other for updating the elected validators. + FinishNextEpochProcess is called twice, first time with going from 6 -> 110 validators which consumes approx. 6M gas and the second time with going from 110 -> 110 validators which consumes approx. 19M gas. + */ + function test_shouldFinishNextEpochProcessing_GasTest_Split() public { + address[] memory lessers; + address[] memory greaters; + GroupWithVotes[] memory groupWithVotes; + (lessers, greaters, groupWithVotes) = getLessersAndGreaters(groups); + epochManager.setToProcessGroups(); + + for (uint256 i = 0; i < groups.length; i++) { + uint256 gasLeftBefore1 = gasleft(); + epochManager.processGroup(groups[i], lessers[i], greaters[i]); + uint256 gasLeftAfter1 = gasleft(); + console.log("processGroup gas used: ", gasLeftBefore1 - gasLeftAfter1); + } + } +} diff --git a/packages/protocol/test-sol/unit/common/EpochManager.t.sol b/packages/protocol/test-sol/unit/common/EpochManager.t.sol index beb3fb1443a..ee81e5a7044 100644 --- a/packages/protocol/test-sol/unit/common/EpochManager.t.sol +++ b/packages/protocol/test-sol/unit/common/EpochManager.t.sol @@ -25,6 +25,7 @@ import { MockElection } from "@celo-contracts/governance/test/MockElection.sol"; import { MockAccounts } from "@celo-contracts-8/common/mocks/MockAccounts.sol"; import { ValidatorsMock } from "@test-sol/unit/governance/validators/mocks/ValidatorsMock.sol"; import { MockCeloUnreleasedTreasury } from "@celo-contracts-8/common/test/MockCeloUnreleasedTreasury.sol"; +import { console } from "forge-std/console.sol"; contract EpochManagerTest is Test, TestConstants, Utils08 { EpochManager_WithMocks epochManager; @@ -643,6 +644,224 @@ contract EpochManagerTest_finishNextEpochProcess is EpochManagerTest { } } +contract EpochManagerTest_setToProcessGroups is EpochManagerTest { + address signer1 = actor("signer1"); + address signer2 = actor("signer2"); + + address validator3 = actor("validator3"); + address validator4 = actor("validator4"); + + address group2 = actor("group2"); + + address[] elected; + + uint256 groupEpochRewards = 44e18; + + function setUp() public override { + super.setUp(); + + validators.setValidatorGroup(group); + validators.setValidator(validator1); + accounts.setValidatorSigner(validator1, signer1); + validators.setValidator(validator2); + accounts.setValidatorSigner(validator2, signer2); + + validators.setValidatorGroup(group2); + validators.setValidator(validator3); + validators.setValidator(validator4); + + address[] memory members = new address[](3); + members[0] = validator1; + members[1] = validator2; + validators.setMembers(group, members); + members[0] = validator3; + members[1] = validator4; + validators.setMembers(group2, members); + + vm.prank(epochManagerEnabler); + initializeEpochManagerSystem(); + + elected = epochManager.getElectedAccounts(); + + election.setGroupEpochRewardsBasedOnScore(group, groupEpochRewards); + } + + function test_Reverts_WhenNotStarted() public { + address[] memory groups = new address[](0); + + vm.expectRevert("Epoch process is not started"); + epochManager.setToProcessGroups(); + } + + function test_setsToProcessGroups() public { + ( + address[] memory groups, + address[] memory lessers, + address[] memory greaters + ) = getGroupsWithLessersAndGreaters(); + + epochManager.startNextEpochProcess(); + epochManager.setToProcessGroups(); + + assertEq(EpochManager(address(epochManager)).toProcessGroups(), groups.length); + } + + function test_setsGroupRewards() public { + ( + address[] memory groups, + address[] memory lessers, + address[] memory greaters + ) = getGroupsWithLessersAndGreaters(); + + epochManager.startNextEpochProcess(); + epochManager.setToProcessGroups(); + + for (uint256 i = 0; i < groups.length; i++) { + assertEq(EpochManager(address(epochManager)).processedGroups(group), groupEpochRewards); + } + } +} + +contract EpochManagerTest_processGroup is EpochManagerTest { + address signer1 = actor("signer1"); + address signer2 = actor("signer2"); + address signer3 = actor("signer3"); + address signer4 = actor("signer4"); + + address validator3 = actor("validator3"); + address validator4 = actor("validator4"); + + address group2 = actor("group2"); + + address[] elected; + + uint256 groupEpochRewards = 44e18; + + function setUp() public override { + super.setUp(); + + validators.setValidatorGroup(group); + validators.setValidator(validator1); + accounts.setValidatorSigner(validator1, signer1); + validators.setValidator(validator2); + accounts.setValidatorSigner(validator2, signer2); + + validators.setValidatorGroup(group2); + validators.setValidator(validator3); + validators.setValidator(validator4); + + address[] memory members = new address[](3); + members[0] = validator1; + members[1] = validator2; + validators.setMembers(group, members); + members[0] = validator3; + members[1] = validator4; + validators.setMembers(group2, members); + + vm.prank(epochManagerEnabler); + initializeEpochManagerSystem(); + + elected = epochManager.getElectedAccounts(); + + election.setGroupEpochRewardsBasedOnScore(group, groupEpochRewards); + } + + function test_Reverts_WhenNotStarted() public { + vm.expectRevert("Indivudual epoch process is not started"); + epochManager.processGroup(group, address(0), address(0)); + } + + function test_Reverts_WhenGroupNotInToProcessGroups() public { + epochManager.startNextEpochProcess(); + epochManager.setToProcessGroups(); + vm.expectRevert("group not from current elected set"); + epochManager.processGroup(group2, address(0), address(0)); + } + + function test_ProcessesGroup() public { + ( + address[] memory groups, + address[] memory lessers, + address[] memory greaters + ) = getGroupsWithLessersAndGreaters(); + + epochManager.startNextEpochProcess(); + epochManager.setToProcessGroups(); + epochManager.processGroup(group, address(0), address(0)); + + (uint256 status, , , , ) = epochManager.getEpochProcessingState(); + assertEq(status, 0); + } + + function test_TransfersToCommunityAndCarbonOffsetting() public { + ( + address[] memory groups, + address[] memory lessers, + address[] memory greaters + ) = getGroupsWithLessersAndGreaters(); + + epochManager.startNextEpochProcess(); + epochManager.setToProcessGroups(); + epochManager.processGroup(group, address(0), address(0)); + + assertEq(celoToken.balanceOf(communityRewardFund), epochRewards.totalRewardsCommunity()); + assertEq(celoToken.balanceOf(carbonOffsettingPartner), epochRewards.totalRewardsCarbonFund()); + } + + function test_TransfersToValidatorGroup() public { + ( + address[] memory groups, + address[] memory lessers, + address[] memory greaters + ) = getGroupsWithLessersAndGreaters(); + + epochManager.startNextEpochProcess(); + epochManager.setToProcessGroups(); + epochManager.processGroup(group, address(0), address(0)); + + assertEq(election.distributedEpochRewards(group), groupEpochRewards); + } + + function test_SetsNewlyElectedCorrectly() public { + ( + address[] memory groups, + address[] memory lessers, + address[] memory greaters + ) = getGroupsWithLessersAndGreaters(); + + epochManager.startNextEpochProcess(); + + address[] memory newElected = new address[](2); + newElected[0] = validator3; + newElected[1] = validator4; + election.setElectedValidators(newElected); + + address[] memory signers = new address[](2); + signers[0] = signer3; + signers[1] = signer4; + accounts.setValidatorSigner(validator3, signer3); + accounts.setValidatorSigner(validator4, signer4); + + epochManager.setToProcessGroups(); + + for (uint256 i = 0; i < groups.length; i++) { + epochManager.processGroup(groups[i], lessers[i], greaters[i]); + } + + address[] memory afterElected = epochManager.getElectedAccounts(); + + for (uint256 i = 0; i < newElected.length; i++) { + assertEq(newElected[i], afterElected[i]); + } + + address[] memory afterSigners = epochManager.getElectedSigners(); + assertEq(afterSigners.length, signers.length); + for (uint256 i = 0; i < signers.length; i++) { + assertEq(signers[i], afterSigners[i]); + } + } +} + contract EpochManagerTest_getEpochByNumber is EpochManagerTest { function test_shouldReturnTheEpochInfoOfSpecifiedEpoch() public { uint256 numberOfEpochsToTravel = 9;