feat: Stabilize ChunkEndorsementsInBlockHeader feature (#12089)

ChunkEndorsementsInBlockHeader enables the following:

- BlockHeaderV5 that includes chunk endorsements bitmap (bitmap version
of the endorsement signatures in the block body)
- Uses chunk endorsements bitmap to calculate the chunk endorsement
ratio.
- Sets chunk endorsement kickout threshold to 70 (previously 80).
- Improves the calculation of validators exempted from kickout.
- Uses the kickout threshold as cutoff threshold for endorsement ratio
(if below 70 use 0 and if above 70 use 100) in rewards calculation.

CHANGELOG.md

@@ -3,7 +3,7 @@
 ## [unreleased]
 
 ### Protocol Changes
-**No Changes**
+* Sets `chunk_validator_only_kickout_threshold` to 70. Uses this kickout threshold as a cutoff threshold for contribution of endorsement ratio in rewards calculation: if endorsement ratio is above 70%, the contribution of endorsement ratio in average uptime calculation is 100%, otherwise it is 0%. Endorsements received are now included in `BlockHeader` to improve kickout and reward calculation for chunk validators. 
 
 ### Non-protocol Changes
 * Added [documentation](./docs/misc/archival_data_recovery.md) and a [reference](./scripts/recover_missing_archival_data.sh) script to recover the data lost in archival nodes at the beginning of 2024.

chain/chain/src/tests/simple_chain.rs

@@ -34,7 +34,7 @@ fn build_chain() {
     if cfg!(feature = "nightly") {
         insta::assert_snapshot!(hash, @"Hc3bWEd7ikHf9BAe2SknvH2jAAakEtBRU1FBu6Udocm3");
     } else {
-        insta::assert_snapshot!(hash, @"GpzRk39JJJZkXiPaXpe7tv1KkyeWYBPYEakZFP6hznGh");
+        insta::assert_snapshot!(hash, @"dY6Z6HdATLWK3wwxkNtUs8T1GaEQqpxUCXm7TectWW7");
     }
 
     for i in 1..5 {
@@ -52,7 +52,7 @@ fn build_chain() {
     if cfg!(feature = "nightly") {
         insta::assert_snapshot!(hash, @"39R6bDFXkPfwdYs4crV3RyCde85ecycqP5DBwdtwyjcJ");
     } else {
-        insta::assert_snapshot!(hash, @"ANjdWVaMLmMKmXMLfkzVbWSWYri6Q5Mvc1qRogUTZQAK");
+        insta::assert_snapshot!(hash, @"6RnKeuiGmxkFxNYeEmAbK6NzwvKYuTcKCwqAmqJ6m3DG");
     }
 }
 

chain/epoch-manager/src/test_utils.rs

@@ -1,6 +1,7 @@
 use std::collections::{BTreeMap, HashMap};
 
 use near_primitives::types::EpochId;
+use near_primitives::types::ProtocolVersion;
 use near_store::Store;
 use num_rational::Ratio;
 
@@ -48,6 +49,7 @@ pub fn epoch_info(
     accounts: Vec<(AccountId, Balance)>,
     block_producers_settlement: Vec<ValidatorId>,
     chunk_producers_settlement: Vec<Vec<ValidatorId>>,
+    protocol_version: ProtocolVersion,
 ) -> EpochInfo {
     let num_seats = block_producers_settlement.len() as u64;
     epoch_info_with_num_seats(
@@ -60,6 +62,7 @@ pub fn epoch_info(
         Default::default(),
         0,
         num_seats,
+        protocol_version,
     )
 }
 
@@ -73,6 +76,7 @@ pub fn epoch_info_with_num_seats(
     validator_reward: HashMap<AccountId, Balance>,
     minted_amount: Balance,
     num_seats: NumSeats,
+    protocol_version: ProtocolVersion,
 ) -> EpochInfo {
     let seat_price =
         find_threshold(&accounts.iter().map(|(_, s)| *s).collect::<Vec<_>>(), num_seats).unwrap();
@@ -114,7 +118,7 @@ pub fn epoch_info_with_num_seats(
         validator_kickout.into_iter().collect(),
         minted_amount,
         seat_price,
-        PROTOCOL_VERSION,
+        protocol_version,
         TEST_SEED,
         validator_mandates,
     )

chain/epoch-manager/src/tests/mod.rs

@@ -72,6 +72,7 @@ fn test_stake_validator() {
         reward(vec![("near".parse().unwrap(), 0)]),
         0,
         4,
+        PROTOCOL_VERSION,
     );
     let compare_epoch_infos = |a: &EpochInfo, b: &EpochInfo| -> bool {
         a.validators_iter().eq(b.validators_iter())
@@ -115,6 +116,7 @@ fn test_stake_validator() {
         reward(vec![("test1".parse().unwrap(), 0), ("near".parse().unwrap(), 0)]),
         0,
         4,
+        PROTOCOL_VERSION,
     );
     // no validator change in the last epoch
     let epoch3 = epoch_manager.get_epoch_id(&h[3]).unwrap();
@@ -2540,7 +2542,13 @@ fn test_validator_kickout_determinism() {
         ("test4".parse().unwrap(), 500),
         ("test5".parse().unwrap(), 500),
     ];
-    let epoch_info = epoch_info(0, accounts, vec![0, 1, 2, 3], vec![vec![0, 1, 2], vec![0, 1, 3]]);
+    let epoch_info = epoch_info(
+        0,
+        accounts,
+        vec![0, 1, 2, 3],
+        vec![vec![0, 1, 2], vec![0, 1, 3]],
+        PROTOCOL_VERSION,
+    );
     let block_validator_tracker = HashMap::from([
         (0, ValidatorStats { produced: 100, expected: 100 }),
         (1, ValidatorStats { produced: 90, expected: 100 }),
@@ -2623,7 +2631,13 @@ fn test_chunk_validators_with_different_endorsement_ratio() {
         ("test2".parse().unwrap(), 500),
         ("test3".parse().unwrap(), 500),
     ];
-    let epoch_info = epoch_info(0, accounts, vec![0, 1, 2, 3], vec![vec![0, 1, 2], vec![0, 1, 3]]);
+    let epoch_info = epoch_info(
+        0,
+        accounts,
+        vec![0, 1, 2, 3],
+        vec![vec![0, 1, 2], vec![0, 1, 3]],
+        PROTOCOL_VERSION,
+    );
     let block_validator_tracker = HashMap::from([
         (0, ValidatorStats { produced: 100, expected: 100 }),
         (1, ValidatorStats { produced: 100, expected: 100 }),
@@ -2679,7 +2693,13 @@ fn test_chunk_validators_with_same_endorsement_ratio_and_different_stake() {
         ("test2".parse().unwrap(), 500),
         ("test3".parse().unwrap(), 499),
     ];
-    let epoch_info = epoch_info(0, accounts, vec![0, 1, 2, 3], vec![vec![0, 1, 2], vec![0, 1, 3]]);
+    let epoch_info = epoch_info(
+        0,
+        accounts,
+        vec![0, 1, 2, 3],
+        vec![vec![0, 1, 2], vec![0, 1, 3]],
+        PROTOCOL_VERSION,
+    );
     let block_validator_tracker = HashMap::from([
         (0, ValidatorStats { produced: 100, expected: 100 }),
         (1, ValidatorStats { produced: 100, expected: 100 }),
@@ -2735,7 +2755,13 @@ fn test_chunk_validators_with_same_endorsement_ratio_and_stake() {
         ("test2".parse().unwrap(), 500),
         ("test3".parse().unwrap(), 500),
     ];
-    let epoch_info = epoch_info(0, accounts, vec![0, 1, 2, 3], vec![vec![0, 1, 2], vec![0, 1, 3]]);
+    let epoch_info = epoch_info(
+        0,
+        accounts,
+        vec![0, 1, 2, 3],
+        vec![vec![0, 1, 2], vec![0, 1, 3]],
+        PROTOCOL_VERSION,
+    );
     let block_validator_tracker = HashMap::from([
         (0, ValidatorStats { produced: 100, expected: 100 }),
         (1, ValidatorStats { produced: 100, expected: 100 }),
@@ -2785,7 +2811,13 @@ fn test_validator_kickout_sanity() {
         ("test4".parse().unwrap(), 500),
         ("test5".parse().unwrap(), 500),
     ];
-    let epoch_info = epoch_info(0, accounts, vec![0, 1, 2, 3], vec![vec![0, 1, 2], vec![0, 1, 3]]);
+    let epoch_info = epoch_info(
+        0,
+        accounts,
+        vec![0, 1, 2, 3],
+        vec![vec![0, 1, 2], vec![0, 1, 3]],
+        PROTOCOL_VERSION,
+    );
     let block_validator_tracker = HashMap::from([
         (0, ValidatorStats { produced: 100, expected: 100 }),
         (1, ValidatorStats { produced: 90, expected: 100 }),
@@ -2916,7 +2948,13 @@ fn test_chunk_endorsement_stats() {
         ("test2".parse().unwrap(), 1000),
         ("test3".parse().unwrap(), 1000),
     ];
-    let epoch_info = epoch_info(0, accounts, vec![0, 1, 2, 3], vec![vec![0, 1, 2], vec![0, 1, 3]]);
+    let epoch_info = epoch_info(
+        0,
+        accounts,
+        vec![0, 1, 2, 3],
+        vec![vec![0, 1, 2], vec![0, 1, 3]],
+        PROTOCOL_VERSION,
+    );
     let (validator_stats, kickouts) = EpochManager::compute_validators_to_reward_and_kickout(
         &epoch_config,
         &epoch_info,
@@ -2994,7 +3032,8 @@ fn test_max_kickout_stake_ratio() {
         ("test3".parse().unwrap(), 1000),
         ("test4".parse().unwrap(), 1000),
     ];
-    let epoch_info = epoch_info(0, accounts, vec![0, 1, 2, 3], vec![vec![0, 1], vec![2, 4]]);
+    let epoch_info =
+        epoch_info(0, accounts, vec![0, 1, 2, 3], vec![vec![0, 1], vec![2, 4]], PROTOCOL_VERSION);
     let block_stats = HashMap::from([
         (0, ValidatorStats { produced: 50, expected: 100 }),
         // here both test1 and test2 produced the most number of blocks, we made that intentionally
@@ -3106,13 +3145,8 @@ fn test_max_kickout_stake_ratio() {
 /// Common test scenario for a couple of tests exercising chunk validator kickouts.
 fn test_chunk_validator_kickout(
     expected_kickouts: HashMap<AccountId, ValidatorKickoutReason>,
-    expect_new_algorithm: bool,
+    use_endorsement_cutoff_threshold: bool,
 ) {
-    if expect_new_algorithm
-        != ProtocolFeature::ChunkEndorsementsInBlockHeader.enabled(PROTOCOL_VERSION)
-    {
-        return;
-    }
     let mut epoch_config = epoch_config(5, 2, 4, 80, 80, 80).for_protocol_version(PROTOCOL_VERSION);
     let accounts = vec![
         ("test0".parse().unwrap(), 1000),
@@ -3122,7 +3156,17 @@ fn test_chunk_validator_kickout(
         ("test4".parse().unwrap(), 1000),
         ("test5".parse().unwrap(), 1000),
     ];
-    let epoch_info = epoch_info(0, accounts, vec![0, 1, 2, 3], vec![vec![0, 1], vec![0, 2]]);
+    let epoch_info = epoch_info(
+        0,
+        accounts,
+        vec![0, 1, 2, 3],
+        vec![vec![0, 1], vec![0, 2]],
+        if use_endorsement_cutoff_threshold {
+            PROTOCOL_VERSION
+        } else {
+            ProtocolFeature::ChunkEndorsementsInBlockHeader.protocol_version() - 1
+        },
+    );
     let block_stats = HashMap::from([
         (0, ValidatorStats { produced: 90, expected: 100 }),
         (1, ValidatorStats { produced: 90, expected: 100 }),
@@ -3189,16 +3233,19 @@ fn test_chunk_validator_kicked_out_for_low_endorsement() {
 #[test]
 /// Tests that a validator is not kicked out due to low endorsement only (as long as it produces most of its blocks and chunks).
 fn test_block_and_chunk_producer_not_kicked_out_for_low_endorsements() {
-    if !ProtocolFeature::ChunkEndorsementsInBlockHeader.enabled(PROTOCOL_VERSION) {
-        return;
-    }
     let mut epoch_config = epoch_config(5, 2, 4, 80, 80, 80).for_protocol_version(PROTOCOL_VERSION);
     let accounts = vec![
         ("test0".parse().unwrap(), 1000),
         ("test1".parse().unwrap(), 1000),
         ("test2".parse().unwrap(), 1000),
     ];
-    let epoch_info = epoch_info(0, accounts, vec![0, 1, 2], vec![vec![0, 1, 2], vec![0, 1, 2]]);
+    let epoch_info = epoch_info(
+        0,
+        accounts,
+        vec![0, 1, 2],
+        vec![vec![0, 1, 2], vec![0, 1, 2]],
+        PROTOCOL_VERSION,
+    );
     let block_stats = HashMap::from([
         (0, ValidatorStats { produced: 90, expected: 100 }),
         (1, ValidatorStats { produced: 90, expected: 100 }),

core/primitives-core/src/version.rs

@@ -225,7 +225,8 @@ impl ProtocolFeature {
             ProtocolFeature::BLS12381 | ProtocolFeature::EthImplicitAccounts => 70,
             ProtocolFeature::FixMinStakeRatio => 71,
             ProtocolFeature::IncreaseStorageProofSizeSoftLimit
-            | ProtocolFeature::ChunkEndorsementV2 => 72,
+            | ProtocolFeature::ChunkEndorsementV2
+            | ProtocolFeature::ChunkEndorsementsInBlockHeader => 72,
 
             // This protocol version is reserved for use in resharding tests. An extra resharding
             // is simulated on top of the latest shard layout in production. Note that later
@@ -244,7 +245,6 @@ impl ProtocolFeature {
             // TODO(#11201): When stabilizing this feature in mainnet, also remove the temporary code
             // that always enables this for mocknet (see config_mocknet function).
             ProtocolFeature::ShuffleShardAssignments => 143,
-            ProtocolFeature::ChunkEndorsementsInBlockHeader => 145,
         }
     }
 

core/primitives/res/epoch_configs/mainnet/100.json

@@ -21,7 +21,7 @@
   ],
   "block_producer_kickout_threshold": 80,
   "chunk_producer_kickout_threshold": 80,
-  "chunk_validator_only_kickout_threshold": 80,
+  "chunk_validator_only_kickout_threshold": 70,
   "target_validator_mandates_per_shard": 68,
   "validator_max_kickout_stake_perc": 30,
   "online_min_threshold": [

core/primitives/res/epoch_configs/mainnet/101.json

@@ -19,7 +19,7 @@
   ],
   "block_producer_kickout_threshold": 80,
   "chunk_producer_kickout_threshold": 80,
-  "chunk_validator_only_kickout_threshold": 80,
+  "chunk_validator_only_kickout_threshold": 70,
   "target_validator_mandates_per_shard": 68,
   "validator_max_kickout_stake_perc": 30,
   "online_min_threshold": [

core/primitives/res/epoch_configs/mainnet/143.json

@@ -19,7 +19,7 @@
   ],
   "block_producer_kickout_threshold": 80,
   "chunk_producer_kickout_threshold": 80,
-  "chunk_validator_only_kickout_threshold": 80,
+  "chunk_validator_only_kickout_threshold": 70,
   "target_validator_mandates_per_shard": 68,
   "validator_max_kickout_stake_perc": 30,
   "online_min_threshold": [

core/primitives/res/epoch_configs/mainnet/145.json → core/primitives/res/epoch_configs/mainnet/72.json

@@ -84,6 +84,6 @@
       62500
     ],
     "chunk_producer_assignment_changes_limit": 5,
-    "shuffle_shard_assignment_for_chunk_producers": true
+    "shuffle_shard_assignment_for_chunk_producers": false
   }
 }

core/primitives/res/epoch_configs/mocknet/100.json

@@ -21,7 +21,7 @@
   ],
   "block_producer_kickout_threshold": 80,
   "chunk_producer_kickout_threshold": 80,
-  "chunk_validator_only_kickout_threshold": 80,
+  "chunk_validator_only_kickout_threshold": 70,
   "target_validator_mandates_per_shard": 68,
   "validator_max_kickout_stake_perc": 30,
   "online_min_threshold": [

core/primitives/res/epoch_configs/mocknet/101.json

@@ -19,7 +19,7 @@
   ],
   "block_producer_kickout_threshold": 80,
   "chunk_producer_kickout_threshold": 80,
-  "chunk_validator_only_kickout_threshold": 80,
+  "chunk_validator_only_kickout_threshold": 70,
   "target_validator_mandates_per_shard": 68,
   "validator_max_kickout_stake_perc": 30,
   "online_min_threshold": [

core/primitives/res/epoch_configs/testnet/100.json

@@ -21,7 +21,7 @@
   ],
   "block_producer_kickout_threshold": 80,
   "chunk_producer_kickout_threshold": 80,
-  "chunk_validator_only_kickout_threshold": 80,
+  "chunk_validator_only_kickout_threshold": 70,
   "target_validator_mandates_per_shard": 68,
   "validator_max_kickout_stake_perc": 30,
   "online_min_threshold": [

core/primitives/res/epoch_configs/testnet/101.json

@@ -19,7 +19,7 @@
   ],
   "block_producer_kickout_threshold": 80,
   "chunk_producer_kickout_threshold": 80,
-  "chunk_validator_only_kickout_threshold": 80,
+  "chunk_validator_only_kickout_threshold": 70,
   "target_validator_mandates_per_shard": 68,
   "validator_max_kickout_stake_perc": 30,
   "online_min_threshold": [

core/primitives/res/epoch_configs/testnet/143.json

@@ -19,7 +19,7 @@
   ],
   "block_producer_kickout_threshold": 80,
   "chunk_producer_kickout_threshold": 80,
-  "chunk_validator_only_kickout_threshold": 80,
+  "chunk_validator_only_kickout_threshold": 70,
   "target_validator_mandates_per_shard": 68,
   "validator_max_kickout_stake_perc": 30,
   "online_min_threshold": [

core/primitives/res/epoch_configs/testnet/145.json → core/primitives/res/epoch_configs/testnet/72.json

@@ -84,6 +84,6 @@
       62500
     ],
     "chunk_producer_assignment_changes_limit": 5,
-    "shuffle_shard_assignment_for_chunk_producers": true
+    "shuffle_shard_assignment_for_chunk_producers": false
   }
 }

core/primitives/src/block_header.rs

@@ -604,10 +604,10 @@ impl BlockHeader {
             block_merkle_root,
         };
 
-        if chunk_endorsements.is_some() {
-            debug_assert!(ProtocolFeature::ChunkEndorsementsInBlockHeader
-                .enabled(this_epoch_protocol_version));
-            let chunk_endorsements = chunk_endorsements.unwrap();
+        if ProtocolFeature::ChunkEndorsementsInBlockHeader.enabled(this_epoch_protocol_version) {
+            let chunk_endorsements = chunk_endorsements.unwrap_or_else(|| {
+                panic!("BlockHeaderV5 is enabled but chunk endorsement bitmap is not provided")
+            });
             let inner_rest = BlockHeaderInnerRestV5 {
                 block_body_hash,
                 prev_chunk_outgoing_receipts_root,

core/primitives/src/epoch_manager.rs

@@ -397,10 +397,10 @@ static CONFIGS: &[(&str, ProtocolVersion, &str)] = &[
     include_config!("mainnet", 69, "69.json"),
     include_config!("mainnet", 70, "70.json"),
     include_config!("mainnet", 71, "71.json"),
+    include_config!("mainnet", 72, "72.json"),
     include_config!("mainnet", 100, "100.json"),
     include_config!("mainnet", 101, "101.json"),
     include_config!("mainnet", 143, "143.json"),
-    include_config!("mainnet", 145, "145.json"),
     // Epoch configs for testnet (genesis protool version is 29).
     include_config!("testnet", 29, "29.json"),
     include_config!("testnet", 48, "48.json"),
@@ -410,10 +410,10 @@ static CONFIGS: &[(&str, ProtocolVersion, &str)] = &[
     include_config!("testnet", 69, "69.json"),
     include_config!("testnet", 70, "70.json"),
     include_config!("testnet", 71, "71.json"),
+    include_config!("testnet", 72, "72.json"),
     include_config!("testnet", 100, "100.json"),
     include_config!("testnet", 101, "101.json"),
     include_config!("testnet", 143, "143.json"),
-    include_config!("testnet", 145, "145.json"),
     // Epoch configs for mocknet (forknet) (genesis protool version is 29).
     // TODO(#11900): Check the forknet config and uncomment this.
     // include_config!("mocknet", 29, "29.json"),
@@ -423,9 +423,9 @@ static CONFIGS: &[(&str, ProtocolVersion, &str)] = &[
     // include_config!("mocknet", 69, "69.json"),
     // include_config!("mocknet", 70, "70.json"),
     // include_config!("mocknet", 71, "71.json"),
+    // include_config!("mocknet", 72, "72.json"),
     // include_config!("mocknet", 100, "100.json"),
     // include_config!("mocknet", 101, "101.json"),
-    // include_config!("mocknet", 145, "145.json"),
 ];
 
 /// Store for `[EpochConfig]` per protocol version.`