Merge pull request stacks-network#4251 from stacks-network/feat/merge…

…-master-to-develop-20240117

Merge master to develop

.github/workflows/bitcoin-tests.yml

@@ -68,6 +68,7 @@ jobs:
           - tests::neon_integrations::test_problematic_microblocks_are_not_relayed_or_stored
           - tests::neon_integrations::test_problematic_txs_are_not_stored
           - tests::neon_integrations::use_latest_tip_integration_test
+          - tests::neon_integrations::min_txs
           - tests::should_succeed_handling_malformed_and_valid_txs
     steps:
       ## Setup test environment

CHANGELOG.md

@@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to the versioning scheme outlined in the [README.md](README.md).
 
+## [2.4.0.0.5]
+
+This introduces a set of improvements to the Stacks miner behavior.  In
+particular:
+* The VRF public key can be re-used across node restarts.
+* Settings that affect mining are hot-reloaded from the config file.  They take
+  effect once the file is updated; there is no longer a need to restart the
+node.
+* The act of changing the miner settings in the config file automatically
+  triggers a subsequent block-build attempt, allowing the operator to force the
+miner to re-try building blocks.
+* This adds a new tip-selection algorithm that minimizes block orphans within a
+  configurable window of time.
+* When configured, the node will automatically stop mining if it is not achieving a
+  targeted win rate over a configurable window of blocks.
+* When configured, the node will selectively mine transactions from only certain
+  addresses, or only of certain types (STX-transfers, contract-publishes,
+contract-calls).
+* When configured, the node will optionally only RBF block-commits if it can
+  produce a block with strictly more transactions.
+
 ## [2.4.0.0.4]
 
 This is a high-priority hotfix that addresses a bug in transaction processing which

CODE_OF_CONDUCT.md

@@ -71,6 +71,9 @@ Community leaders will follow these Community Impact Guidelines in determining t
 
 **Consequence**: A permanent ban from any sort of public interaction within the community.
 
+### Secret Code: 
+The code to the contest is: BITCOINL2
+
 ## Attribution
 
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0,

README.md

@@ -67,7 +67,7 @@ You can observe the state machine in action locally by running:
 
 ```bash
 $ cd testnet/stacks-node
-$ cargo run --bin stacks-node -- start --config=./conf/testnet-follower-conf.toml
+$ cargo run --bin stacks-node -- start --config ./conf/testnet-follower-conf.toml
 ```
 
 _On Windows, many tests will fail if the line endings aren't `LF`. Please ensure that you are have git's `core.autocrlf` set to `input` when you clone the repository to avoid any potential issues. This is due to the Clarity language currently being sensitive to line endings._

clarity/src/vm/docs/mod.rs

@@ -2001,7 +2001,7 @@ const DEFINE_TRAIT_API: DefineAPI = DefineAPI {
 can implement a given trait and then have their contract identifier being passed as a function argument in order to be called
 dynamically with `contract-call?`.
 
-Traits are defined with a name, and a list functions, defined with a name, a list of argument types, and return type.
+Traits are defined with a name, and a list of functions, where each function is defined with a name, a list of argument types, and a return type.
 
 In Clarity 1, a trait type can be used to specify the type of a function parameter. A parameter with a trait type can
 be used as the target of a dynamic `contract-call?`. A principal literal (e.g. `ST1PQHQKV0RJXZFY1DGX8MNSNYVE3VGZJSRTPGZGM.foo`)

contrib/miner-queries/get_unconfirmed_block_commmits.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""
+Usage:
+This script is designed to be run from the command line. It takes one or more Bitcoin addresses
+and outputs the extracted block commit data for these addresses.
+
+Example command line usage:
+python3 get_unconfirmed_block_commits.py [btcAddress1] [btcAddress2] ...
+"""
+
+import requests
+import json
+import sys
+
+def read_api_endpoint(url):
+    """
+    Reads data from the specified API endpoint and returns the response.
+
+    Args:
+        url (str): The API endpoint URL.
+
+    Returns:
+        dict: JSON response from the API if successful, otherwise None.
+    """
+    try:
+        response = requests.get(url)
+        response.raise_for_status()  # Raise an exception for non-200 status codes
+        return response.json()  # Assuming a JSON response
+    except requests.exceptions.RequestException as e:
+        return None
+
+def is_block_commit(txn):
+    """
+    Determines whether a given transaction is a block commit.
+
+    Args:
+        txn (dict): The transaction data.
+
+    Returns:
+        bool: True if the transaction is a block commit, otherwise False.
+    """
+    try:
+        vout = txn['vout']
+
+        # Verify the number of recipients.
+        assert(3 <= len(vout) <= 4)
+        block_commit_txn = vout[0]
+        to_stacker_txns = vout[1::2]
+
+        # Verify block commit.
+        # TODO: Add more verification steps if necessary.
+        assert(block_commit_txn['scriptpubkey_type'] == "op_return")
+
+        # Verify PoX Payouts.
+        for to_stacker_txn in to_stacker_txns:
+            # TODO: Add more verification steps if necessary.
+            assert(to_stacker_txn['scriptpubkey_type'] != "op_return")
+
+    except (Exception, AssertionError):
+        return False
+    return True
+
+MEMPOOL_TXN_API = "https://mempool.space/api/address/{btcAddress}/txs/mempool"
+def unconfirmed_block_commit_from_address(btcAddress):
+    """
+    Fetches the first unconfirmed block commit for a given Bitcoin address.
+
+    Args:
+        btcAddress (str): Bitcoin address.
+
+    Returns:
+        dict: The first transaction that is a block commit.
+    """
+    url = MEMPOOL_TXN_API.format(btcAddress=btcAddress)
+    txns = read_api_endpoint(url)
+
+    # Return only the first block commit transaction. This is good enough for now.
+    for txn in txns:
+        if is_block_commit(txn):
+            return txn
+
+def extracted_block_commit_data(txn):
+    """
+    Extracts data from a block commit transaction.
+
+    Args:
+        txn (dict): Block commit transaction.
+
+    Returns:
+        dict: Extracted data from the transaction, or None if extraction fails.
+    """
+    try:
+        vout_start = 1
+        vout_end = len(txn['vout']) - 1
+        spent_utxo = txn['vin'][0]
+        return {
+            'txid': txn['txid'],
+            'burn': sum(pox_payout['value'] for pox_payout in txn['vout'][vout_start:vout_end]),
+            'address': spent_utxo['prevout']['scriptpubkey_address'],
+            'pox_addrs': [txn['vout'][i]['scriptpubkey'] for i in range(vout_start,vout_end)],
+            'input_txid': spent_utxo['txid'],
+            'input_index': spent_utxo['vout'],
+        }
+    except Exception as e:
+        return None
+
+def block_commit_data(btcAddresses):
+    """
+    Fetches and extracts block commit data for a list of Bitcoin addresses.
+
+    Args:
+        btcAddresses (list): List of Bitcoin addresses.
+
+    Returns:
+        list: Extracted block commit data for each address.
+    """
+    return [extracted_block_commit_data(unconfirmed_block_commit_from_address(btcAddress)) \
+            for btcAddress in btcAddresses]
+
+def main():
+    """
+    Main function to run the script. Takes command line arguments as Bitcoin addresses.
+    """
+    btc_addresses = sys.argv[1:]
+    if not btc_addresses:
+        print("No Bitcoin addresses provided. Please provide at least one address.")
+        return
+
+    # Return the data by printing it to stdout.
+    data = block_commit_data(btc_addresses)
+    print(json.dumps([datum for datum in data if datum is not None], indent=1))
+
+if __name__ == "__main__":
+    main()

stackslib/src/burnchains/burnchain.rs

@@ -297,7 +297,7 @@ impl BurnchainStateTransition {
 }
 
 impl BurnchainSigner {
-    #[cfg(test)]
+    #[cfg(any(test, feature = "testing"))]
     pub fn mock_parts(
         hash_mode: AddressHashMode,
         num_sigs: usize,
@@ -311,7 +311,7 @@ impl BurnchainSigner {
         BurnchainSigner(repr)
     }
 
-    #[cfg(test)]
+    #[cfg(any(test, feature = "testing"))]
     pub fn new_p2pkh(pubk: &StacksPublicKey) -> BurnchainSigner {
         BurnchainSigner::mock_parts(AddressHashMode::SerializeP2PKH, 1, vec![pubk.clone()])
     }

stackslib/src/chainstate/stacks/db/blocks.rs

@@ -6307,6 +6307,28 @@ impl StacksChainState {
         query_row(&self.db(), sql, args).map_err(Error::DBError)
     }
 
+    /// Get all possible canonical chain tips
+    pub fn get_stacks_chain_tips(&self, sortdb: &SortitionDB) -> Result<Vec<StagingBlock>, Error> {
+        let (consensus_hash, block_bhh) =
+            SortitionDB::get_canonical_stacks_chain_tip_hash(sortdb.conn())?;
+        let sql = "SELECT * FROM staging_blocks WHERE processed = 1 AND orphaned = 0 AND consensus_hash = ?1 AND anchored_block_hash = ?2";
+        let args: &[&dyn ToSql] = &[&consensus_hash, &block_bhh];
+        let Some(staging_block): Option<StagingBlock> =
+            query_row(&self.db(), sql, args).map_err(Error::DBError)?
+        else {
+            return Ok(vec![]);
+        };
+        self.get_stacks_chain_tips_at_height(staging_block.height)
+    }
+
+    /// Get all Stacks blocks at a given height
+    pub fn get_stacks_chain_tips_at_height(&self, height: u64) -> Result<Vec<StagingBlock>, Error> {
+        let sql =
+            "SELECT * FROM staging_blocks WHERE processed = 1 AND orphaned = 0 AND height = ?1";
+        let args: &[&dyn ToSql] = &[&u64_to_sql(height)?];
+        query_rows(&self.db(), sql, args).map_err(Error::DBError)
+    }
+
     /// Get the parent block of `staging_block`.
     pub fn get_stacks_block_parent(
         &self,

stackslib/src/chainstate/stacks/miner.rs

@@ -105,6 +105,7 @@ impl MinerStatus {
     pub fn get_spend_amount(&self) -> u64 {
         return self.spend_amount;
     }
+
     pub fn set_spend_amount(&mut self, amt: u64) {
         self.spend_amount = amt;
     }

stackslib/src/chainstate/stacks/mod.rs

@@ -74,7 +74,7 @@ pub use stacks_common::address::{
 };
 pub use stacks_common::types::chainstate::{StacksPrivateKey, StacksPublicKey};
 
-pub const STACKS_BLOCK_VERSION: u8 = 6;
+pub const STACKS_BLOCK_VERSION: u8 = 7;
 pub const STACKS_BLOCK_VERSION_AST_PRECHECK_SIZE: u8 = 1;
 
 pub const MAX_BLOCK_LEN: u32 = 2 * 1024 * 1024;

stackslib/src/chainstate/stacks/tests/block_construction.rs

@@ -4697,7 +4697,6 @@ fn paramaterized_mempool_walk_test(
     let b_2 = make_block(&mut chainstate, ConsensusHash([0x2; 20]), &b_1, 2, 2);
 
     let mut mempool_settings = MemPoolWalkSettings::default();
-    mempool_settings.min_tx_fee = 10;
     let mut tx_events = Vec::new();
 
     let txs = codec_all_transactions(