From cb8f1578e707f1eb274df650bd4750a86ab9b290 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 13 Oct 2023 12:25:59 +1000 Subject: [PATCH 01/13] Enable getblocktemplate-rpcs in production builds --- zebrad/Cargo.toml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/zebrad/Cargo.toml b/zebrad/Cargo.toml index ff38f3f5b23..a662163a653 100644 --- a/zebrad/Cargo.toml +++ b/zebrad/Cargo.toml @@ -52,14 +52,14 @@ features = [ [features] # In release builds, don't compile debug logging code, to improve performance. -default = ["release_max_level_info", "progress-bar"] +default = ["release_max_level_info", "progress-bar", "getblocktemplate-rpcs"] # Default features for official ZF binary release builds default-release-binaries = ["default", "sentry"] # Production features that activate extra dependencies, or extra features in dependencies -# Experimental mining RPC support +# Mining RPC support getblocktemplate-rpcs = [ "zebra-rpc/getblocktemplate-rpcs", "zebra-consensus/getblocktemplate-rpcs", @@ -68,12 +68,13 @@ getblocktemplate-rpcs = [ "zebra-chain/getblocktemplate-rpcs", ] +# Experimental elasticsearch indexing elasticsearch = [ "zebra-state/elasticsearch", ] +# Tracing and monitoring sentry = ["dep:sentry"] -flamegraph = ["tracing-flame", "inferno"] journald = ["tracing-journald"] filter-reload = ["hyper"] @@ -105,6 +106,8 @@ max_level_info = ["tracing/max_level_info", "log/max_level_info"] max_level_debug = ["tracing/max_level_debug", "log/max_level_debug"] # Testing features that activate extra dependencies +flamegraph = ["tracing-flame", "inferno"] + proptest-impl = [ "proptest", "proptest-derive", From 6b53ad7ae5af762cd389755f9dcc9a5b03cf8221 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 13 Oct 2023 13:00:28 +1000 Subject: [PATCH 02/13] Update mining instructions --- README.md | 6 +++++- book/src/user/mining-docker.md | 15 ++++++++------- book/src/user/mining-testnet-s-nomp.md | 4 ++-- book/src/user/mining.md | 17 ++++++----------- zebrad/src/lib.rs | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 52cec8703ff..675c64e267d 100644 --- a/README.md +++ b/README.md @@ -128,11 +128,15 @@ In future releases, the `progress_bar = "summary"` config will show a few key me and the "detailed" config will show all available metrics. Please let us know which metrics are important to you! +##### Configuring Mining + +Zebra can be configured for mining by passing a `MINER_ADDRESS` and port mapping to Docker. +See the [mining support docs](https://zebra.zfnd.org/user/mining-docker.html) for more details. + ##### Custom Build Features You can also build Zebra with additional [Cargo features](https://doc.rust-lang.org/cargo/reference/features.html#command-line-feature-options): -- `getblocktemplate-rpcs` for [mining support](https://zebra.zfnd.org/user/mining.html) - `prometheus` for [Prometheus metrics](https://zebra.zfnd.org/user/metrics.html) - `sentry` for [Sentry monitoring](https://zebra.zfnd.org/user/tracing.html#sentry-production-monitoring) - `elasticsearch` for [experimental Elasticsearch support](https://zebra.zfnd.org/user/elasticsearch.html) diff --git a/book/src/user/mining-docker.md b/book/src/user/mining-docker.md index e5d974317b8..69723ded880 100644 --- a/book/src/user/mining-docker.md +++ b/book/src/user/mining-docker.md @@ -1,15 +1,16 @@ # Mining with Zebra in Docker -Some of our published [Docker images](https://hub.docker.com/r/zfnd/zebra/tags) -have the `.experimental` suffix in their name. We compile these images with the -`getblocktemplate-rpcs` feature, and you can use them for your mining -operations. For example, executing +Zebra's [Docker images](https://hub.docker.com/r/zfnd/zebra/tags) can be used for your mining +operations. If you don't have Docker, see the +[manual configuration instructions](https://zebra.zfnd.org/user/mining.html). + +Using docker, you can start mining by running: ```bash -docker run -e MINER_ADDRESS="t1XhG6pT9xRqRQn3BHP7heUou1RuYrbcrCc" -p 8232:8232 zfnd/zebra:v1.1.0.experimental +docker run -e MINER_ADDRESS="t1XhG6pT9xRqRQn3BHP7heUou1RuYrbcrCc" -p 8232:8232 zfnd/zebra:latest ``` -will start a container on Mainnet and bind port 8232 on your Docker host. If you +This command starts a container on Mainnet and binds port 8232 on your Docker host. If you want to start generating blocks, you need to let Zebra sync first. Note that you must pass the address for your mining rewards via the @@ -35,7 +36,7 @@ variable to `Testnet` and use a Testnet address for the rewards. For example, running ```bash -docker run -e NETWORK="Testnet" -e MINER_ADDRESS="t27eWDgjFYJGVXmzrXeVjnb5J3uXDM9xH9v" -p 18232:18232 zfnd/zebra:v1.1.0.experimental +docker run -e NETWORK="Testnet" -e MINER_ADDRESS="t27eWDgjFYJGVXmzrXeVjnb5J3uXDM9xH9v" -p 18232:18232 zfnd/zebra:latest ``` will start a container on Testnet and bind port 18232 on your Docker host, which diff --git a/book/src/user/mining-testnet-s-nomp.md b/book/src/user/mining-testnet-s-nomp.md index 711afb24b6e..70fbc7a7647 100644 --- a/book/src/user/mining-testnet-s-nomp.md +++ b/book/src/user/mining-testnet-s-nomp.md @@ -82,7 +82,7 @@ These fixes disable mining pool operator payments and miner payments: they just ## Install `s-nomp`
General instructions with Debian/Ubuntu examples - + #### Install dependencies 1. Install `redis` and run it on the default port: @@ -136,7 +136,7 @@ These fixes disable mining pool operator payments and miner payments: they just
Arch-specific instructions - + #### Install dependencies 1. Install [`redis`](https://redis.io/docs/getting-started/) and run it on the default port: diff --git a/book/src/user/mining.md b/book/src/user/mining.md index cb7a2be2ccf..ef858bf4a58 100644 --- a/book/src/user/mining.md +++ b/book/src/user/mining.md @@ -1,10 +1,10 @@ # Mining Zcash with zebra -Zebra's RPC methods support miners and mining pools. +Zebra's RPC methods support miners and mining pools. Contents: -- [Download and build Zebra](#download-and-build-zebra) +- [Download Zebra](#download-and-build-zebra) - [Configure zebra for mining](#configure-zebra-for-mining) - [Miner address](#miner-address) - [RPC section](#rpc-section) @@ -12,18 +12,13 @@ Contents: - [Testing the setup](#testing-the-setup) - [Run a mining pool](#run-a-mining-pool) -## Download and build Zebra for mining +## Download Zebra [#download-and-build-zebra]: #download-and-build-zebra -Before installing please make sure you have the [Zebra dependencies](https://github.com/ZcashFoundation/zebra#build-instructions) in your OS. +The easiest way to run Zebra for mining is [our docker images](https://zebra.zfnd.org/user/mining-docker.html). -```console -cargo install --locked --features getblocktemplate-rpcs --git https://github.com/ZcashFoundation/zebra zebrad -``` - -Zebra binary will be at ` ~/.cargo/bin/zebrad`. - -It is important to install with the `getblocktemplate-rpcs` feature so the final binary will contain mining support. +If you have [installed Zebra another way](https://zebra.zfnd.org/user/install.html), follow the +instructions below to start mining: ## Configure zebra for mining [#configure-zebra-for-mining]: #configure-zebra-for-mining diff --git a/zebrad/src/lib.rs b/zebrad/src/lib.rs index 9ebe9b3ffff..b36ca2e182a 100644 --- a/zebrad/src/lib.rs +++ b/zebrad/src/lib.rs @@ -57,7 +57,7 @@ //! //! ### JSON-RPC //! -//! * `getblocktemplate-rpcs`: Experimental mining pool RPC support (currently incomplete) +//! * `getblocktemplate-rpcs`: Mining pool RPC support, enabled by default in production builds. //! //! ### Metrics //! From 4fe0b205531205c53c416e99293d3cd3528b9b6d Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 13 Oct 2023 13:42:35 +1000 Subject: [PATCH 03/13] Consistently use ZF funding stream addresses in examples --- book/src/user/mining-docker.md | 4 ++-- book/src/user/mining.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/book/src/user/mining-docker.md b/book/src/user/mining-docker.md index 69723ded880..002848c0ca3 100644 --- a/book/src/user/mining-docker.md +++ b/book/src/user/mining-docker.md @@ -7,7 +7,7 @@ operations. If you don't have Docker, see the Using docker, you can start mining by running: ```bash -docker run -e MINER_ADDRESS="t1XhG6pT9xRqRQn3BHP7heUou1RuYrbcrCc" -p 8232:8232 zfnd/zebra:latest +docker run -e MINER_ADDRESS="t3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1" -p 8232:8232 zfnd/zebra:latest ``` This command starts a container on Mainnet and binds port 8232 on your Docker host. If you @@ -15,7 +15,7 @@ want to start generating blocks, you need to let Zebra sync first. Note that you must pass the address for your mining rewards via the `MINER_ADDRESS` environment variable when you are starting the container, as we -did in the example above. The address we used starts with the prefix `t1`, +did with the ZF funding stream address above. The address we used starts with the prefix `t1`, meaning it is a Mainnet P2PKH address. Please remember to set your own address for the rewards. diff --git a/book/src/user/mining.md b/book/src/user/mining.md index ef858bf4a58..14d14cf5f0a 100644 --- a/book/src/user/mining.md +++ b/book/src/user/mining.md @@ -41,10 +41,10 @@ Node miner address is required. At the moment zebra only allows `p2pkh` or `p2sh ``` [mining] -miner_address = 't3Vz22vK5z2LcKEdg16Yv4FFneEL1zg9ojd' +miner_address = 't3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1' ``` -The above address is just the first address of the [Founders' Reward](https://zips.z.cash/protocol/protocol.pdf#foundersreward) section of the Zcash protocol. It's a Mainnet address and it is used here purely as an example. +The above address is the ZF Mainnet funding stream address. It is used here purely as an example. ### RPC section [#rpc-section]: #rpc-section From b8ec0014191ef5e4405ec53a4df98cce32159194 Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 13 Oct 2023 13:43:06 +1000 Subject: [PATCH 04/13] Simplify skip_getblocktemplate() --- zebra-rpc/src/config/mining.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/zebra-rpc/src/config/mining.rs b/zebra-rpc/src/config/mining.rs index f8224859c3f..1a27baa3646 100644 --- a/zebra-rpc/src/config/mining.rs +++ b/zebra-rpc/src/config/mining.rs @@ -46,9 +46,6 @@ impl Config { /// This is used to ignore the mining section of the configuration if the feature is not /// enabled, allowing us to log a warning when the config found is different from the default. pub fn skip_getblocktemplate(&self) -> bool { - #[cfg(feature = "getblocktemplate-rpcs")] - return false; - #[cfg(not(feature = "getblocktemplate-rpcs"))] - return true; + !cfg!(feature = "getblocktemplate-rpcs") } } From 99ad388dc86476227cc01f75a8d2b0d8492048ce Mon Sep 17 00:00:00 2001 From: teor Date: Fri, 13 Oct 2023 14:03:36 +1000 Subject: [PATCH 05/13] Set the test miner address based on the network --- zebrad/tests/acceptance.rs | 49 ++++++++++++++++---------------- zebrad/tests/common/config.rs | 31 ++++++++++++++++---- zebrad/tests/common/launch.rs | 25 +++++++++------- zebrad/tests/common/sync.rs | 10 +++---- zebrad/tests/common/test_type.rs | 12 +++----- 5 files changed, 72 insertions(+), 55 deletions(-) diff --git a/zebrad/tests/acceptance.rs b/zebrad/tests/acceptance.rs index c4fbdc707e2..0c40df31a50 100644 --- a/zebrad/tests/acceptance.rs +++ b/zebrad/tests/acceptance.rs @@ -207,7 +207,7 @@ fn generate_no_args() -> Result<()> { let _init_guard = zebra_test::init(); let child = testdir()? - .with_config(&mut default_test_config()?)? + .with_config(&mut default_test_config(Mainnet)?)? .spawn_child(args!["generate"])?; let output = child.wait_with_output()?; @@ -269,7 +269,7 @@ fn generate_args() -> Result<()> { fn help_no_args() -> Result<()> { let _init_guard = zebra_test::init(); - let testdir = testdir()?.with_config(&mut default_test_config()?)?; + let testdir = testdir()?.with_config(&mut default_test_config(Mainnet)?)?; let child = testdir.spawn_child(args!["help"])?; let output = child.wait_with_output()?; @@ -314,7 +314,7 @@ fn start_no_args() -> Result<()> { let _init_guard = zebra_test::init(); // start caches state, so run one of the start tests with persistent state - let testdir = testdir()?.with_config(&mut persistent_test_config()?)?; + let testdir = testdir()?.with_config(&mut persistent_test_config(Mainnet)?)?; let mut child = testdir.spawn_child(args!["-v", "start"])?; @@ -341,7 +341,7 @@ fn start_no_args() -> Result<()> { fn start_args() -> Result<()> { let _init_guard = zebra_test::init(); - let testdir = testdir()?.with_config(&mut default_test_config()?)?; + let testdir = testdir()?.with_config(&mut default_test_config(Mainnet)?)?; let testdir = &testdir; let mut child = testdir.spawn_child(args!["start"])?; @@ -366,7 +366,7 @@ fn start_args() -> Result<()> { #[tokio::test] async fn db_init_outside_future_executor() -> Result<()> { let _init_guard = zebra_test::init(); - let config = default_test_config()?; + let config = default_test_config(Mainnet)?; let start = Instant::now(); @@ -392,7 +392,7 @@ async fn db_init_outside_future_executor() -> Result<()> { fn persistent_mode() -> Result<()> { let _init_guard = zebra_test::init(); - let testdir = testdir()?.with_config(&mut persistent_test_config()?)?; + let testdir = testdir()?.with_config(&mut persistent_test_config(Mainnet)?)?; let testdir = &testdir; let mut child = testdir.spawn_child(args!["-v", "start"])?; @@ -457,7 +457,7 @@ fn ephemeral(cache_dir_config: EphemeralConfig, cache_dir_check: EphemeralCheck) let _init_guard = zebra_test::init(); - let mut config = default_test_config()?; + let mut config = default_test_config(Mainnet)?; let run_dir = testdir()?; let ignored_cache_dir = run_dir.path().join("state"); @@ -547,7 +547,7 @@ fn ephemeral(cache_dir_config: EphemeralConfig, cache_dir_check: EphemeralCheck) fn version_no_args() -> Result<()> { let _init_guard = zebra_test::init(); - let testdir = testdir()?.with_config(&mut default_test_config()?)?; + let testdir = testdir()?.with_config(&mut default_test_config(Mainnet)?)?; let child = testdir.spawn_child(args!["--version"])?; let output = child.wait_with_output()?; @@ -568,7 +568,7 @@ fn version_no_args() -> Result<()> { fn version_args() -> Result<()> { let _init_guard = zebra_test::init(); - let testdir = testdir()?.with_config(&mut default_test_config()?)?; + let testdir = testdir()?.with_config(&mut default_test_config(Mainnet)?)?; let testdir = &testdir; // unrecognized option `-f` @@ -622,7 +622,7 @@ fn app_no_args() -> Result<()> { let _init_guard = zebra_test::init(); // start caches state, so run one of the start tests with persistent state - let testdir = testdir()?.with_config(&mut persistent_test_config()?)?; + let testdir = testdir()?.with_config(&mut persistent_test_config(Mainnet)?)?; tracing::info!(?testdir, "running zebrad with no config (default settings)"); @@ -1319,7 +1319,7 @@ async fn metrics_endpoint() -> Result<()> { let url = format!("http://{endpoint}"); // Write a configuration that has metrics endpoint_addr set - let mut config = default_test_config()?; + let mut config = default_test_config(Mainnet)?; config.metrics.endpoint_addr = Some(endpoint.parse().unwrap()); let dir = testdir()?.with_config(&mut config)?; @@ -1376,7 +1376,7 @@ async fn tracing_endpoint() -> Result<()> { let url_filter = format!("{url_default}/filter"); // Write a configuration that has tracing endpoint_addr option set - let mut config = default_test_config()?; + let mut config = default_test_config(Mainnet)?; config.tracing.endpoint_addr = Some(endpoint.parse().unwrap()); let dir = testdir()?.with_config(&mut config)?; @@ -1483,7 +1483,7 @@ async fn rpc_endpoint(parallel_cpu_threads: bool) -> Result<()> { // Write a configuration that has RPC listen_addr set // [Note on port conflict](#Note on port conflict) - let mut config = random_known_rpc_port_config(parallel_cpu_threads)?; + let mut config = random_known_rpc_port_config(parallel_cpu_threads, Mainnet)?; let dir = testdir()?.with_config(&mut config)?; let mut child = dir.spawn_child(args!["start"])?; @@ -1542,7 +1542,7 @@ async fn rpc_endpoint_client_content_type() -> Result<()> { // Write a configuration that has RPC listen_addr set // [Note on port conflict](#Note on port conflict) - let mut config = random_known_rpc_port_config(true)?; + let mut config = random_known_rpc_port_config(true, Mainnet)?; let dir = testdir()?.with_config(&mut config)?; let mut child = dir.spawn_child(args!["start"])?; @@ -1628,7 +1628,7 @@ fn non_blocking_logger() -> Result<()> { // Write a configuration that has RPC listen_addr set // [Note on port conflict](#Note on port conflict) - let mut config = random_known_rpc_port_config(false)?; + let mut config = random_known_rpc_port_config(false, Mainnet)?; config.tracing.filter = Some("trace".to_string()); config.tracing.buffer_limit = 100; let zebra_rpc_address = config.rpc.listen_addr.unwrap(); @@ -2049,7 +2049,7 @@ fn zebra_zcash_listener_conflict() -> Result<()> { let listen_addr = format!("127.0.0.1:{port}"); // Write a configuration that has our created network listen_addr - let mut config = default_test_config()?; + let mut config = default_test_config(Mainnet)?; config.network.listen_addr = listen_addr.parse().unwrap(); let dir1 = testdir()?.with_config(&mut config)?; let regex1 = regex::escape(&format!("Opened Zcash protocol endpoint at {listen_addr}")); @@ -2078,7 +2078,7 @@ fn zebra_metrics_conflict() -> Result<()> { let listen_addr = format!("127.0.0.1:{port}"); // Write a configuration that has our created metrics endpoint_addr - let mut config = default_test_config()?; + let mut config = default_test_config(Mainnet)?; config.metrics.endpoint_addr = Some(listen_addr.parse().unwrap()); let dir1 = testdir()?.with_config(&mut config)?; let regex1 = regex::escape(&format!(r"Opened metrics endpoint at {listen_addr}")); @@ -2107,7 +2107,7 @@ fn zebra_tracing_conflict() -> Result<()> { let listen_addr = format!("127.0.0.1:{port}"); // Write a configuration that has our created tracing endpoint_addr - let mut config = default_test_config()?; + let mut config = default_test_config(Mainnet)?; config.tracing.endpoint_addr = Some(listen_addr.parse().unwrap()); let dir1 = testdir()?.with_config(&mut config)?; let regex1 = regex::escape(&format!(r"Opened tracing endpoint at {listen_addr}")); @@ -2141,7 +2141,7 @@ fn zebra_rpc_conflict() -> Result<()> { // [Note on port conflict](#Note on port conflict) // // This is the required setting to detect port conflicts. - let mut config = random_known_rpc_port_config(false)?; + let mut config = random_known_rpc_port_config(false, Mainnet)?; let dir1 = testdir()?.with_config(&mut config)?; let regex1 = regex::escape(&format!( @@ -2168,7 +2168,7 @@ fn zebra_state_conflict() -> Result<()> { // A persistent config has a fixed temp state directory, but asks the OS to // automatically choose an unused port - let mut config = persistent_test_config()?; + let mut config = persistent_test_config(Mainnet)?; let dir_conflict = testdir()?.with_config(&mut config)?; // Windows problems with this match will be worked on at #1654 @@ -2333,7 +2333,7 @@ async fn delete_old_databases() -> Result<()> { return Ok(()); } - let mut config = default_test_config()?; + let mut config = default_test_config(Mainnet)?; let run_dir = testdir()?; let cache_dir = run_dir.path().join("state"); @@ -2449,7 +2449,7 @@ async fn submit_block() -> Result<()> { #[test] fn end_of_support_is_checked_at_start() -> Result<()> { let _init_guard = zebra_test::init(); - let testdir = testdir()?.with_config(&mut default_test_config()?)?; + let testdir = testdir()?.with_config(&mut default_test_config(Mainnet)?)?; let mut child = testdir.spawn_child(args!["start"])?; // Give enough time to start up the eos task. @@ -2608,10 +2608,9 @@ async fn state_format_test( let test_name = &format!("{base_test_name}/apply_fake_version/{fake_version}"); tracing::info!(?network, "running {test_name} using zebra-state"); - let mut config = UseAnyState - .zebrad_config(test_name, false, Some(dir.path())) + let config = UseAnyState + .zebrad_config(test_name, false, Some(dir.path()), network) .expect("already checked config")?; - config.network.network = network; zebra_state::write_database_format_version_to_disk(fake_version, &config.state, network) .expect("can't write fake database version to disk"); diff --git a/zebrad/tests/common/config.rs b/zebrad/tests/common/config.rs index b6eead94f3e..3abf83d7eca 100644 --- a/zebrad/tests/common/config.rs +++ b/zebrad/tests/common/config.rs @@ -15,6 +15,7 @@ use std::{ use color_eyre::eyre::Result; use tempfile::TempDir; +use zebra_chain::parameters::Network::{self, *}; use zebra_test::net::random_known_port; use zebrad::{ components::{mempool, sync, tracing}, @@ -28,10 +29,11 @@ use crate::common::cached_state::DATABASE_FORMAT_CHECK_INTERVAL; /// - an ephemeral state, /// - the minimum syncer lookahead limit, and /// - shorter task intervals, to improve test coverage. -pub fn default_test_config() -> Result { +pub fn default_test_config(net: Network) -> Result { const TEST_DURATION: Duration = Duration::from_secs(30); let network = zebra_network::Config { + network: net, // The OS automatically chooses an unused port. listen_addr: "127.0.0.1:0".parse()?, crawl_new_peer_interval: TEST_DURATION, @@ -65,6 +67,21 @@ pub fn default_test_config() -> Result { let mut state = zebra_state::Config::ephemeral(); state.debug_validity_check_interval = Some(DATABASE_FORMAT_CHECK_INTERVAL); + // These are the ZF funding stream addresses for mainnet and testnet. + #[allow(unused_mut)] + let mut mining = zebra_rpc::config::mining::Config::default(); + + #[cfg(feature = "getblocktemplate-rpcs")] + { + let miner_address = if network.network == Mainnet { + "t3dvVE3SQEi7kqNzwrfNePxZ1d4hUyztBA1" + } else { + "t27eWDgjFYJGVXmzrXeVjnb5J3uXDM9xH9v" + }; + + mining.miner_address = Some(miner_address.parse().expect("hard-coded address is valid")); + } + let config = ZebradConfig { network, state, @@ -72,14 +89,15 @@ pub fn default_test_config() -> Result { mempool, consensus, tracing, + mining, ..ZebradConfig::default() }; Ok(config) } -pub fn persistent_test_config() -> Result { - let mut config = default_test_config()?; +pub fn persistent_test_config(network: Network) -> Result { + let mut config = default_test_config(network)?; config.state.ephemeral = false; Ok(config) } @@ -105,7 +123,10 @@ pub fn config_file_full_path(config_file: PathBuf) -> PathBuf { /// Returns a `zebrad` config with a random known RPC port. /// /// Set `parallel_cpu_threads` to true to auto-configure based on the number of CPU cores. -pub fn random_known_rpc_port_config(parallel_cpu_threads: bool) -> Result { +pub fn random_known_rpc_port_config( + parallel_cpu_threads: bool, + network: Network, +) -> Result { // [Note on port conflict](#Note on port conflict) let listen_port = random_known_port(); let listen_ip = "127.0.0.1".parse().expect("hard-coded IP is valid"); @@ -113,7 +134,7 @@ pub fn random_known_rpc_port_config(parallel_cpu_threads: bool) -> Result + Debug>( } // Get the zebrad config - let mut config = test_type - .zebrad_config(test_name, use_internet_connection, None) + let config = test_type + .zebrad_config(test_name, use_internet_connection, None, network) .expect("already checked config")?; - config.network.network = network; - let (zebrad_failure_messages, zebrad_ignore_messages) = test_type.zebrad_failure_messages(); // Writes a configuration that has RPC listen_addr set (if needed). @@ -311,12 +309,15 @@ where } // Get the zebrad config - let mut config = test_type - .zebrad_config(test_name, use_internet_connection, replace_cache_dir) + let config = test_type + .zebrad_config( + test_name, + use_internet_connection, + replace_cache_dir, + network, + ) .expect("already checked config")?; - config.network.network = network; - let (zebrad_failure_messages, zebrad_ignore_messages) = test_type.zebrad_failure_messages(); // Writes a configuration that does not have RPC listen_addr set. @@ -350,8 +351,10 @@ pub fn can_spawn_zebrad_for_test_type + Debug>( } // Check if we have any necessary cached states for the zebrad config. - // The cache_dir value doesn't matter here. - test_type.zebrad_config(test_name, true, None).is_some() + // The cache_dir and network values don't matter here. + test_type + .zebrad_config(test_name, true, None, Mainnet) + .is_some() } /// Panics if `$pred` is false, with an error report containing: diff --git a/zebrad/tests/common/sync.rs b/zebrad/tests/common/sync.rs index dd0a1390294..599a10c4b5a 100644 --- a/zebrad/tests/common/sync.rs +++ b/zebrad/tests/common/sync.rs @@ -206,8 +206,7 @@ pub fn sync_until( let reuse_tempdir = reuse_tempdir.into(); // Use a persistent state, so we can handle large syncs - let mut config = persistent_test_config()?; - config.network.network = network; + let mut config = persistent_test_config(network)?; config.state.debug_stop_at_height = Some(height.0); config.mempool.debug_enable_at_height = mempool_behavior.enable_at_height(); config.consensus.checkpoint_sync = checkpoint_sync; @@ -338,8 +337,8 @@ pub fn check_sync_logs_until( } /// Returns a test config for caching Zebra's state up to the mandatory checkpoint. -pub fn cached_mandatory_checkpoint_test_config() -> Result { - let mut config = persistent_test_config()?; +pub fn cached_mandatory_checkpoint_test_config(network: Network) -> Result { + let mut config = persistent_test_config(network)?; config.state.cache_dir = "/zebrad-cache".into(); // To get to the mandatory checkpoint, we need to sync lots of blocks. @@ -387,9 +386,8 @@ pub fn create_cached_database_height( eprintln!("creating cached database"); // Use a persistent state, so we can handle large syncs - let mut config = cached_mandatory_checkpoint_test_config()?; + let mut config = cached_mandatory_checkpoint_test_config(network)?; // TODO: add convenience methods? - config.network.network = network; config.state.debug_stop_at_height = Some(height.0); config.consensus.debug_skip_parameter_preload = debug_skip_parameter_preload; config.consensus.checkpoint_sync = checkpoint_sync; diff --git a/zebrad/tests/common/test_type.rs b/zebrad/tests/common/test_type.rs index adb5fd4b897..ddd7386c137 100644 --- a/zebrad/tests/common/test_type.rs +++ b/zebrad/tests/common/test_type.rs @@ -8,6 +8,7 @@ use std::{ use indexmap::IndexSet; +use zebra_chain::parameters::Network; use zebra_network::CacheDir; use zebra_test::{command::NO_MATCHES_REGEX_ITER, prelude::*}; use zebrad::config::ZebradConfig; @@ -176,12 +177,13 @@ impl TestType { test_name: Str, use_internet_connection: bool, replace_cache_dir: Option<&Path>, + network: Network, ) -> Option> { let config = if self.needs_zebra_rpc_server() { // This is what we recommend our users configure. - random_known_rpc_port_config(true) + random_known_rpc_port_config(true, network) } else { - default_test_config() + default_test_config(network) }; let mut config = match config { @@ -209,12 +211,6 @@ impl TestType { config.mempool.debug_enable_at_height = Some(0); } - // Add a fake miner address for mining RPCs - #[cfg(feature = "getblocktemplate-rpcs")] - let _ = config.mining.miner_address.insert( - zebra_chain::transparent::Address::from_script_hash(config.network.network, [0x7e; 20]), - ); - // If we have a cached state, or we don't want to be ephemeral, update the config to use it if replace_cache_dir.is_some() || self.needs_zebra_cached_state() { let zebra_state_path = replace_cache_dir From a30d3c19af3206e1af1c59307675d52331ae634e Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 16 Oct 2023 07:49:16 +1000 Subject: [PATCH 06/13] Correctly skip checks in lightwalletd send transactions test --- .../common/lightwalletd/send_transaction_test.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 5cfb1c68a30..f63ea9a6014 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -210,10 +210,11 @@ pub async fn run() -> Result<()> { .await? .into_inner(); - // check that lightwalletd queries the mempool. - zebrad.expect_stdout_line_matches("answered mempool request .*req.*=.*TransactionIds")?; + if has_tx_with_shielded_elements { + // check that lightwalletd queries the mempool. + zebrad.expect_stdout_line_matches("answered mempool request .*req.*=.*TransactionIds")?; + } - // GetMempoolTx: make sure at least one of the transactions were inserted into the mempool. let mut counter = 0; while let Some(tx) = transactions_stream.message().await? { let hash: [u8; 32] = tx.hash.clone().try_into().expect("hash is correct length"); @@ -228,6 +229,8 @@ pub async fn run() -> Result<()> { counter += 1; } + // GetMempoolTx: make sure at least one of the transactions were inserted into the mempool. + // // TODO: Update `load_transactions_from_future_blocks()` to return block height offsets and, // only check if a transaction from the first block has shielded elements assert!( @@ -235,7 +238,7 @@ pub async fn run() -> Result<()> { "failed to read v4+ transactions with shielded elements from future blocks in mempool via lightwalletd" ); - // GetMempoolTx: make sure at least one of the transactions were inserted into the mempool. + // TODO: GetMempoolStream: make sure at least one of the transactions were inserted into the mempool. tracing::info!("calling GetMempoolStream gRPC to fetch transactions..."); let mut transaction_stream = rpc_client.get_mempool_stream(Empty {}).await?.into_inner(); From d61e1397de37e6fe1bf742bb38805f86aac6c043 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 16 Oct 2023 08:02:56 +1000 Subject: [PATCH 07/13] Wait longer for transaction query --- .../common/lightwalletd/send_transaction_test.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index f63ea9a6014..57844bfa93c 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -159,10 +159,12 @@ pub async fn run() -> Result<()> { .await? .into_inner(); - // Lightwalletd won't call `get_raw_mempool` again until 2 seconds after the last call + // Lightwalletd won't call `get_raw_mempool` again until 2 seconds after the last call: // + // + // So we need to wait longer than that here. let sleep_until_lwd_last_mempool_refresh = - tokio::time::sleep(std::time::Duration::from_secs(2)); + tokio::time::sleep(std::time::Duration::from_secs(3)); let transaction_hashes: Vec = transactions.iter().map(|tx| tx.hash()).collect(); @@ -201,7 +203,7 @@ pub async fn run() -> Result<()> { zebrad.expect_stdout_line_matches("sending mempool transaction broadcast")?; // Wait for more transactions to verify, `GetMempoolTx` only returns txs where tx.HasShieldedElements() // - tokio::time::sleep(std::time::Duration::from_secs(1)).await; + tokio::time::sleep(std::time::Duration::from_secs(2)).await; sleep_until_lwd_last_mempool_refresh.await; tracing::info!("calling GetMempoolTx gRPC to fetch transactions..."); @@ -210,11 +212,10 @@ pub async fn run() -> Result<()> { .await? .into_inner(); - if has_tx_with_shielded_elements { - // check that lightwalletd queries the mempool. - zebrad.expect_stdout_line_matches("answered mempool request .*req.*=.*TransactionIds")?; - } + tracing::info!("waiting for lightwalletd to query the mempool..."); + zebrad.expect_stdout_line_matches("answered mempool request .*req.*=.*TransactionIds")?; + tracing::info!("checking the mempool contains some of the sent transactions..."); let mut counter = 0; while let Some(tx) = transactions_stream.message().await? { let hash: [u8; 32] = tx.hash.clone().try_into().expect("hash is correct length"); From f10929df975304511644005c7195c84f60b3928f Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 16 Oct 2023 10:34:38 +1000 Subject: [PATCH 08/13] Wait longer before re-checking the mempool --- zebrad/tests/common/lightwalletd/send_transaction_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 57844bfa93c..439207189c8 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -164,7 +164,7 @@ pub async fn run() -> Result<()> { // // So we need to wait longer than that here. let sleep_until_lwd_last_mempool_refresh = - tokio::time::sleep(std::time::Duration::from_secs(3)); + tokio::time::sleep(std::time::Duration::from_secs(5)); let transaction_hashes: Vec = transactions.iter().map(|tx| tx.hash()).collect(); From fe6a7cb2eed32119a23f636cf7f646f0ab51a080 Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 16 Oct 2023 13:22:20 +1000 Subject: [PATCH 09/13] Skip mempool checks if it wasn't queried by lightwalletd --- .../lightwalletd/send_transaction_test.rs | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index 439207189c8..ba34dec46f0 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -16,7 +16,7 @@ //! were obtained. This is to ensure that zebra does not reject the transactions because they have //! already been seen in a block. -use std::{cmp::min, sync::Arc}; +use std::{cmp::min, sync::Arc, time::Duration}; use color_eyre::eyre::Result; @@ -36,6 +36,7 @@ use crate::common::{ sync::wait_for_zebrad_and_lightwalletd_sync, wallet_grpc::{self, connect_to_lightwalletd, Empty, Exclude}, }, + sync::LARGE_CHECKPOINT_TIMEOUT, test_type::TestType::{self, *}, }; @@ -162,9 +163,9 @@ pub async fn run() -> Result<()> { // Lightwalletd won't call `get_raw_mempool` again until 2 seconds after the last call: // // - // So we need to wait longer than that here. + // So we need to wait much longer than that here. let sleep_until_lwd_last_mempool_refresh = - tokio::time::sleep(std::time::Duration::from_secs(5)); + tokio::time::sleep(std::time::Duration::from_secs(4)); let transaction_hashes: Vec = transactions.iter().map(|tx| tx.hash()).collect(); @@ -212,9 +213,21 @@ pub async fn run() -> Result<()> { .await? .into_inner(); - tracing::info!("waiting for lightwalletd to query the mempool..."); - zebrad.expect_stdout_line_matches("answered mempool request .*req.*=.*TransactionIds")?; + // Sometimes lightwalletd doesn't check the mempool, and waits for the next block instead. + // If that happens, we skip the rest of the test. + tracing::info!("checking if lightwalletd has queried the mempool..."); + // We need a short timeout here, because sometimes this message is not logged. + zebrad = zebrad.with_timeout(Duration::from_secs(60)); + let tx_log = zebrad.expect_stdout_line_matches("answered mempool request .*req.*=.*TransactionIds"); + // Reset the failed timeout and give the rest of the test enough time to finish. + zebrad = zebrad.with_timeout(LARGE_CHECKPOINT_TIMEOUT); + + if tx_log.is_err() { + tracing::info!("lightwalletd didn't query the mempool, skipping mempool contents checks"); + return Ok(()); + } + tracing::info!("checking the mempool contains some of the sent transactions..."); let mut counter = 0; while let Some(tx) = transactions_stream.message().await? { From 73407e8e4cb63dd310524e067eca7e03c6e9a00e Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 16 Oct 2023 13:25:03 +1000 Subject: [PATCH 10/13] rustfmt --- zebrad/tests/common/lightwalletd/send_transaction_test.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index ba34dec46f0..b67485981f7 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -219,7 +219,8 @@ pub async fn run() -> Result<()> { // We need a short timeout here, because sometimes this message is not logged. zebrad = zebrad.with_timeout(Duration::from_secs(60)); - let tx_log = zebrad.expect_stdout_line_matches("answered mempool request .*req.*=.*TransactionIds"); + let tx_log = + zebrad.expect_stdout_line_matches("answered mempool request .*req.*=.*TransactionIds"); // Reset the failed timeout and give the rest of the test enough time to finish. zebrad = zebrad.with_timeout(LARGE_CHECKPOINT_TIMEOUT); From 3221a0852c56ee2e30c00c585741ac01072a5fce Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 17 Oct 2023 05:59:05 +1000 Subject: [PATCH 11/13] clippy and rustfmt --- zebrad/tests/common/lightwalletd/send_transaction_test.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index b67485981f7..b0cf4e4ffa0 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -222,13 +222,14 @@ pub async fn run() -> Result<()> { let tx_log = zebrad.expect_stdout_line_matches("answered mempool request .*req.*=.*TransactionIds"); // Reset the failed timeout and give the rest of the test enough time to finish. + #[allow(unused_assignments)] zebrad = zebrad.with_timeout(LARGE_CHECKPOINT_TIMEOUT); if tx_log.is_err() { tracing::info!("lightwalletd didn't query the mempool, skipping mempool contents checks"); return Ok(()); } - + tracing::info!("checking the mempool contains some of the sent transactions..."); let mut counter = 0; while let Some(tx) = transactions_stream.message().await? { From d5d29e7f1ae014aa3813e05f22f9f5c56bb78340 Mon Sep 17 00:00:00 2001 From: teor Date: Tue, 17 Oct 2023 06:34:43 +1000 Subject: [PATCH 12/13] Fix attribute issue --- zebrad/tests/common/lightwalletd/send_transaction_test.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zebrad/tests/common/lightwalletd/send_transaction_test.rs b/zebrad/tests/common/lightwalletd/send_transaction_test.rs index b0cf4e4ffa0..870d3992b56 100644 --- a/zebrad/tests/common/lightwalletd/send_transaction_test.rs +++ b/zebrad/tests/common/lightwalletd/send_transaction_test.rs @@ -223,7 +223,9 @@ pub async fn run() -> Result<()> { zebrad.expect_stdout_line_matches("answered mempool request .*req.*=.*TransactionIds"); // Reset the failed timeout and give the rest of the test enough time to finish. #[allow(unused_assignments)] - zebrad = zebrad.with_timeout(LARGE_CHECKPOINT_TIMEOUT); + { + zebrad = zebrad.with_timeout(LARGE_CHECKPOINT_TIMEOUT); + } if tx_log.is_err() { tracing::info!("lightwalletd didn't query the mempool, skipping mempool contents checks"); From 5ca527f8445c687cf4b046ef3409c62e76a40130 Mon Sep 17 00:00:00 2001 From: teor Date: Wed, 18 Oct 2023 07:37:59 +1000 Subject: [PATCH 13/13] Fix typo Co-authored-by: Arya --- book/src/user/mining.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/book/src/user/mining.md b/book/src/user/mining.md index 14d14cf5f0a..fd593096012 100644 --- a/book/src/user/mining.md +++ b/book/src/user/mining.md @@ -15,7 +15,7 @@ Contents: ## Download Zebra [#download-and-build-zebra]: #download-and-build-zebra -The easiest way to run Zebra for mining is [our docker images](https://zebra.zfnd.org/user/mining-docker.html). +The easiest way to run Zebra for mining is with [our docker images](https://zebra.zfnd.org/user/mining-docker.html). If you have [installed Zebra another way](https://zebra.zfnd.org/user/install.html), follow the instructions below to start mining: