diff --git a/Cargo.lock b/Cargo.lock index 86284315686c..b77bef5d972a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -135,6 +135,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "aho-corasick" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8f9420f797f2d9e935edf629310eb938a0d839f984e25327f3c7eed22300c" +dependencies = [ + "memchr", +] + [[package]] name = "allocator-api2" version = "0.2.16" @@ -620,7 +629,7 @@ checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" dependencies = [ "lazy_static", "memchr", - "regex-automata", + "regex-automata 0.1.10", ] [[package]] @@ -2845,7 +2854,7 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd" dependencies = [ - "aho-corasick", + "aho-corasick 0.7.18", "bstr", "fnv", "log", @@ -4449,7 +4458,7 @@ version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] @@ -4458,7 +4467,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] @@ -4478,9 +4487,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memfd" @@ -6432,16 +6441,19 @@ checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630" name = "polkadot" version = "0.9.43" dependencies = [ + "ansi_term", "assert_cmd", "color-eyre", "nix 0.26.2", "polkadot-cli", "polkadot-core-primitives", + "polkadot-execute-worker", "polkadot-node-core-pvf", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-overseer", + "polkadot-prepare-worker", + "regex", + "rustc_version", "substrate-build-script-utils", "substrate-rpc-client", "tempfile", @@ -6675,6 +6687,15 @@ dependencies = [ "thiserror", ] +[[package]] +name = "polkadot-execute-worker" +version = "0.9.43" +dependencies = [ + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-execute-worker", + "substrate-build-script-utils", +] + [[package]] name = "polkadot-gossip-support" version = "0.9.43" @@ -7004,12 +7025,14 @@ dependencies = [ "parity-scale-codec", "pin-project", "polkadot-core-primitives", + "polkadot-execute-worker", "polkadot-node-core-pvf-common", "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", "polkadot-node-metrics", "polkadot-node-primitives", "polkadot-parachain", + "polkadot-prepare-worker", "polkadot-primitives", "rand 0.8.5", "slotmap", @@ -7108,6 +7131,7 @@ dependencies = [ "sp-io", "sp-maybe-compressed-blob", "sp-tracing", + "substrate-build-script-utils", "tikv-jemalloc-ctl", "tokio", "tracing-gum", @@ -7372,6 +7396,15 @@ dependencies = [ "thiserror", ] +[[package]] +name = "polkadot-prepare-worker" +version = "0.9.43" +dependencies = [ + "polkadot-node-core-pvf-common", + "polkadot-node-core-pvf-prepare-worker", + "substrate-build-script-utils", +] + [[package]] name = "polkadot-primitives" version = "0.9.43" @@ -7885,9 +7918,6 @@ dependencies = [ "polkadot-node-core-backing", "polkadot-node-core-candidate-validation", "polkadot-node-core-dispute-coordinator", - "polkadot-node-core-pvf-common", - "polkadot-node-core-pvf-execute-worker", - "polkadot-node-core-pvf-prepare-worker", "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", @@ -8651,13 +8681,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.6.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ - "aho-corasick", + "aho-corasick 1.0.3", "memchr", - "regex-syntax", + "regex-automata 0.3.6", + "regex-syntax 0.7.4", ] [[package]] @@ -8666,7 +8697,18 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.27", +] + +[[package]] +name = "regex-automata" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +dependencies = [ + "aho-corasick 1.0.3", + "memchr", + "regex-syntax 0.7.4", ] [[package]] @@ -8675,6 +8717,12 @@ version = "0.6.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +[[package]] +name = "regex-syntax" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" + [[package]] name = "remote-ext-tests-bags-list" version = "0.9.43" diff --git a/Cargo.toml b/Cargo.toml index 0a6fc1b97891..f787ebc1a3a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,14 +2,6 @@ name = "polkadot" path = "src/main.rs" -[[bin]] -name = "polkadot-execute-worker" -path = "src/bin/execute-worker.rs" - -[[bin]] -name = "polkadot-prepare-worker" -path = "src/bin/prepare-worker.rs" - [package] name = "polkadot" description = "Implementation of a `https://polkadot.network` node in Rust based on the Substrate framework." @@ -38,10 +30,6 @@ polkadot-node-core-pvf = { path = "node/core/pvf" } polkadot-node-core-pvf-prepare-worker = { path = "node/core/pvf/prepare-worker" } polkadot-overseer = { path = "node/overseer" } -# Needed for worker binaries. -polkadot-node-core-pvf-common = { path = "node/core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "node/core/pvf/execute-worker" } - [dev-dependencies] assert_cmd = "2.0.4" nix = { version = "0.26.1", features = ["signal"] } @@ -49,13 +37,24 @@ tempfile = "3.2.0" tokio = "1.24.2" substrate-rpc-client = { git = "https://github.com/paritytech/substrate", branch = "master" } polkadot-core-primitives = { path = "core-primitives" } +polkadot-execute-worker = { path = "bin/execute-worker" } +polkadot-prepare-worker = { path = "bin/prepare-worker" } [build-dependencies] +ansi_term = "0.12.1" +regex = "1.9.3" +rustc_version = "0.4.0" substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } +# To get the binary names. +polkadot-execute-worker = { path = "bin/execute-worker" } +polkadot-prepare-worker = { path = "bin/prepare-worker" } + [workspace] members = [ "cli", + "bin/execute-worker", + "bin/prepare-worker", "core-primitives", "erasure-coding", "erasure-coding/fuzzer", @@ -142,6 +141,12 @@ members = [ "utils/generate-bags", ] +default-members = [ + ".", + "bin/execute-worker", + "bin/prepare-worker", +] + [badges] maintenance = { status = "actively-developed" } diff --git a/bin/execute-worker/Cargo.toml b/bin/execute-worker/Cargo.toml new file mode 100644 index 000000000000..73820df432d8 --- /dev/null +++ b/bin/execute-worker/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "polkadot-execute-worker" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[[bin]] +name = "polkadot-execute-worker" +path = "src/main.rs" + +[dependencies] +polkadot-node-core-pvf-common = { path = "../../node/core/pvf/common" } +polkadot-node-core-pvf-execute-worker = { path = "../../node/core/pvf/execute-worker" } + +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/bin/execute-worker/build.rs b/bin/execute-worker/build.rs new file mode 100644 index 000000000000..84fe22e23ed6 --- /dev/null +++ b/bin/execute-worker/build.rs @@ -0,0 +1,22 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the node and binary workers + // when the version changes. + substrate_build_script_utils::rerun_if_git_head_changed(); +} diff --git a/bin/execute-worker/src/lib.rs b/bin/execute-worker/src/lib.rs new file mode 100644 index 000000000000..0d5591d45852 --- /dev/null +++ b/bin/execute-worker/src/lib.rs @@ -0,0 +1,18 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +/// Execute worker binary name +pub const BINARY_NAME: &str = "polkadot-execute-worker"; diff --git a/bin/execute-worker/src/main.rs b/bin/execute-worker/src/main.rs new file mode 100644 index 000000000000..32d3cbc1d131 --- /dev/null +++ b/bin/execute-worker/src/main.rs @@ -0,0 +1,33 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Execute worker. + +#[cfg(test)] +use polkadot_execute_worker::BINARY_NAME; + +polkadot_node_core_pvf_common::decl_worker_main!( + "execute-worker", + polkadot_node_core_pvf_execute_worker::worker_entrypoint, + env!("SUBSTRATE_CLI_IMPL_VERSION") +); + +#[test] +fn name_test() { + // If binary name ever needs to change, it must be changed in the package manifest + // as well as in the constant value + assert_eq!(env!("CARGO_BIN_NAME"), BINARY_NAME); +} diff --git a/src/bin/execute-worker.rs b/bin/execute-worker/tests/integration.rs similarity index 79% rename from src/bin/execute-worker.rs rename to bin/execute-worker/tests/integration.rs index 72cab799d753..a3b6e2647784 100644 --- a/src/bin/execute-worker.rs +++ b/bin/execute-worker/tests/integration.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -//! Execute worker. +//! Dummy integration test needed for the binary to get build when other intergation +//! tests are run -polkadot_node_core_pvf_common::decl_worker_main!( - "execute-worker", - polkadot_node_core_pvf_execute_worker::worker_entrypoint, - env!("SUBSTRATE_CLI_IMPL_VERSION") -); +#[test] +fn dummy_test() { + assert!(true); +} diff --git a/bin/prepare-worker/Cargo.toml b/bin/prepare-worker/Cargo.toml new file mode 100644 index 000000000000..50597c91ab2c --- /dev/null +++ b/bin/prepare-worker/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "polkadot-prepare-worker" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[[bin]] +name = "polkadot-prepare-worker" +path = "src/main.rs" + +[dependencies] +polkadot-node-core-pvf-common = { path = "../../node/core/pvf/common" } +polkadot-node-core-pvf-prepare-worker = { path = "../../node/core/pvf/prepare-worker" } + +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/bin/prepare-worker/build.rs b/bin/prepare-worker/build.rs new file mode 100644 index 000000000000..84fe22e23ed6 --- /dev/null +++ b/bin/prepare-worker/build.rs @@ -0,0 +1,22 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_cargo_keys(); + // For the node/worker version check, make sure we always rebuild the node and binary workers + // when the version changes. + substrate_build_script_utils::rerun_if_git_head_changed(); +} diff --git a/bin/prepare-worker/src/lib.rs b/bin/prepare-worker/src/lib.rs new file mode 100644 index 000000000000..058ace9d5f6d --- /dev/null +++ b/bin/prepare-worker/src/lib.rs @@ -0,0 +1,18 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +/// Prepare worker binary name +pub const BINARY_NAME: &str = "polkadot-prepare-worker"; diff --git a/bin/prepare-worker/src/main.rs b/bin/prepare-worker/src/main.rs new file mode 100644 index 000000000000..e3cd31fd26f1 --- /dev/null +++ b/bin/prepare-worker/src/main.rs @@ -0,0 +1,33 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Prepare worker. + +#[cfg(test)] +use polkadot_prepare_worker::BINARY_NAME; + +polkadot_node_core_pvf_common::decl_worker_main!( + "prepare-worker", + polkadot_node_core_pvf_prepare_worker::worker_entrypoint, + env!("SUBSTRATE_CLI_IMPL_VERSION") +); + +#[test] +fn name_test() { + // If binary name ever needs to change, it must be changed in the package manifest + // as well as in the constant value + assert_eq!(env!("CARGO_BIN_NAME"), BINARY_NAME); +} diff --git a/src/bin/prepare-worker.rs b/bin/prepare-worker/tests/integration.rs similarity index 79% rename from src/bin/prepare-worker.rs rename to bin/prepare-worker/tests/integration.rs index 695f66cc7b7d..a3b6e2647784 100644 --- a/src/bin/prepare-worker.rs +++ b/bin/prepare-worker/tests/integration.rs @@ -14,10 +14,10 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -//! Prepare worker. +//! Dummy integration test needed for the binary to get build when other intergation +//! tests are run -polkadot_node_core_pvf_common::decl_worker_main!( - "prepare-worker", - polkadot_node_core_pvf_prepare_worker::worker_entrypoint, - env!("SUBSTRATE_CLI_IMPL_VERSION") -); +#[test] +fn dummy_test() { + assert!(true); +} diff --git a/build.rs b/build.rs index 84fe22e23ed6..001728e5b421 100644 --- a/build.rs +++ b/build.rs @@ -14,9 +14,118 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +//! Build script to ensure that node/worker versions stay in sync and PVF worker binaries are always +//! built whenever polkadot is built. +//! +//! NOTE: The crates for the workers should still be specified in `default-members` in order for +//! `cargo install` to work. +//! +//! # Testing +//! +//! The following scenarios have been tested. Run these tests after changes to this script. +//! +//! - [x] `cargo clean` then `cargo run` (workers should be built) +//! +//! - [x] `cargo run`, commit, `cargo run` again (workers should be rebuilt the second time) +//! +//! - [x] `cargo clean` then `cargo build` (workers should be built) +//! +//! - [x] `cargo + run` (same as `cargo run`) +//! +//! - [ ] TODO: `cargo clean` then `cargo install` (latest workers should be installed) +//! +//! - [ ] TODO: `cargo clean`, `cargo run`, then `cargo install` with binaries already having been +//! built. +//! +//! - [x] `cargo run` with a profile, like `--profile testnet` (same as `cargo run`) + +use std::{env::var, path::Path, process::Command}; + +use polkadot_execute_worker::BINARY_NAME as EXECUTE_BINARY_NAME; +use polkadot_prepare_worker::BINARY_NAME as PREPARE_BINARY_NAME; + fn main() { - substrate_build_script_utils::generate_cargo_keys(); + // Always build PVF worker binaries whenever polkadot is built. + // + // This is needed because `default-members` does the same thing, but only for `cargo build` -- + // by itself, `default-members` does not work for `cargo run`. + { + let cargo = var("CARGO").expect("`CARGO` env variable is always set by cargo"); + let target = var("TARGET").expect("`TARGET` env variable is always set by cargo"); + let out_dir = var("OUT_DIR").expect("`OUT_DIR` env variable is always set by cargo"); + let target_dir = format!("{}/workers", out_dir); + + // HACK: Get the profile from OUT_DIR instead of the PROFILE env var. If we are using e.g. + // testnet which inherits from release, then PROFILE will be release. The only way to get + // the actual profile (e.g. testnet) is with this hacky parsing code. 🙈 We need to get the + // actual profile to pass along settings like LTO (which is not even available to this build + // script) and build the binaries as expected. + let re = regex::Regex::new(r".*/target/(?\w+)/build/.*").unwrap(); + let caps = re + .captures(&out_dir) + .expect("regex broke, please contact your local regex repair facility."); + let profile = &caps["profile"]; + + // Construct the `cargo build ...` command. + // + // NOTE: We don't pass along features because the workers don't use any right now. + let mut args = vec![ + "build", + "-p", + EXECUTE_BINARY_NAME, + "-p", + PREPARE_BINARY_NAME, + "--target", + &target, + "--target-dir", + &target_dir, + ]; + // Needed to prevent an error. + if profile != "debug" { + args.push("--profile"); + args.push(&profile); + } + let mut build_cmd = Command::new(cargo); + build_cmd.args(&args); + + eprintln!( + "{}", + colorize_info_message("Information that should be included in a bug report.") + ); + eprintln!("{} {:?}", colorize_info_message("Executing build command:"), build_cmd); + eprintln!( + "{} {:?}", + colorize_info_message("Using rustc version:"), + rustc_version::version() + ); + + match build_cmd.status().map(|s| s.success()) { + Ok(true) => (), + // Use `process.exit(1)` to have a clean error output. + _ => std::process::exit(1), + } + + // Move the built worker binaries into the same location as the `polkadot` binary. + std::fs::rename( + Path::new(&format!("{target_dir}/{target}/{profile}/{EXECUTE_BINARY_NAME}")), + Path::new(&format!("{target_dir}/../../../../{EXECUTE_BINARY_NAME}")), + ) + .unwrap(); + std::fs::rename( + Path::new(&format!("{target_dir}/{target}/{profile}/{PREPARE_BINARY_NAME}")), + Path::new(&format!("{target_dir}/../../../../{PREPARE_BINARY_NAME}")), + ) + .unwrap(); + } + // For the node/worker version check, make sure we always rebuild the node and binary workers // when the version changes. substrate_build_script_utils::rerun_if_git_head_changed(); } + +/// Colorize an info message. +/// +/// Returns the colorized message. +fn colorize_info_message(message: &str) -> String { + ansi_term::Color::Yellow.bold().paint(message).to_string() +} diff --git a/node/core/pvf/Cargo.toml b/node/core/pvf/Cargo.toml index 02a56ed9d2df..9fe072561872 100644 --- a/node/core/pvf/Cargo.toml +++ b/node/core/pvf/Cargo.toml @@ -32,6 +32,8 @@ polkadot-node-core-pvf-prepare-worker = { path = "prepare-worker" } polkadot-node-metrics = { path = "../../metrics" } polkadot-node-primitives = { path = "../../primitives" } polkadot-primitives = { path = "../../../primitives" } +polkadot-prepare-worker = { path = "../../../bin/prepare-worker" } +polkadot-execute-worker = { path = "../../../bin/execute-worker" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-wasm-interface = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/node/core/pvf/prepare-worker/Cargo.toml b/node/core/pvf/prepare-worker/Cargo.toml index 9ee009de44bb..81213ab2d368 100644 --- a/node/core/pvf/prepare-worker/Cargo.toml +++ b/node/core/pvf/prepare-worker/Cargo.toml @@ -27,6 +27,9 @@ sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "master" } sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" } +[build-dependencies] +substrate-build-script-utils = { git = "https://github.com/paritytech/substrate", branch = "master" } + [target.'cfg(target_os = "linux")'.dependencies] tikv-jemalloc-ctl = "0.5.0" diff --git a/node/core/pvf/src/host.rs b/node/core/pvf/src/host.rs index a5772e34e16e..5e51f9a2dfc4 100644 --- a/node/core/pvf/src/host.rs +++ b/node/core/pvf/src/host.rs @@ -52,12 +52,6 @@ pub const PREPARE_FAILURE_COOLDOWN: Duration = Duration::from_millis(200); /// The amount of times we will retry failed prepare jobs. pub const NUM_PREPARE_RETRIES: u32 = 5; -/// The name of binary spawned to prepare a PVF artifact -pub const PREPARE_BINARY_NAME: &str = "polkadot-prepare-worker"; - -/// The name of binary spawned to execute a PVF -pub const EXECUTE_BINARY_NAME: &str = "polkadot-execute-worker"; - /// An alias to not spell the type for the oneshot sender for the PVF execution result. pub(crate) type ResultSender = oneshot::Sender>; diff --git a/node/core/pvf/src/lib.rs b/node/core/pvf/src/lib.rs index 2ed3f5242ded..50011cf360eb 100644 --- a/node/core/pvf/src/lib.rs +++ b/node/core/pvf/src/lib.rs @@ -105,8 +105,10 @@ pub mod testing; pub use sp_tracing; pub use error::{InvalidCandidate, ValidationError}; -pub use host::{start, Config, ValidationHost, EXECUTE_BINARY_NAME, PREPARE_BINARY_NAME}; +pub use host::{start, Config, ValidationHost}; pub use metrics::Metrics; +pub use polkadot_execute_worker::BINARY_NAME as EXECUTE_BINARY_NAME; +pub use polkadot_prepare_worker::BINARY_NAME as PREPARE_BINARY_NAME; pub use priority::Priority; pub use worker_intf::{framed_recv, framed_send, JOB_TIMEOUT_WALL_CLOCK_FACTOR}; diff --git a/node/malus/Cargo.toml b/node/malus/Cargo.toml index 08656ea9f3da..8589d05a5522 100644 --- a/node/malus/Cargo.toml +++ b/node/malus/Cargo.toml @@ -12,19 +12,6 @@ publish = false name = "malus" path = "src/malus.rs" -# Use artifact dependencies once stable. -# See https://github.com/rust-lang/cargo/issues/9096. -[[bin]] -name = "polkadot-execute-worker" -path = "../../src/bin/execute-worker.rs" -# Prevent rustdoc error. Already documented from top-level Cargo.toml. -doc = false -[[bin]] -name = "polkadot-prepare-worker" -path = "../../src/bin/prepare-worker.rs" -# Prevent rustdoc error. Already documented from top-level Cargo.toml. -doc = false - [dependencies] polkadot-cli = { path = "../../cli", features = [ "malus", "rococo-native", "kusama-native", "westend-native", "polkadot-native" ] } polkadot-node-subsystem = { path = "../subsystem" } @@ -47,11 +34,6 @@ gum = { package = "tracing-gum", path = "../gum/" } erasure = { package = "polkadot-erasure-coding", path = "../../erasure-coding" } rand = "0.8.5" -# Required for worker binaries to build. -polkadot-node-core-pvf-common = { path = "../core/pvf/common" } -polkadot-node-core-pvf-execute-worker = { path = "../core/pvf/execute-worker" } -polkadot-node-core-pvf-prepare-worker = { path = "../core/pvf/prepare-worker" } - [dev-dependencies] polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" } diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index dafca393cd4f..5b6fe261a690 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -79,7 +79,7 @@ build-malus: - .compiler-info - .collect-artifacts script: - - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus + - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus -p polkadot-execute-worker -p polkadot-prepare-worker # pack artifacts - mkdir -p ./artifacts - mv ./target/testnet/malus ./artifacts/. diff --git a/tests/workers.rs b/tests/workers.rs index 2872a1298dcd..07dad41f1492 100644 --- a/tests/workers.rs +++ b/tests/workers.rs @@ -15,22 +15,27 @@ // along with Polkadot. If not, see . use polkadot_cli::NODE_VERSION; +use polkadot_execute_worker::BINARY_NAME as EXECUTE_WORKER_EXE; +use polkadot_prepare_worker::BINARY_NAME as PREPARE_WORKER_EXE; use std::process::Command; -const PREPARE_WORKER_EXE: &str = env!("CARGO_BIN_EXE_polkadot-prepare-worker"); -const EXECUTE_WORKER_EXE: &str = env!("CARGO_BIN_EXE_polkadot-execute-worker"); - #[test] fn worker_binaries_have_same_version_as_node() { + // We're in `deps/` directory, target directory is one level up + let mut path = std::env::current_exe().unwrap(); + path.pop(); + path.pop(); + path.push(&PREPARE_WORKER_EXE); let prep_worker_version = - Command::new(&PREPARE_WORKER_EXE).args(["--version"]).output().unwrap().stdout; + Command::new(path.clone()).args(["--version"]).output().unwrap().stdout; let prep_worker_version = std::str::from_utf8(&prep_worker_version) .expect("version is printed as a string; qed") .trim(); assert_eq!(prep_worker_version, NODE_VERSION); - let exec_worker_version = - Command::new(&EXECUTE_WORKER_EXE).args(["--version"]).output().unwrap().stdout; + path.pop(); + path.push(&EXECUTE_WORKER_EXE); + let exec_worker_version = Command::new(path).args(["--version"]).output().unwrap().stdout; let exec_worker_version = std::str::from_utf8(&exec_worker_version) .expect("version is printed as a string; qed") .trim();