diff --git a/.github/workflows/crates-io.yml b/.github/workflows/crates-io.yml index 65fc7876e06..faf80549216 100644 --- a/.github/workflows/crates-io.yml +++ b/.github/workflows/crates-io.yml @@ -30,7 +30,7 @@ jobs: uses: dtolnay/rust-toolchain@stable with: targets: wasm32-unknown-unknown - components: llvm-tools + components: llvm-tools, rust-src - name: "Publish packages (simulate)" if: ${{ !inputs.publish }} diff --git a/Cargo.lock b/Cargo.lock index 2ec6039cf82..44f413a92f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5773,8 +5773,8 @@ dependencies = [ name = "gbuiltin-proxy" version = "1.6.2" dependencies = [ - "derive_more 0.99.18", "gprimitives", + "parity-scale-codec", "scale-info", ] @@ -13033,7 +13033,6 @@ checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "base64 0.22.1", "bytes", - "futures-channel", "futures-core", "futures-util", "http 1.1.0", diff --git a/docker/runtime-fuzzer/scripts/fuzzer.sh b/docker/runtime-fuzzer/scripts/fuzzer.sh index 727078c6900..2886a840570 100755 --- a/docker/runtime-fuzzer/scripts/fuzzer.sh +++ b/docker/runtime-fuzzer/scripts/fuzzer.sh @@ -67,8 +67,8 @@ function start_container_post { --workdir /gear/utils/runtime-fuzzer \ --name ${CONTAINER_NAME_GEAR} ${IMAGE} \ -c "cargo install cargo-binutils && \ - rustup component add llvm-tools-preview && \ - rustup component add --toolchain nightly llvm-tools-preview && \ + rustup component add llvm-tools && \ + rustup component add --toolchain nightly llvm-tools && \ cargo fuzz coverage --release --sanitizer=none main /corpus/main -- \ -rss_limit_mb=8192 -max_len=450000 -len_control=0 && \ cargo cov -- show target/x86_64-unknown-linux-gnu/coverage/x86_64-unknown-linux-gnu/release/main \ diff --git a/gbuiltins/proxy/Cargo.toml b/gbuiltins/proxy/Cargo.toml index 4912223625b..8631097c686 100644 --- a/gbuiltins/proxy/Cargo.toml +++ b/gbuiltins/proxy/Cargo.toml @@ -9,6 +9,6 @@ homepage.workspace = true repository.workspace = true [dependencies] -gprimitives.workspace = true -derive_more.workspace = true +parity-scale-codec = { workspace = true, features = ["derive"] } scale-info = { workspace = true, features = ["derive"] } +gprimitives = { workspace = true, features = ["codec"] } diff --git a/gbuiltins/proxy/src/lib.rs b/gbuiltins/proxy/src/lib.rs index 49eecad834b..53147fc556a 100644 --- a/gbuiltins/proxy/src/lib.rs +++ b/gbuiltins/proxy/src/lib.rs @@ -21,14 +21,13 @@ #![no_std] use gprimitives::ActorId; -use scale_info::scale::{self, Decode, Encode}; +use parity_scale_codec::{Decode, Encode}; /// Request that can be handled by the proxy builtin. /// /// Currently all proxies aren't required to send announcement, /// i.e. no delays for the delegate actions. #[derive(Debug, Clone, Copy, Eq, PartialEq, Encode, Decode)] -#[codec(crate = scale)] pub enum Request { /// Add proxy request. /// @@ -53,7 +52,6 @@ pub enum Request { /// /// The mirror enum for the one defined in vara-runtime crate. #[derive(Debug, Clone, Copy, Eq, PartialEq, Encode, Decode)] -#[codec(crate = scale)] pub enum ProxyType { Any, NonTransfer, diff --git a/scripts/pin-rust-nightly.sh b/scripts/pin-rust-nightly.sh index 5919095ee2d..22b195e24fe 100755 --- a/scripts/pin-rust-nightly.sh +++ b/scripts/pin-rust-nightly.sh @@ -12,7 +12,7 @@ fi pin_date=$1 os_name="$(uname)" suffix=$(rustc -Vv | grep "host: " | sed "s/^host: \(.*\)$/\1/") -rustup toolchain install nightly-$pin_date --component llvm-tools-preview +rustup toolchain install nightly-$pin_date --component llvm-tools --component rust-src rustup target add wasm32-unknown-unknown --toolchain nightly-$pin_date rm -rf ~/.rustup/toolchains/nightly-$suffix ln -s ~/.rustup/toolchains/nightly-$pin_date-$suffix ~/.rustup/toolchains/nightly-$suffix diff --git a/utils/crates-io/Cargo.toml b/utils/crates-io/Cargo.toml index 002bb04b0d4..4299d6f6f16 100644 --- a/utils/crates-io/Cargo.toml +++ b/utils/crates-io/Cargo.toml @@ -9,7 +9,7 @@ cargo-http-registry.workspace = true cargo_metadata.workspace = true clap = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] } -reqwest = { workspace = true, features = ["blocking", "json", "default-tls"] } +reqwest = { workspace = true, features = ["json", "default-tls"] } tempfile.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } toml_edit.workspace = true diff --git a/utils/crates-io/src/handler.rs b/utils/crates-io/src/handler.rs index bf3c23ff203..9caab9acec0 100644 --- a/utils/crates-io/src/handler.rs +++ b/utils/crates-io/src/handler.rs @@ -38,8 +38,8 @@ pub fn crates_io_name(pkg: &str) -> &str { } /// Patch specified manifest by provided name. -pub fn patch(pkg: &Package) -> Result { - let mut manifest = Manifest::new(pkg)?; +pub fn patch(pkg: &Package, is_published: bool) -> Result { + let mut manifest = Manifest::new(pkg, is_published)?; let doc = &mut manifest.mutable_manifest; match manifest.name.as_str() { diff --git a/utils/crates-io/src/lib.rs b/utils/crates-io/src/lib.rs index ea24b531bee..5d7feba633b 100644 --- a/utils/crates-io/src/lib.rs +++ b/utils/crates-io/src/lib.rs @@ -29,11 +29,20 @@ pub use self::{ manifest::{LockFile, Manifest, Workspace}, publisher::Publisher, simulator::Simulator, - version::verify, + version::{verify, verify_owners, PackageStatus}, }; use anyhow::Result; use std::process::{Command, ExitStatus}; +/// Username that owns crates. +pub const USER_OWNER: &str = "breathx"; + +/// Team that owns crates. +pub const TEAM_OWNER: &str = "github:gear-tech:dev"; + +/// Expected owners of crates. +pub const EXPECTED_OWNERS: [&str; 2] = [USER_OWNER, TEAM_OWNER]; + /// Required Packages without local dependencies. pub const SAFE_DEPENDENCIES: &[&str] = &[ "actor-system-error", @@ -60,6 +69,7 @@ pub const SAFE_DEPENDENCIES: &[&str] = &[ pub const STACKED_DEPENDENCIES: &[&str] = &[ "gprimitives", "gbuiltin-eth-bridge", + "gbuiltin-proxy", "gbuiltin-staking", "gstd-codegen", "gcore", @@ -106,14 +116,6 @@ pub const PACKAGE_ALIAS: [(&str, &str); 2] = [ /// Name for temporary cargo registry. pub const CARGO_REGISTRY_NAME: &str = "cargo-http-registry"; -/// Check the input package -pub fn check(manifest: &str) -> Result { - Command::new("cargo") - .args(["+stable", "check", "--manifest-path", manifest]) - .status() - .map_err(Into::into) -} - /// Test the input package pub fn test(package: &str, test: &str) -> Result { Command::new("cargo") @@ -135,3 +137,11 @@ pub fn publish(manifest: &str) -> Result { .status() .map_err(Into::into) } + +/// Add owner to the input package +pub fn add_owner(package: &str, owner: &str) -> Result { + Command::new("cargo") + .args(["+stable", "owner", "--add", owner, package]) + .status() + .map_err(Into::into) +} diff --git a/utils/crates-io/src/main.rs b/utils/crates-io/src/main.rs index 35e83c5d8c6..16efce381e6 100644 --- a/utils/crates-io/src/main.rs +++ b/utils/crates-io/src/main.rs @@ -62,14 +62,16 @@ async fn main() -> Result<()> { simulate, registry_path, } => { - let publisher = - Publisher::with_simulation(simulate, registry_path)?.build(true, version)?; + let publisher = Publisher::with_simulation(simulate, registry_path)? + .build(true, version) + .await? + .check()?; let result = publisher.publish(); publisher.restore()?; result } Command::Build => { - Publisher::new()?.build(false, None)?; + Publisher::new()?.build(false, None).await?; Ok(()) } } diff --git a/utils/crates-io/src/manifest.rs b/utils/crates-io/src/manifest.rs index 3e72febdbff..31faae0bb48 100644 --- a/utils/crates-io/src/manifest.rs +++ b/utils/crates-io/src/manifest.rs @@ -52,6 +52,7 @@ impl Workspace { original_manifest, mutable_manifest, path, + is_published: true, }, lock_file: LockFile { content, @@ -187,12 +188,14 @@ pub struct Manifest { pub mutable_manifest: DocumentMut, /// Path of the manifest pub path: PathBuf, + /// Whether the crate is published + pub is_published: bool, } impl Manifest { /// Complete the manifest of the specified crate from /// the workspace manifest - pub fn new(pkg: &Package) -> Result { + pub fn new(pkg: &Package, is_published: bool) -> Result { let original_manifest: DocumentMut = fs::read_to_string(&pkg.manifest_path)?.parse()?; let mut mutable_manifest = original_manifest.clone(); @@ -206,6 +209,7 @@ impl Manifest { original_manifest, mutable_manifest, path: pkg.manifest_path.clone().into(), + is_published, }) } diff --git a/utils/crates-io/src/publisher.rs b/utils/crates-io/src/publisher.rs index bc9e349e6d2..440d6b910c1 100644 --- a/utils/crates-io/src/publisher.rs +++ b/utils/crates-io/src/publisher.rs @@ -19,7 +19,8 @@ //! Packages publisher use crate::{ - handler, Manifest, Simulator, Workspace, PACKAGES, SAFE_DEPENDENCIES, STACKED_DEPENDENCIES, + handler, Manifest, PackageStatus, Simulator, Workspace, PACKAGES, SAFE_DEPENDENCIES, + STACKED_DEPENDENCIES, TEAM_OWNER, }; use anyhow::{bail, Result}; use cargo_metadata::{Metadata, MetadataCommand}; @@ -58,7 +59,7 @@ impl Publisher { /// 1. Replace git dependencies to crates-io dependencies. /// 2. Rename version of all local packages /// 3. Patch dependencies if needed - pub fn build(mut self, verify: bool, version: Option) -> Result { + pub async fn build(mut self, verify: bool, version: Option) -> Result { let mut workspace = Workspace::lookup(version)?; let version = workspace.version()?; @@ -68,12 +69,22 @@ impl Publisher { continue; }; - if verify && crate::verify(name, &version, self.simulator.as_ref())? { + let mut is_published = false; + + if verify { + match crate::verify_owners(name).await? { + PackageStatus::InvalidOwners => bail!("Package {name} has invalid owners!"), + PackageStatus::NotPublished => is_published = false, + PackageStatus::ValidOwners => is_published = true, + } + } + + if verify && crate::verify(name, &version, self.simulator.as_ref()).await? { println!("Package {name}@{version} already published!"); continue; } - self.graph.push(handler::patch(pkg)?); + self.graph.push(handler::patch(pkg, is_published)?); } workspace.complete(self.index.clone(), self.simulator.is_some())?; @@ -121,26 +132,7 @@ impl Publisher { } /// Check the to-be-published packages - /// - /// TODO: Complete the check process (#3565) - pub fn check(&self) -> Result<()> { - let mut failed = Vec::new(); - for Manifest { path, name, .. } in self.graph.iter() { - if !PACKAGES.contains(&name.as_str()) { - continue; - } - - println!("Checking {path:?}"); - let status = crate::check(&path.to_string_lossy())?; - if !status.success() { - failed.push(path); - } - } - - if !failed.is_empty() { - bail!("Packages {failed:?} failed to pass the check ..."); - } - + pub fn check(self) -> Result { // Post tests for gtest and gclient for (pkg, test) in [ ("demo-syscall-error", "program_can_be_initialized"), @@ -151,17 +143,30 @@ impl Publisher { } } - Ok(()) + Ok(self) } /// Publish packages pub fn publish(&self) -> Result<()> { - for Manifest { path, .. } in self.graph.iter() { + for Manifest { + name, + path, + is_published, + .. + } in self.graph.iter() + { println!("Publishing {path:?}"); let status = crate::publish(&path.to_string_lossy())?; if !status.success() { bail!("Failed to publish package {path:?} ..."); } + + if self.simulator.is_none() && !is_published { + let status = crate::add_owner(name, TEAM_OWNER)?; + if !status.success() { + bail!("Failed to add owner to package {name} ..."); + } + } } Ok(()) diff --git a/utils/crates-io/src/version.rs b/utils/crates-io/src/version.rs index 411638fe6aa..90845fabc2b 100644 --- a/utils/crates-io/src/version.rs +++ b/utils/crates-io/src/version.rs @@ -18,26 +18,27 @@ //! Crate verifier -use crate::{handler, Simulator}; +use crate::{handler, Simulator, EXPECTED_OWNERS}; use anyhow::{anyhow, Result}; +use reqwest::{Client, StatusCode}; use serde::Deserialize; use std::process::Command; #[derive(Debug, Deserialize)] -struct Resp { - pub versions: Vec, +struct VersionsResponse { + versions: Vec, } #[derive(Debug, Deserialize)] struct Version { - pub num: String, + num: String, } /// Verify if the package has already been published. -pub fn verify(name: &str, version: &str, simulator: Option<&Simulator>) -> Result { +pub async fn verify(name: &str, version: &str, simulator: Option<&Simulator>) -> Result { println!("Verifying {name}@{version} ..."); - let client = reqwest::blocking::Client::builder() + let client = Client::builder() .user_agent("gear-crates-io-manager") .build()?; @@ -48,26 +49,84 @@ pub fn verify(name: &str, version: &str, simulator: Option<&Simulator>) -> Resul simulator.addr(), handler::crates_io_name(name) )) - .send()? + .send() + .await? .error_for_status() .is_ok() { return Ok(true); } - } else if let Ok(resp) = client + } else if let Ok(response) = client .get(format!( "https://crates.io/api/v1/crates/{}/versions", handler::crates_io_name(name) )) - .send()? - .json::() + .send() + .await? + .json::() + .await { - return Ok(resp.versions.into_iter().any(|v| v.num == version)); + return Ok(response.versions.into_iter().any(|v| v.num == version)); } Ok(false) } +#[derive(Debug, Deserialize)] +struct OwnersResponse { + users: Vec, +} + +#[derive(Debug, Deserialize)] +struct User { + login: String, +} + +/// Package status. +#[derive(Debug, PartialEq)] +pub enum PackageStatus { + /// Package has not been published. + NotPublished, + /// Package has invalid owners. + InvalidOwners, + /// Package has valid owners. + ValidOwners, +} + +/// Verify if the package has valid owners. +pub async fn verify_owners(name: &str) -> Result { + println!("Verifying {name} owners ..."); + + let client = Client::builder() + .user_agent("gear-crates-io-manager") + .build()?; + + let response = client + .get(format!( + "https://crates.io/api/v1/crates/{}/owners", + handler::crates_io_name(name) + )) + .send() + .await?; + + if response.status() == StatusCode::NOT_FOUND { + return Ok(PackageStatus::NotPublished); + } + + let response = response.json::().await?; + let package_status = if response.users.len() == EXPECTED_OWNERS.len() + && EXPECTED_OWNERS + .into_iter() + .all(|owner| response.users.iter().any(|u| u.login == owner)) + { + PackageStatus::ValidOwners + } else { + PackageStatus::InvalidOwners + }; + + Ok(package_status) +} + /// Get the short hash of the current commit. pub fn hash() -> Result { Ok(Command::new("git") diff --git a/utils/runtime-fuzzer/README.md b/utils/runtime-fuzzer/README.md index 114d5e270ef..65a151847e0 100644 --- a/utils/runtime-fuzzer/README.md +++ b/utils/runtime-fuzzer/README.md @@ -6,7 +6,7 @@ This is an internal product that is used to find panics that may occur in our ru Pre-requirements: -- llvm-tools: `rustup component add llvm-tools-preview` +- llvm-tools: `rustup component add llvm-tools` - [cargo-fuzz](https://github.com/rust-fuzz/cargo-fuzz): `cargo install cargo-fuzz` - [cargo-binutils](https://github.com/rust-embedded/cargo-binutils): `cargo install cargo-binutils` - [rustfilt](https://github.com/luser/rustfilt): `cargo install rustfilt` diff --git a/utils/wasm-builder/test-program/src/lib.rs b/utils/wasm-builder/test-program/src/lib.rs index b5d2b50cec5..c8cba137922 100644 --- a/utils/wasm-builder/test-program/src/lib.rs +++ b/utils/wasm-builder/test-program/src/lib.rs @@ -27,12 +27,13 @@ extern "C" fn init() { mod gtest_tests { extern crate std; - use gtest::{Log, Program, System}; + use gtest::{constants::UNITS, Log, Program, System}; #[test] fn init_self() { let system = System::new(); system.init_logger(); + system.mint_to(123, UNITS * 100); let this_program = Program::current(&system); diff --git a/utils/wasm-builder/tests/smoke.rs b/utils/wasm-builder/tests/smoke.rs index 8c3eeb44d56..7118bdd5ae1 100644 --- a/utils/wasm-builder/tests/smoke.rs +++ b/utils/wasm-builder/tests/smoke.rs @@ -59,6 +59,8 @@ fn install_stable_toolchain() { .arg("stable") .arg("--component") .arg("llvm-tools") + .arg("--component") + .arg("rust-src") .arg("--target") .arg("wasm32-unknown-unknown") .status() @@ -72,8 +74,7 @@ fn install_stable_toolchain() { fn test_debug() { install_stable_toolchain(); - //TODO: uncomment after solving issue #3915 - //CargoRunner::new().args(["test"]).run(); + CargoRunner::new().args(["test"]).run(); CargoRunner::stable().args(["test"]).run(); } @@ -91,8 +92,7 @@ fn build_debug() { fn test_release() { install_stable_toolchain(); - //TODO: uncomment after solving issue #3915 - //CargoRunner::new().args(["test", "--release"]).run(); + CargoRunner::new().args(["test", "--release"]).run(); CargoRunner::stable().args(["test", "--release"]).run(); } @@ -164,7 +164,7 @@ fn build_release_for_target_deny_duplicate_crate() { .0; cmd.arg("--color=always"); cmd.arg("--manifest-path=test-program/Cargo.toml"); - cmd.arg("--config=env.GEAR_WASM_BUILDER_DENIED_DUPLICATE_CRATES=\'syn\'"); + cmd.env("__GEAR_WASM_BUILDER_DENIED_DUPLICATE_CRATES", "syn"); let status = cmd.status().expect("cargo run error"); assert!(!status.success())