From 65dde111f83a692cd96c0db9f2cddab81760877d Mon Sep 17 00:00:00 2001 From: Tadeo Hepperle <62739623+tadeohepperle@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:33:01 +0100 Subject: [PATCH] CLI tool: recursive derives and attributes (#1379) * restructure cli commands * config: Add `SkipCheckIfFeeless` signed extension (#1264) * config: Add `SkipCheckIfFeeless` signed extension Signed-off-by: Alexandru Vasile * config: Add extra extension to the default params Signed-off-by: Alexandru Vasile * examples: Adjust signed extension example Signed-off-by: Alexandru Vasile * config: Extend SkipCheckIfFeeless with inner signed extension Signed-off-by: Alexandru Vasile * config: Configure SkipCheck with inner signed extension params Signed-off-by: Alexandru Vasile * config: Implement Deafult for SkipCheckIfFeelessParams with Option Signed-off-by: Alexandru Vasile * examples: Fix example with proper extension Signed-off-by: Alexandru Vasile * config: Extend ::AssetId with EncodeAsType and Clone Signed-off-by: Alexandru Vasile * config: Add SkipCheck with AssetTx Signed-off-by: Alexandru Vasile * config: Encode as type from metadata the inner signed extensions Signed-off-by: Alexandru Vasile * Adjust examples Signed-off-by: Alexandru Vasile * blocks: Use `SkipCheckIfFeeless` for decoding the tip of extensions Signed-off-by: Alexandru Vasile * config: Decode `SkipCheckIfFeeless` with `Self` Signed-off-by: Alexandru Vasile * tests: Adjust testing Signed-off-by: Alexandru Vasile * config: Descriptive errors for building `SkipCheckIfFeeless` Signed-off-by: Alexandru Vasile * config: Add docs for extra error types Signed-off-by: Alexandru Vasile * subxt: Add extra derives to signed extensions Signed-off-by: Alexandru Vasile * config: Use `Default::default` to simplify type init Signed-off-by: Alexandru Vasile --------- Signed-off-by: Alexandru Vasile * subxt: Replace removed lint (#1270) Signed-off-by: Alexandru Vasile * lightclient: Add support for multi-chain usecase (#1238) * lightclient: Make `smoldot::chainID` part of the RPC requests Signed-off-by: Alexandru Vasile * lightclient: Make `BackgroundTask` generic over `PlatformRef` and chain Signed-off-by: Alexandru Vasile * lightclient: Construct from raw smoldot and target different chains Signed-off-by: Alexandru Vasile * testing: Update cargo lock for wasm tests Signed-off-by: Alexandru Vasile * lightclient: Reuse `new_from_client` method and removed unused imports Signed-off-by: Alexandru Vasile * lightclient: Reexport smoldot client and RPC objects used in pub interface Signed-off-by: Alexandru Vasile * lightclient: Adjust `new_from_client` interface Signed-off-by: Alexandru Vasile * lightclient: Extend background to poll over multiple RPC objects Signed-off-by: Alexandru Vasile * subxt: Build light client from raw and target different chains Signed-off-by: Alexandru Vasile * artifacts: Add demo chain specs Signed-off-by: Alexandru Vasile * artifacts: Move artifacts to dedicated folder Signed-off-by: Alexandru Vasile * lightclient: Use SelectAll to drive all streams Signed-off-by: Alexandru Vasile * lightclient: Fetch initial data from the target chain Signed-off-by: Alexandru Vasile * lightclient: Reexport other smoldot objects Signed-off-by: Alexandru Vasile * subxt: Target chain with potentially different config Signed-off-by: Alexandru Vasile * subxt/rpc: Log chainID for debugging Signed-off-by: Alexandru Vasile * subxt/examples: Add smoldot client with parachain example Signed-off-by: Alexandru Vasile * lightclient: Propagate chain ID together with rpc responses object Signed-off-by: Alexandru Vasile * lightclient: Multiplex responses by request ID and chain ID Signed-off-by: Alexandru Vasile * subxt: Add raw light client builder Signed-off-by: Alexandru Vasile * subxt: Add cargo feature flag for parachains example Signed-off-by: Alexandru Vasile * lightclient: Derive default for internal structure Signed-off-by: Alexandru Vasile * lightclient: Guard reexports by std feature flag Signed-off-by: Alexandru Vasile * Update subxt/src/client/light_client/mod.rs Co-authored-by: James Wilson * lightclient: Update the builder pattern and chain targetting Signed-off-by: Alexandru Vasile * lightclient: Fix documentation Signed-off-by: Alexandru Vasile * Provide more insightful docs wrt native/wasm panics Signed-off-by: Alexandru Vasile * examples: Adjust comment location Signed-off-by: Alexandru Vasile * lightclient: Refactor UniqueChainId into the background task Signed-off-by: Alexandru Vasile * Update lightclient/src/background.rs Co-authored-by: Niklas Adolfsson * Update subxt/src/client/light_client/builder.rs Co-authored-by: James Wilson * lightclient: Update docs wrt panics Signed-off-by: Alexandru Vasile * subxt: Update docs wrt to smoldot instance -> client Signed-off-by: Alexandru Vasile * lightclient: Use IntoIter instead of Iterator Signed-off-by: Alexandru Vasile * subxt: Adjsut docs wrt [`Self::new_from_client`] Signed-off-by: Alexandru Vasile * subxt: Remove RawRpc from LightClient in favor of chainID Signed-off-by: Alexandru Vasile * lightclient: Reexport everything under smoldot module Signed-off-by: Alexandru Vasile * artifacts: Use stateRootHash instead of genesis.raw Signed-off-by: Alexandru Vasile --------- Signed-off-by: Alexandru Vasile Co-authored-by: James Wilson Co-authored-by: Niklas Adolfsson * Bump futures from 0.3.28 to 0.3.29 (#1272) Bumps [futures](https://github.com/rust-lang/futures-rs) from 0.3.28 to 0.3.29. - [Release notes](https://github.com/rust-lang/futures-rs/releases) - [Changelog](https://github.com/rust-lang/futures-rs/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/futures-rs/compare/0.3.28...0.3.29) --- updated-dependencies: - dependency-name: futures dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump zeroize from 1.6.0 to 1.7.0 (#1274) Bumps [zeroize](https://github.com/RustCrypto/utils) from 1.6.0 to 1.7.0. - [Commits](https://github.com/RustCrypto/utils/commits) --- updated-dependencies: - dependency-name: zeroize dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tracing-subscriber from 0.3.17 to 0.3.18 (#1275) Bumps [tracing-subscriber](https://github.com/tokio-rs/tracing) from 0.3.17 to 0.3.18. - [Release notes](https://github.com/tokio-rs/tracing/releases) - [Commits](https://github.com/tokio-rs/tracing/compare/tracing-subscriber-0.3.17...tracing-subscriber-0.3.18) --- updated-dependencies: - dependency-name: tracing-subscriber dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump tracing-subscriber from 0.3.17 to 0.3.18 (#1275) Bumps [tracing-subscriber](https://github.com/tokio-rs/tracing) from 0.3.17 to 0.3.18. - [Release notes](https://github.com/tokio-rs/tracing/releases) - [Commits](https://github.com/tokio-rs/tracing/compare/tracing-subscriber-0.3.17...tracing-subscriber-0.3.18) --- updated-dependencies: - dependency-name: tracing-subscriber dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump getrandom from 0.2.10 to 0.2.11 (#1273) Bumps [getrandom](https://github.com/rust-random/getrandom) from 0.2.10 to 0.2.11. - [Changelog](https://github.com/rust-random/getrandom/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-random/getrandom/compare/v0.2.10...v0.2.11) --- updated-dependencies: - dependency-name: getrandom dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * impl RpcClientT for Arc and Box (#1277) * impl RpcClientT for Arc * fix grumbles: impl for Box and Arc * grumbles: move RpcClientT impls * first iteration of using scale_typegen * introduce indoc for formatting * calls, constants and home are cleaner now * added event subcommand * show runtime apis working * add better code formatting * fix style * adjust tests, use owo_colorize to not add extra dependency * fmt * adjust docs * move scale-typegen-description dependency to workspace * improve `substrate-compat` (#1265) * improve `substrate-compat` * From => Into --------- Co-authored-by: James Wilson * Bump proc-macro2 from 1.0.69 to 1.0.70 (#1292) Bumps [proc-macro2](https://github.com/dtolnay/proc-macro2) from 1.0.69 to 1.0.70. - [Release notes](https://github.com/dtolnay/proc-macro2/releases) - [Commits](https://github.com/dtolnay/proc-macro2/compare/1.0.69...1.0.70) --- updated-dependencies: - dependency-name: proc-macro2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump serde from 1.0.192 to 1.0.193 (#1291) Bumps [serde](https://github.com/serde-rs/serde) from 1.0.192 to 1.0.193. - [Release notes](https://github.com/serde-rs/serde/releases) - [Commits](https://github.com/serde-rs/serde/compare/v1.0.192...v1.0.193) --- updated-dependencies: - dependency-name: serde dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * lightclient: Fix wasm socket closure called after being dropped (#1289) * lightclient: Close wasm socket while dropping from connecting state Signed-off-by: Alexandru Vasile * lightclient: Construct one time only closures Signed-off-by: Alexandru Vasile * testing: Enable console logs for lightclient WASM testing Signed-off-by: Alexandru Vasile * lightclient: Separate wakes and check connectivity on poll_read Signed-off-by: Alexandru Vasile * lightclient: Close the socket depending on internal state Signed-off-by: Alexandru Vasile * Revert "lightclient: Separate wakes and check connectivity on poll_read" This reverts commit 866094001d4c0b119a80ed681a74b323f74eae1b. * lightclient: Return pending if socket is opening from poll_read Signed-off-by: Alexandru Vasile * lightclient: Close the socket on `poll_close` Signed-off-by: Alexandru Vasile * lightclient: Reset closures on Drop to avoid recursive invokation Signed-off-by: Alexandru Vasile * lightclient: Close the socket if not already closing Signed-off-by: Alexandru Vasile --------- Signed-off-by: Alexandru Vasile * workflows: Install rustup component for building substrate (#1295) Signed-off-by: Alexandru Vasile * cli: Command to fetch chainSpec and optimise its size (#1278) * cli: Add chainSpec command Signed-off-by: Alexandru Vasile * cli/chainSpec: Move to dedicated module Signed-off-by: Alexandru Vasile * cli: Compute the state root hash Signed-off-by: Alexandru Vasile * cli: Remove code substitutes Signed-off-by: Alexandru Vasile * artifacts: Update polkadot.json Signed-off-by: Alexandru Vasile * scripts: Generate the chain spec Signed-off-by: Alexandru Vasile * cli: Remove testing artifacts Signed-off-by: Alexandru Vasile * cli: Fix clippy Signed-off-by: Alexandru Vasile * cli: Apply rustfmt Signed-off-by: Alexandru Vasile * cli: Introduce feature flag for smoldot dependency Signed-off-by: Alexandru Vasile * cli: Rename chain-spec to chain-spec-pruning Signed-off-by: Alexandru Vasile * scripts: Update chain-spec command Signed-off-by: Alexandru Vasile --------- Signed-off-by: Alexandru Vasile * update to new scale-typegen interfaces * use released version of scale-typegen * Merge branch 'master' into tadeohepperle/cli-support-runtime-apis * remove unused debug file * resolve merge errors * adjustments * constants file adjustment * method renaming * fix issue with encoding runtime api params * Add logging to submit_transaction and unstable driver, and ensure unpin evs complete * panic if None returned from subscription too, also with stats * change panic to Err just to be on the safe side * clippy * make long tests run only after clippy + fmt pass * megre in light client test change pr * chore(subxt/src): typo fix (#1370) * rpcmethods * followstr * mod and else * Weekly Cronjob fetching artifacts and generating polkadot.rs file. (#1352) * github CI action cronjob * add commit message * fix the CI yml files * binary crate for CI script with substrate-runner * update the CI script * correct the artifacts script * remove bash script * lightclient(fix): Ensure lightclient chainSpec is at least one block old (#1372) * testing(fix): Ensure lightclient chainSpec is at least one block old Signed-off-by: Alexandru Vasile * Revert "testing(fix): Ensure lightclient chainSpec is at least one block old" This reverts commit 0eafcb2ca59d1f1cd2cef86b770f5a0401cce59f. * lightclient(fix): Ensure lightclient chainSpec is at least one block old Signed-off-by: Alexandru Vasile * lightclient: Link smoldot issue Signed-off-by: Alexandru Vasile * subxt: Use tokio under lightclient feature flag Signed-off-by: Alexandru Vasile * lightclient: Do not sleep on errors to fetch the chainSpec Signed-off-by: Alexandru Vasile * artifacts: Remove test file Signed-off-by: Alexandru Vasile * lightclient: Subscribe to two finalized blocks Signed-off-by: Alexandru Vasile * subxt: Revert cargo toml Signed-off-by: Alexandru Vasile --------- Signed-off-by: Alexandru Vasile * ci: Reduce the light client timeout to 15 minutes (#1373) * ci: Reduce the light client timpeut to 15 seconds Signed-off-by: Alexandru Vasile * ci: Use ubuntu-latest for light-client tests Signed-off-by: Alexandru Vasile --------- Signed-off-by: Alexandru Vasile * actually only wait for machete+fmt, clippy can be much slower * update CI file from Alex PR * resolve clippy err * Try a few RPC nodes in case one of them is not working * fix submit_transaction debug logging of message * Improve Signed Extension and Block Decoding Examples/Book (#1357) * asset hub example and book adjustment * formatting * recursive derives * polkadot monitor example and book adjustments * formatting * adjust docs and examples, add dynamic example * james suggestions * fmt * chore(subxt/src): typo fix (#1370) * rpcmethods * followstr * mod and else * Weekly Cronjob fetching artifacts and generating polkadot.rs file. (#1352) * github CI action cronjob * add commit message * fix the CI yml files * binary crate for CI script with substrate-runner * update the CI script * correct the artifacts script * remove bash script --------- Co-authored-by: James Wilson Co-authored-by: Pan chao <152830401+Pan-chao@users.noreply.github.com> * fix formatting of returned sections * add recursive derive and attribute options in the cli * format tuples uncaught * add tests and rename type map parser * make parsing more strict --------- Signed-off-by: Alexandru Vasile Signed-off-by: dependabot[bot] Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com> Co-authored-by: James Wilson Co-authored-by: Niklas Adolfsson Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: yjh Co-authored-by: Pan chao <152830401+Pan-chao@users.noreply.github.com> --- cli/src/commands/codegen.rs | 112 ++++++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 24 deletions(-) diff --git a/cli/src/commands/codegen.rs b/cli/src/commands/codegen.rs index 1e5d0d7b78..ddc3c59f2f 100644 --- a/cli/src/commands/codegen.rs +++ b/cli/src/commands/codegen.rs @@ -25,14 +25,16 @@ pub struct Opts { attributes: Vec, /// Additional derives for a given type. /// - /// Example `--derive-for-type my_module::my_type=serde::Serialize`. + /// Example 1: `--derive-for-type my_module::my_type=serde::Serialize`. + /// Example 2: `--derive-for-type my_module::my_type=serde::Serialize,recursive`. #[clap(long = "derive-for-type", value_parser = derive_for_type_parser)] - derives_for_type: Vec<(String, String)>, + derives_for_type: Vec, /// Additional attributes for a given type. /// - /// Example `--attributes-for-type my_module::my_type=#[allow(clippy::all)]`. + /// Example 1: `--attributes-for-type my_module::my_type=#[allow(clippy::all)]`. + /// Example 2: `--attributes-for-type my_module::my_type=#[allow(clippy::all)],recursive`. #[clap(long = "attributes-for-type", value_parser = attributes_for_type_parser)] - attributes_for_type: Vec<(String, String)>, + attributes_for_type: Vec, /// Substitute a type for another. /// /// Example `--substitute-type sp_runtime::MultiAddress=subxt::utils::Static<::sp_runtime::MultiAddress>` @@ -67,20 +69,63 @@ pub struct Opts { allow_insecure: bool, } -fn derive_for_type_parser(src: &str) -> Result<(String, String), String> { - let (ty, derive) = src - .split_once('=') - .ok_or_else(|| String::from("Invalid pattern for `derive-for-type`. It should be `type=derive`, like `my_type=serde::Serialize`"))?; +#[derive(Debug, Clone)] +struct DeriveForType { + type_path: String, + trait_path: String, + recursive: bool, +} - Ok((ty.to_string(), derive.to_string())) +#[derive(Debug, Clone)] +struct AttributeForType { + type_path: String, + attribute: String, + recursive: bool, } -fn attributes_for_type_parser(src: &str) -> Result<(String, String), String> { - let (ty, attribute) = src - .split_once('=') - .ok_or_else(|| String::from("Invalid pattern for `attribute-type`. It should be `type=attribute`, like `my_type=serde::#[allow(clippy::all)]`"))?; +fn derive_for_type_parser(src: &str) -> Result { + let (type_path, trait_path, recursive) = type_map_parser(src) + .ok_or_else(|| String::from("Invalid pattern for `derive-for-type`. It should be `type=derive` or `type=derive,recursive`, like `my_type=serde::Serialize` or `my_type=serde::Serialize,recursive`"))?; + Ok(DeriveForType { + type_path: type_path.to_string(), + trait_path: trait_path.to_string(), + recursive, + }) +} - Ok((ty.to_string(), attribute.to_string())) +fn attributes_for_type_parser(src: &str) -> Result { + let (type_path, attribute, recursive) = type_map_parser(src) + .ok_or_else(|| String::from("Invalid pattern for `attributes-for-type`. It should be `type=attribute` like `my_type=serde::#[allow(clippy::all)]` or `type=attribute,recursive` like `my_type=serde::#[allow(clippy::all)],recursive`"))?; + Ok(AttributeForType { + type_path: type_path.to_string(), + attribute: attribute.to_string(), + recursive, + }) +} + +/// Parses a `&str` of the form `str1=str2` into `(str1, str2, false)` or `str1=str2,recursive` into `(str1, str2, true)`. +/// +/// A `None` value returned is a parsing error. +fn type_map_parser(src: &str) -> Option<(&str, &str, bool)> { + let (str1, rest) = src.split_once('=')?; + + let mut split_rest = rest.split(','); + let str2 = split_rest + .next() + .expect("split iter always returns at least one element; qed"); + + let mut recursive = false; + for r in split_rest { + match r { + // Note: later we can add other attributes to this match + "recursive" => { + recursive = true; + } + _ => return None, + } + } + + Some((str1, str2, recursive)) } fn substitute_type_parser(src: &str) -> Result<(String, String), String> { @@ -127,8 +172,8 @@ fn codegen( metadata_bytes: &[u8], raw_derives: Vec, raw_attributes: Vec, - derives_for_type: Vec<(String, String)>, - attributes_for_type: Vec<(String, String)>, + derives_for_type: Vec, + attributes_for_type: Vec, substitute_types: Vec<(String, String)>, crate_path: Option, no_docs: bool, @@ -168,13 +213,14 @@ fn codegen( .map_err(|e| eyre!("Cannot parse global derives: {e}"))?; codegen.set_additional_global_derives(global_derives); - for (ty_str, derive) in derives_for_type { - let ty: syn::TypePath = syn::parse_str(&ty_str) + for d in derives_for_type { + let ty_str = &d.type_path; + let ty: syn::TypePath = syn::parse_str(ty_str) .map_err(|e| eyre!("Cannot parse derive for type {ty_str}: {e}"))?; - let derive = syn::parse_str(&derive) + let derive = syn::parse_str(&d.trait_path) .map_err(|e| eyre!("Cannot parse derive for type {ty_str}: {e}"))?; // Note: recursive derives and attributes not supported in the CLI => recursive: false - codegen.add_derives_for_type(ty, std::iter::once(derive), false); + codegen.add_derives_for_type(ty, std::iter::once(derive), d.recursive); } // Configure attribtues: @@ -186,13 +232,14 @@ fn codegen( .map_err(|e| eyre!("Cannot parse global attributes: {e}"))?; codegen.set_additional_global_attributes(universal_attributes); - for (ty_str, attr) in attributes_for_type { - let ty = syn::parse_str(&ty_str) + for a in attributes_for_type { + let ty_str = &a.type_path; + let ty = syn::parse_str(ty_str) .map_err(|e| eyre!("Cannot parse attribute for type {ty_str}: {e}"))?; - let attribute: OuterAttribute = syn::parse_str(&attr) + let attribute: OuterAttribute = syn::parse_str(&a.attribute) .map_err(|e| eyre!("Cannot parse attribute for type {ty_str}: {e}"))?; // Note: recursive derives and attributes not supported in the CLI => recursive: false - codegen.add_attributes_for_type(ty, std::iter::once(attribute.0), false); + codegen.add_attributes_for_type(ty, std::iter::once(attribute.0), a.recursive); } // Insert type substitutions: @@ -213,3 +260,20 @@ fn codegen( writeln!(output, "{code}")?; Ok(()) } + +#[cfg(test)] +mod tests { + use crate::commands::codegen::type_map_parser; + + #[test] + fn parse_types() { + assert_eq!(type_map_parser("Foo"), None); + assert_eq!(type_map_parser("Foo=Bar"), Some(("Foo", "Bar", false))); + assert_eq!( + type_map_parser("Foo=Bar,recursive"), + Some(("Foo", "Bar", true)) + ); + assert_eq!(type_map_parser("Foo=Bar,a"), None); + assert_eq!(type_map_parser("Foo=Bar,a,b,c,recursive"), None); + } +}