Add --build-tonic flag to proto-compiler, sync protobuf script, and generate protobuf for std and no_std mode

ci/sync-protobuf.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+set -eoux pipefail
+
+# syn-protobuf.sh is a bash script to sync the protobuf
+# files using the proto-compiler project. It will check
+# out the protobuf files from the git versions specified in
+# proto/src/prost/COSMOS_SDK_COMMIT and
+# proto/src/prost/COSMOS_IBC_COMMIT. If you want to sync
+# the protobuf files to a newer version, modify the
+# relevant files with the new commit IDs.
+
+# This script should be run from the root directory of ibc-rs
+
+# We can specify where to clone the git repositories
+# for cosmos-sdk and ibc-go. By default they are cloned
+# to /tmp/cosmos-sdk.git and /tmp/ibc-go.git.
+# We can override this to existing directories
+# that already have a clone of the repositories,
+# so that there is no need to clone the entire
+# repositories over and over again every time
+# the script is called.
+
+COSMOS_SDK_GIT=${COSMOS_SDK_GIT:-/tmp/cosmos-sdk.git}
+IBC_GO_GIT=${IBC_GO_GIT:-/tmp/ibc-go.git}
+
+COSMOS_SDK_COMMIT=$(cat proto/src/prost/COSMOS_SDK_COMMIT)
+COSMOS_IBC_COMMIT=$(cat proto/src/prost/COSMOS_IBC_COMMIT)
+
+echo "COSMOS_SDK_COMMIT: $COSMOS_SDK_COMMIT"
+echo "COSMOS_IBC_COMMIT: $COSMOS_IBC_COMMIT"
+
+# Use either --sdk-commit flag for commit ID,
+# or --sdk-tag for git tag. Because we can't modify
+# proto-compiler to have smart detection on that.
+
+if [[ "$COSMOS_SDK_COMMIT" =~ ^[a-zA-Z0-9]{40}$ ]]
+then
+	SDK_COMMIT_OPTION="--sdk-commit"
+else
+	SDK_COMMIT_OPTION="--sdk-tag"
+fi
+
+# If the git directories does not exist, clone them as
+# bare git repositories so that no local modification
+# can be done there.
+
+if [[ ! -e "$COSMOS_SDK_GIT" ]]
+then
+	git clone --mirror https://github.com/cosmos/cosmos-sdk.git "$COSMOS_SDK_GIT"
+fi
+
+if [[ ! -e "$IBC_GO_GIT" ]]
+then
+	git clone --mirror https://github.com/cosmos/ibc-go.git "$IBC_GO_GIT"
+fi
+
+# Update the repositories using git fetch. This is so that
+# we keep local copies of the repositories up to sync first.
+pushd "$COSMOS_SDK_GIT"
+git fetch
+popd
+
+pushd "$IBC_GO_GIT"
+git fetch
+popd
+
+# Create a new temporary directory to check out the
+# actual source files from the bare git repositories.
+# This is so that we do not accidentally use an unclean
+# local copy of the source files to generate the protobuf.
+COSMOS_SDK_DIR=$(mktemp -d /tmp/cosmos-sdk-XXXXXXXX)
+
+pushd "$COSMOS_SDK_DIR"
+git clone "$COSMOS_SDK_GIT" .
+git checkout "$COSMOS_SDK_COMMIT"
+
+# We have to name the commit as a branch because
+# proto-compiler uses the branch name as the commit
+# output. Otherwise it will just output HEAD
+git checkout -b "$COSMOS_SDK_COMMIT"
+popd
+
+IBC_GO_DIR=$(mktemp -d /tmp/ibc-go-XXXXXXXX)
+
+pushd "$IBC_GO_DIR"
+git clone "$IBC_GO_GIT" .
+git checkout "$COSMOS_IBC_COMMIT"
+git checkout -b "$COSMOS_IBC_COMMIT"
+popd
+
+# Remove the existing generated protobuf files
+# so that the newly generated code does not
+# contain removed files.
+
+rm -rf proto/src/prost/std
+rm -rf proto/src/prost/no_std
+
+mkdir -p proto/src/prost/std
+mkdir -p proto/src/prost/no_std
+
+cd proto-compiler
+
+cargo build --locked
+
+# Run the proto-compiler twice,
+# once for std version with --build-tonic set to true
+# and once for no-std version with --build-tonic set to false
+
+cargo run --locked -- compile \
+	--sdk "$COSMOS_SDK_DIR" --ibc "$IBC_GO_DIR" --build-tonic true --out ../proto/src/prost/std
+
+cargo run --locked -- compile \
+	--sdk "$COSMOS_SDK_DIR" --ibc "$IBC_GO_DIR" --build-tonic false --out ../proto/src/prost/no_std
+
+# Remove the temporary checkouts of the repositories
+
+rm -rf "$COSMOS_SDK_DIR"
+rm -rf "$IBC_GO_DIR"

proto-compiler/src/cmd/clone.rs

@@ -173,4 +173,4 @@ fn checkout_tag(repo: &Repository, tag_name: &str) -> Result<(), git2::Error> {
     }
 
     Ok(())
-}
+}

proto-compiler/src/cmd/compile.rs

@@ -23,13 +23,18 @@ pub struct CompileCmd {
     #[argh(option, short = 'o')]
     /// path to output the generated Rust sources into
     out: PathBuf,
+
+    #[argh(option)]
+    /// generate tonic client code
+    build_tonic: bool,
+
 }
 
 impl CompileCmd {
     pub fn run(&self) {
         let tmp_sdk = TempDir::new("ibc-proto-sdk").unwrap();
         Self::output_version(&self.sdk, tmp_sdk.as_ref(), "COSMOS_SDK_COMMIT");
-        Self::compile_sdk_protos(&self.sdk, tmp_sdk.as_ref(), self.ibc.clone());
+        Self::compile_sdk_protos(&self.sdk, tmp_sdk.as_ref(), self.ibc.clone(), self.build_tonic);
 
         match &self.ibc {
             None => {
@@ -39,7 +44,7 @@ impl CompileCmd {
             Some(ibc_path) => {
                 let tmp_ibc = TempDir::new("ibc-proto-ibc-go").unwrap();
                 Self::output_version(ibc_path, tmp_ibc.as_ref(), "COSMOS_IBC_COMMIT");
-                Self::compile_ibc_protos(ibc_path, tmp_ibc.as_ref());
+                Self::compile_ibc_protos(ibc_path, tmp_ibc.as_ref(), self.build_tonic);
 
                 // Merge the generated files into a single directory, taking care not to overwrite anything
                 Self::copy_generated_files(tmp_sdk.as_ref(), Some(tmp_ibc.as_ref()), &self.out);
@@ -56,7 +61,7 @@ impl CompileCmd {
         std::fs::write(path, rev).unwrap();
     }
 
-    fn compile_ibc_protos(ibc_dir: &Path, out_dir: &Path) {
+    fn compile_ibc_protos(ibc_dir: &Path, out_dir: &Path, build_tonic: bool) {
         println!(
             "[info ] Compiling IBC .proto files to Rust into '{}'...",
             out_dir.display()
@@ -102,7 +107,7 @@ impl CompileCmd {
         let includes: Vec<PathBuf> = proto_includes_paths.iter().map(PathBuf::from).collect();
 
         let compilation = tonic_build::configure()
-            .build_client(true)
+            .build_client(build_tonic)
             .build_server(false)
             .format(true)
             .out_dir(out_dir)
@@ -120,7 +125,7 @@ impl CompileCmd {
         }
     }
 
-    fn compile_sdk_protos(sdk_dir: &Path, out_dir: &Path, ibc_dep: Option<PathBuf>) {
+    fn compile_sdk_protos(sdk_dir: &Path, out_dir: &Path, ibc_dep: Option<PathBuf>, build_tonic: bool) {
         println!(
             "[info ] Compiling Cosmos-SDK .proto files to Rust into '{}'...",
             out_dir.display()
@@ -180,7 +185,7 @@ impl CompileCmd {
         let includes: Vec<PathBuf> = proto_includes_paths.iter().map(PathBuf::from).collect();
 
         let compilation = tonic_build::configure()
-            .build_client(true)
+            .build_client(build_tonic)
             .build_server(false)
             .format(true)
             .out_dir(out_dir)

proto/Cargo.toml

@@ -22,11 +22,15 @@ path = "src/lib.rs"
 all-features = true
 
 [dependencies]
-prost = "0.9"
-prost-types = "0.9"
-bytes = "1.1"
-tonic = "0.6"
+prost = { version = "0.9", default-features = false }
+prost-types = { version = "0.9", default-features = false }
+bytes = { version = "1.1", default-features = false }
+tonic = { version = "0.6", optional = true }
 getrandom = { version = "0.2", features = ["js"] }
 
 [dependencies.tendermint-proto]
 version = "=0.23.0"
+
+[features]
+default = ["std"]
+std = ["tonic"]

proto/src/lib.rs

@@ -21,14 +21,27 @@ macro_rules! format {
         ::alloc::format!($( $args )*)
     }
 }
+#[cfg(feature = "std")]
+macro_rules! include_proto {
+    ($path:literal) => {
+        include!(concat!("prost/std/", $path));
+    };
+}
+
+#[cfg(not(feature = "std"))]
+macro_rules! include_proto {
+    ($path:literal) => {
+        include!(concat!("prost/no_std/", $path));
+    };
+}
 
 /// The version (commit hash) of the Cosmos SDK used when generating this library.
 pub const COSMOS_SDK_VERSION: &str = include_str!("prost/COSMOS_SDK_COMMIT");
 
 pub mod cosmos {
     pub mod auth {
         pub mod v1beta1 {
-            include!("prost/cosmos.auth.v1beta1.rs");
+            include_proto!("cosmos.auth.v1beta1.rs");
             /// EthAccount defines an Ethermint account.
             /// TODO: remove when/if a canonical `EthAccount`
             /// lands in the next Cosmos SDK release
@@ -45,23 +58,23 @@ pub mod cosmos {
     }
     pub mod staking {
         pub mod v1beta1 {
-            include!("prost/cosmos.staking.v1beta1.rs");
+            include_proto!("cosmos.staking.v1beta1.rs");
         }
     }
     pub mod base {
         pub mod abci {
             pub mod v1beta1 {
-                include!("prost/cosmos.base.abci.v1beta1.rs");
+                include_proto!("cosmos.base.abci.v1beta1.rs");
             }
         }
         pub mod kv {
             pub mod v1beta1 {
-                include!("prost/cosmos.base.kv.v1beta1.rs");
+                include_proto!("cosmos.base.kv.v1beta1.rs");
             }
         }
         pub mod query {
             pub mod v1beta1 {
-                include!("prost/cosmos.base.query.v1beta1.rs");
+                include_proto!("cosmos.base.query.v1beta1.rs");
             }
 
             pub mod pagination {
@@ -77,48 +90,48 @@ pub mod cosmos {
         }
         pub mod reflection {
             pub mod v1beta1 {
-                include!("prost/cosmos.base.reflection.v1beta1.rs");
+                include_proto!("cosmos.base.reflection.v1beta1.rs");
             }
         }
         pub mod store {
             pub mod v1beta1 {
-                include!("prost/cosmos.base.store.v1beta1.rs");
+                include_proto!("cosmos.base.store.v1beta1.rs");
             }
         }
         pub mod v1beta1 {
-            include!("prost/cosmos.base.v1beta1.rs");
+            include_proto!("cosmos.base.v1beta1.rs");
         }
         pub mod tendermint {
             pub mod v1beta1 {
-                include!("prost/cosmos.base.tendermint.v1beta1.rs");
+                include_proto!("cosmos.base.tendermint.v1beta1.rs");
             }
         }
     }
     pub mod crypto {
         pub mod multisig {
             pub mod v1beta1 {
-                include!("prost/cosmos.crypto.multisig.v1beta1.rs");
+                include_proto!("cosmos.crypto.multisig.v1beta1.rs");
             }
         }
     }
     pub mod tx {
         pub mod signing {
             pub mod v1beta1 {
-                include!("prost/cosmos.tx.signing.v1beta1.rs");
+                include_proto!("cosmos.tx.signing.v1beta1.rs");
             }
         }
         pub mod v1beta1 {
-            include!("prost/cosmos.tx.v1beta1.rs");
+            include_proto!("cosmos.tx.v1beta1.rs");
         }
     }
     pub mod upgrade {
         pub mod v1beta1 {
-            include!("prost/cosmos.upgrade.v1beta1.rs");
+            include_proto!("cosmos.upgrade.v1beta1.rs");
         }
     }
     pub mod gov {
         pub mod v1beta1 {
-            include!("prost/cosmos.gov.v1beta1.rs");
+            include_proto!("cosmos.gov.v1beta1.rs");
         }
     }
 }
@@ -127,59 +140,59 @@ pub mod ibc {
     pub mod apps {
         pub mod transfer {
             pub mod v1 {
-                include!("prost/ibc.applications.transfer.v1.rs");
+                include_proto!("ibc.applications.transfer.v1.rs");
             }
         }
     }
     pub mod core {
         pub mod channel {
             pub mod v1 {
-                include!("prost/ibc.core.channel.v1.rs");
+                include_proto!("ibc.core.channel.v1.rs");
             }
         }
         pub mod client {
             pub mod v1 {
-                include!("prost/ibc.core.client.v1.rs");
+                include_proto!("ibc.core.client.v1.rs");
             }
         }
         pub mod commitment {
             pub mod v1 {
-                include!("prost/ibc.core.commitment.v1.rs");
+                include_proto!("ibc.core.commitment.v1.rs");
             }
         }
         pub mod connection {
             pub mod v1 {
-                include!("prost/ibc.core.connection.v1.rs");
+                include_proto!("ibc.core.connection.v1.rs");
             }
         }
         pub mod types {
             pub mod v1 {
-                include!("prost/ibc.core.types.v1.rs");
+                include_proto!("ibc.core.types.v1.rs");
             }
         }
     }
     pub mod lightclients {
         pub mod localhost {
             pub mod v1 {
-                include!("prost/ibc.lightclients.localhost.v1.rs");
+                include_proto!("ibc.lightclients.localhost.v1.rs");
             }
         }
         pub mod solomachine {
             pub mod v1 {
-                include!("prost/ibc.lightclients.solomachine.v1.rs");
+                include_proto!("ibc.lightclients.solomachine.v1.rs");
             }
         }
         pub mod tendermint {
             pub mod v1 {
-                include!("prost/ibc.lightclients.tendermint.v1.rs");
+                include_proto!("ibc.lightclients.tendermint.v1.rs");
             }
         }
     }
     pub mod mock {
-        include!("prost/ibc.mock.rs");
+        include_proto!("ibc.mock.rs");
     }
 }
 
 pub mod ics23 {
-    include!("prost/ics23.rs");
+    include_proto!("ics23.rs");
 }