From 8a76aff6484816acac575ab39dbbee97c8c95c7b Mon Sep 17 00:00:00 2001 From: dndll Date: Wed, 27 Mar 2024 11:28:32 +0000 Subject: [PATCH] tracing --- Cargo.lock | 107 ++++++++++++++++++++++++++--- Cargo.toml | 26 ++++--- Dockerfile.operator | 5 +- Makefile | 7 +- bin/operator/Cargo.toml | 40 +++++------ bin/operator/src/engine/mod.rs | 41 +++++++---- bin/operator/src/main.rs | 21 +++--- bin/operator/src/rpc.rs | 8 +-- bin/operator/src/succinct/mod.rs | 55 +++++++++------ bin/operator/src/succinct/types.rs | 6 +- bin/operator/tests/succinct.rs | 8 +-- config.toml | 34 +++++++++ crates/primitives/Cargo.toml | 2 +- crates/primitives/src/config.rs | 54 +++++++++++---- default.toml | 14 ---- testnet.toml | 14 ---- 16 files changed, 294 insertions(+), 148 deletions(-) create mode 100644 config.toml delete mode 100644 default.toml delete mode 100644 testnet.toml diff --git a/Cargo.lock b/Cargo.lock index eea58fd..ce912de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -800,6 +800,15 @@ dependencies = [ "rustc_version 0.4.0", ] +[[package]] +name = "atomic" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +dependencies = [ + "bytemuck", +] + [[package]] name = "atty" version = "0.2.14" @@ -1087,6 +1096,12 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" +[[package]] +name = "bytemuck" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" + [[package]] name = "byteorder" version = "1.5.0" @@ -2331,7 +2346,7 @@ dependencies = [ "tokio", "tracing", "walkdir", - "yansi", + "yansi 0.5.1", ] [[package]] @@ -2396,6 +2411,20 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1676f435fc1dadde4d03e43f5d62b259e1ce5f40bd4ffb21db2b42ebe59c1382" +[[package]] +name = "figment" +version = "0.10.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7270677e7067213e04f323b55084586195f18308cd7546cfac9f873344ccceb6" +dependencies = [ + "atomic", + "pear", + "serde", + "toml 0.8.10", + "uncased", + "version_check", +] + [[package]] name = "fixed-hash" version = "0.7.0" @@ -2718,7 +2747,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ "ahash 0.7.8", - "serde", ] [[package]] @@ -3158,6 +3186,12 @@ dependencies = [ "serde", ] +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + [[package]] name = "inout" version = "0.1.3" @@ -3858,7 +3892,7 @@ dependencies = [ "near-light-client-rpc", "near-primitives", "pretty_env_logger", - "rand 0.7.3", + "rand 0.8.5", "serde", "sled", "tokio", @@ -3868,7 +3902,7 @@ dependencies = [ name = "near-light-client-primitives" version = "0.3.0" dependencies = [ - "config", + "figment", "log", "near-primitives-core", "serde", @@ -3890,7 +3924,7 @@ dependencies = [ "near-primitives", "near-primitives-core", "pretty_env_logger", - "rand 0.7.3", + "rand 0.8.5", "serde", "serde_json", "test-utils", @@ -3913,7 +3947,7 @@ dependencies = [ "near-primitives", "near-primitives-core", "pretty_env_logger", - "rand 0.7.3", + "rand 0.8.5", "serde", "serde_json", "tokio", @@ -4121,21 +4155,19 @@ dependencies = [ "actix", "alloy", "anyhow", - "config", + "figment", "futures", - "hashbrown 0.12.3", + "hashbrown 0.14.3", "hex", "http-cache-reqwest", "jsonrpsee", "jsonrpsee-core", - "log", "near-light-client-primitives", "near-light-client-protocol", "near-light-client-rpc", "near-light-clientx", "paste", "pretty_assertions", - "pretty_env_logger", "priority-queue", "reqwest", "reqwest-middleware", @@ -4146,6 +4178,8 @@ dependencies = [ "test-utils", "thiserror", "tokio", + "tracing", + "tracing-subscriber", "uuid 1.7.0", "wiremock", ] @@ -4618,6 +4652,29 @@ dependencies = [ "hmac", ] +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi 1.0.1", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.53", +] + [[package]] name = "pem" version = "1.1.1" @@ -5005,7 +5062,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" dependencies = [ "diff", - "yansi", + "yansi 0.5.1", ] [[package]] @@ -5124,6 +5181,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", + "version_check", + "yansi 1.0.1", +] + [[package]] name = "prometheus" version = "0.13.3" @@ -7241,6 +7311,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicase" version = "2.7.0" @@ -7876,6 +7955,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "zerocopy" version = "0.7.32" diff --git a/Cargo.toml b/Cargo.toml index 2f1ca79..4e27c74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,17 +8,21 @@ members = [ "bin/*", "crates/*", "nearx" ] resolver = "2" [workspace.dependencies] -anyhow = "1.0" -async-trait = "0.1" -config = "0.13" -derive_more = "0.99" -either = { version = "1.9", features = [ "serde" ] } -itertools = "0.12" -log = "0.4" -pretty_assertions = "1.4" -pretty_env_logger = "0.5" -sled = "0.34" # TODO: maybe heavy, use heed instead -thiserror = "1.0" +anyhow = "1.0" +async-trait = "0.1" +config = "0.13" +derive_more = "0.99" +dotenv = "0.15" +either = { version = "1.9", features = [ "serde" ] } +figment = { version = "0.10", features = [ "toml", "env" ] } +itertools = "0.12" +log = "0.4" +pretty_assertions = "1.4" +pretty_env_logger = "0.5" +sled = "0.34" # TODO: maybe heavy, use heed instead +thiserror = "1.0" +tracing = "0.1" +tracing-subscriber = "0.3" # Async axum = "*" diff --git a/Dockerfile.operator b/Dockerfile.operator index 7452abf..e39849c 100644 --- a/Dockerfile.operator +++ b/Dockerfile.operator @@ -32,8 +32,9 @@ RUN cp target/release/nearx-operator /near/near-light-client FROM debian:bookworm-slim RUN apt-get update && apt-get install -y openssl libssl-dev pkg-config ca-certificates && rm -rf /var/lib/apt/lists/* COPY --from=build /near/target/release/nearx-operator /usr/local/bin -COPY --from=build /near/default.toml /var/near-light-client/default.toml +COPY --from=build /near/config.toml /var/near-light-client/config.toml -ENV NEAR_LIGHT_CLIENT_DIR=/var/near-light-client/ +ENV NEAR_LIGHT_CLIENT_DIR=/var/near-light-client +ENV NEAR_LIGHT_CLIENT_CONFIG_FILE=/var/near-light-client/config.toml ENTRYPOINT ["/usr/local/bin/nearx-operator"] diff --git a/Makefile b/Makefile index ea6af16..542355b 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,12 @@ .EXPORT_ALL_VARIABLES: -include .env -TAG_PREFIX?=near -IMAGE_TAG?=0.0.3 +TAG_PREFIX?=us-docker.pkg.dev/pagoda-solutions-dev/rollup-data-availability/ +IMAGE_TAG?=dev docker: - DOCKER_BUILDKIT=1 docker build --progress=plain -t $(TAG_PREFIX)/light-client:$(IMAGE_TAG) . + DOCKER_BUILDKIT=1 docker build --progress=plain -t $(TAG_PREFIX)/light-client:$(IMAGE_TAG) -f Dockerfile.offchain . + docker build -t $(TAG_PREFIX)/nearx-operator:$(IMAGE_TAG) -f Dockerfile.operator . test: cargo test --workspace diff --git a/bin/operator/Cargo.toml b/bin/operator/Cargo.toml index 71f0b49..d93c861 100644 --- a/bin/operator/Cargo.toml +++ b/bin/operator/Cargo.toml @@ -7,26 +7,26 @@ version.workspace = true [lib] [dependencies] -actix = { version = "*", features = [ "macros" ] } -anyhow.workspace = true -config.workspace = true -futures.workspace = true -hashbrown = { version = "*", features = [ "serde" ] } -hex.workspace = true -http-cache-reqwest = "0.13.0" -jsonrpsee = { version = "0.22.1", features = [ "server", "macros" ] } -jsonrpsee-core = "0.22.1" -log.workspace = true -pretty_env_logger.workspace = true -priority-queue = "*" -reqwest-middleware = "*" -reqwest-retry = "*" -reqwest.workspace = true -serde.workspace = true -serde_json.workspace = true -thiserror.workspace = true -tokio.workspace = true -uuid = { version = "1.4.1", features = [ "serde", "v4" ] } +actix = { version = "*", features = [ "macros" ] } +anyhow.workspace = true +figment.workspace = true +futures.workspace = true +hashbrown = { version = "*", features = [ "serde" ] } +hex.workspace = true +http-cache-reqwest = "0.13.0" +jsonrpsee = { version = "0.22.1", features = [ "server", "macros" ] } +jsonrpsee-core = "0.22.1" +priority-queue = "*" +reqwest-middleware = "*" +reqwest-retry = "*" +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio.workspace = true +tracing-subscriber.workspace = true +tracing.workspace = true +uuid = { version = "1.4.1", features = [ "serde", "v4" ] } alloy = { git = "https://github.com/alloy-rs/alloy.git", features = [ "contract", "provider-http" ] } diff --git a/bin/operator/src/engine/mod.rs b/bin/operator/src/engine/mod.rs index f3679cb..75db7f2 100644 --- a/bin/operator/src/engine/mod.rs +++ b/bin/operator/src/engine/mod.rs @@ -4,10 +4,14 @@ use actix::prelude::*; use anyhow::{anyhow, Result}; use futures::FutureExt; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; -use near_light_client_rpc::{prelude::Itertools, TransactionOrReceiptId}; +use near_light_client_rpc::{ + prelude::{info, Itertools}, + TransactionOrReceiptId, +}; use near_light_clientx::config::bps_from_network; use priority_queue::PriorityQueue; use serde::{Deserialize, Serialize}; +use tracing::{debug, error, trace}; pub use types::RegistryInfo; use self::types::{PriorityWeight, TransactionOrReceiptIdNewtype}; @@ -56,7 +60,7 @@ pub struct Engine { impl Engine { pub fn new(config: &super::Config, succinct_client: Arc) -> Self { - log::info!("starting queue manager"); + info!("starting queue manager"); let state = PersistedState::try_from("state.json"); @@ -80,6 +84,7 @@ impl Engine { } } + #[tracing::instrument(skip(self))] fn prove( &mut self, id: Option, @@ -93,7 +98,7 @@ impl Engine { } else { 1 }; - log::debug!("enqueuing {:?} with weight: {weight}", tx); + debug!("enqueuing {:?} with weight: {weight}", tx); self.proving_queue.push(tx.into(), weight); Ok(()) } @@ -164,6 +169,7 @@ impl From for CheckProof { impl Handler for Engine { type Result = ResponseFuture>; + #[tracing::instrument(skip(self))] fn handle(&mut self, _msg: CheckProof, _ctx: &mut Self::Context) -> Self::Result { // // This feels like a succinct client job, keep checking queues // let fut = self.check_proof(msg.0); @@ -172,25 +178,27 @@ impl Handler for Engine { } } -#[derive(Message)] +#[derive(Message, Debug)] #[rtype(result = "()")] pub struct Register(pub RegistryInfo); impl Handler for Engine { type Result = (); + #[tracing::instrument(skip(self))] fn handle(&mut self, msg: Register, _ctx: &mut Self::Context) -> Self::Result { self.registry.insert(msg.0.id, msg.0); } } -#[derive(Message)] +#[derive(Message, Debug)] #[rtype(result = "()")] pub struct Sync; impl Handler for Engine { type Result = AtomicResponse; + #[tracing::instrument(skip(self))] fn handle(&mut self, _msg: Sync, _ctx: &mut Self::Context) -> Self::Result { let client = self.succinct_client.clone(); // Not ssure this needs atomic @@ -198,20 +206,21 @@ impl Handler for Engine { async move { client.sync(true).await.unwrap() } .into_actor(self) .map(move |r, this, _| { - log::debug!("sync requested {:?}", r); + debug!("sync requested {:?}", r); this.request_info.insert(r, None); }), )) } } -#[derive(Message)] +#[derive(Message, Debug)] #[rtype(result = "()")] pub struct Drain; impl Handler for Engine { type Result = AtomicResponse; + #[tracing::instrument(skip(self))] fn handle(&mut self, _msg: Drain, _ctx: &mut Self::Context) -> Self::Result { AtomicResponse::new(if let Some((id, batch)) = self.make_batch() { let client = self.succinct_client.clone(); @@ -221,7 +230,7 @@ impl Handler for Engine { // Once verify proof id is returned, write the result to ourselves .map(move |r, this, _| { this.batches.get_mut(&id).unwrap().replace(r); - log::debug!("batch {id} done, batches: {:?}", this.batches); + debug!("batch {id} done, batches: {:?}", this.batches); }); Box::pin(fut) } else { @@ -230,15 +239,16 @@ impl Handler for Engine { } } -#[derive(Message)] +#[derive(Message, Debug)] #[rtype(result = "()")] pub struct Cleanup; impl Handler for Engine { type Result = AtomicResponse; + #[tracing::instrument(skip(self))] fn handle(&mut self, _msg: Cleanup, _ctx: &mut Self::Context) -> Self::Result { - log::trace!("cleaning up"); + trace!("cleaning up"); let to_check = self .batches @@ -258,7 +268,7 @@ impl Handler for Engine { AtomicResponse::new(Box::pin( async move { let mut futs = vec![]; - log::debug!("checking on {} proofs", to_check.len()); + debug!("checking on {} proofs", to_check.len()); for pid in to_check { futs.push(client.get_proof(pid).map(move |p| (pid, p))); } @@ -268,11 +278,11 @@ impl Handler for Engine { .map(move |r, this, _| { r.into_iter().for_each(|(pid, p)| match p { Ok(p) => { - log::debug!("proof: {:?} status: {:?}", pid, p.status); + debug!("proof: {:?} status: {:?}", pid, p.status); this.request_info.insert(pid, Some(p.status)); } Err(e) => { - log::error!("{:?}", e); + error!("{:?}", e); } }); }), @@ -301,15 +311,16 @@ impl TryFrom<&str> for PersistedState { } } -#[derive(Message)] +#[derive(Message, Debug)] #[rtype(result = "anyhow::Result<()>")] pub struct Persist; impl Handler for Engine { type Result = anyhow::Result<()>; + #[tracing::instrument(skip(self))] fn handle(&mut self, _msg: Persist, _ctx: &mut Self::Context) -> Self::Result { - log::trace!("persisting state"); + trace!("persisting state"); let state = PersistedState { registry: self.registry.clone(), batches: self.batches.clone(), diff --git a/bin/operator/src/main.rs b/bin/operator/src/main.rs index ed04f0e..1798698 100644 --- a/bin/operator/src/main.rs +++ b/bin/operator/src/main.rs @@ -1,26 +1,31 @@ use std::sync::Arc; -use log::LevelFilter; -use nearx_operator::{config::Config, *}; +use nearx_operator::*; +use tracing::{debug, info}; +use tracing_subscriber::{filter::EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt}; #[actix::main] pub async fn main() -> anyhow::Result<()> { - pretty_env_logger::formatted_builder() - .parse_default_env() - .filter_module("hyper", LevelFilter::Info) - .filter_module("reqwest", LevelFilter::Info) + let filter = EnvFilter::from_default_env() + .add_directive("hyper=info".parse()?) + .add_directive("reqwest=info".parse()?); + + tracing_subscriber::registry() + .with(fmt::layer().compact().with_line_number(true)) + .with(filter) .init(); - let config = Config::new(std::env::var("NEAR_LIGHT_CLIENT_DIR").ok().as_deref())?; + let config = config::Config::new(std::env::var("NEAR_LIGHT_CLIENT_DIR").ok().as_deref())?; let client = Arc::new(SuccinctClient::new(&config).await?); let engine = Engine::new(&config, client.clone()).start(); + debug!("Running operator with host: {}", config.host); let server_handle = RpcServer::new(client, engine.clone()).run(&config).await?; if tokio::signal::ctrl_c().await.is_ok() { - log::info!("Shutting down.."); + info!("Shutting down.."); server_handle.abort(); System::current().stop(); } diff --git a/bin/operator/src/rpc.rs b/bin/operator/src/rpc.rs index 43f0405..57977bd 100644 --- a/bin/operator/src/rpc.rs +++ b/bin/operator/src/rpc.rs @@ -26,7 +26,7 @@ pub struct RpcServerImpl { impl RpcServerImpl { pub fn new(succinct_client: Arc, queue: Addr) -> Self { - log::info!("starting rpc server"); + tracing::info!("starting rpc server"); Self { succinct_client, queue, @@ -65,7 +65,7 @@ pub trait ProveRpc { impl ProveRpcServer for RpcServerImpl { async fn sync(&self) -> Result { let pid = self.succinct_client.sync(true).await.map_err(|e| { - log::error!("{:?}", e); + tracing::error!("{:?}", e); ErrorCode::ServerError(500) })?; Ok(pid) @@ -73,7 +73,7 @@ impl ProveRpcServer for RpcServerImpl { async fn verify(&self, ids: Vec) -> Result { let pid = self.succinct_client.verify(ids, true).await.map_err(|e| { - log::error!("{:?}", e); + tracing::error!("{:?}", e); ErrorCode::ServerError(500) })?; Ok(pid) @@ -85,7 +85,7 @@ impl ProveRpcServer for RpcServerImpl { .send(ProveTransaction { id: None, tx }) .await .map_err(|e| { - log::error!("{:?}", e); + tracing::error!("{:?}", e); ErrorCode::ServerError(500) })? .unwrap(); diff --git a/bin/operator/src/succinct/mod.rs b/bin/operator/src/succinct/mod.rs index 877b6e4..486a44b 100644 --- a/bin/operator/src/succinct/mod.rs +++ b/bin/operator/src/succinct/mod.rs @@ -28,6 +28,7 @@ use reqwest_middleware::ClientWithMiddleware; use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; use serde::Deserialize; use succinct_client::{request::SuccinctClient as SuccinctClientExt, utils::get_gateway_address}; +use tracing::{debug, info, trace}; use types::TransactionOrReceiptIdPrimitive; use self::types::{Circuit, Deployment, NearX::NearXInstance, NearXClient, ProofResponse}; @@ -96,7 +97,7 @@ pub struct Client { impl Client { pub async fn new(config: &config::Config) -> anyhow::Result { - log::info!("starting succinct client"); + info!("starting succinct client"); let (contract, chain_id) = Self::init_contract_client(&config.succinct).await?; @@ -130,7 +131,7 @@ impl Client { } async fn init_contract_client(config: &Config) -> anyhow::Result<(NearXClient, u32)> { - log::debug!("initializing contract client"); + debug!("initializing contract client"); let url = Url::from_str(&config.eth_rpc_url).with_context(|| "invalid rpc url")?; let inner = Provider::new_http(url); @@ -139,7 +140,7 @@ impl Client { .await .with_context(|| "failed to get chain id")?; let contract = NearXInstance::new(config.contract_address, inner.into()); - log::debug!("chain id: {}", chain_id); + debug!("chain id: {}", chain_id); Ok((contract, chain_id.to())) } @@ -168,8 +169,9 @@ impl Client { Ok(client) } + #[tracing::instrument(skip(self))] async fn fetch_releases(&self, chain_id: &u32) -> anyhow::Result> { - log::debug!( + debug!( "filtering releases for {chain_id} and version {}", self.config.version ); @@ -178,7 +180,7 @@ impl Client { chain_id, &self.config.version, )) - .inspect(|r| log::trace!("releases: {:?}", r)) + .inspect(|r| trace!("releases: {:?}", r)) } fn extract_release_details( @@ -198,7 +200,7 @@ impl Client { d }) } else { - log::debug!("matched deployment: {:#?}", d); + debug!("matched deployment: {:#?}", d); Some(d) } }) @@ -210,7 +212,7 @@ impl Client { release_id: &str, data: BytesRequestData, ) -> ProofRequest { - log::trace!( + trace!( "building proof request for {:?} with data {:?}", release_id, hex!(&data.input) @@ -224,7 +226,7 @@ impl Client { } fn build_sync_request(&self, trusted_header_hash: CryptoHash) -> BytesRequestData { - log::debug!("building sync request for {:?}", trusted_header_hash); + debug!("building sync request for {:?}", trusted_header_hash); BytesRequestData { input: trusted_header_hash.0.to_vec(), } @@ -235,8 +237,8 @@ impl Client { trusted_header_hash: CryptoHash, ids: Vec, ) -> BytesRequestData { - log::debug!("building verify request for {:?}", trusted_header_hash); - log::trace!("ids {:?}", ids); + debug!("building verify request for {:?}", trusted_header_hash); + trace!("ids {:?}", ids); BytesRequestData { // TODO: define this input by abi input: [ @@ -250,8 +252,9 @@ impl Client { } } + #[tracing::instrument(skip(self))] async fn fetch_deployments(&self) -> anyhow::Result> { - log::debug!("getting deployments"); + debug!("getting deployments"); Ok(self .inner .get(format!( @@ -263,9 +266,10 @@ impl Client { .error_for_status()? .json() .await - .inspect(|d| log::debug!("fetched deployments: {:?}", d))?) + .inspect(|d| debug!("fetched deployments: {:?}", d))?) } + #[tracing::instrument(skip(self))] async fn request_relayed_proof( &self, circuit: &Circuit, @@ -284,15 +288,16 @@ impl Client { circuit .function_id(&self.contract) .await - .inspect(|d| log::debug!("function_id: {:?}", d))? + .inspect(|d| debug!("function_id: {:?}", d))? .into(), req.input.into(), ) .await - .inspect(|d| log::debug!("requested relay proof: {:?}", d))?; + .inspect(|d| debug!("requested relay proof: {:?}", d))?; self.wait_for_proof(&request_id).await } + #[tracing::instrument(skip(self))] async fn fetch_proofs(&self) -> anyhow::Result> { let res: anyhow::Result<_> = Ok(self .inner @@ -314,6 +319,7 @@ impl Client { /// Wait for the proof to be submitted to the explorer so we can track them /// by their proof id + #[tracing::instrument(skip(self))] pub async fn wait_for_proof(&self, request_id: &str) -> anyhow::Result { let mut interval = tokio::time::interval(Duration::from_secs(5)); let mut attempts = 0; @@ -341,16 +347,17 @@ impl Client { .requests .iter() .find(|r| { - log::debug!("checking if {:?} matches {:?}", r.id, request_id); + debug!("checking if {:?} matches {:?}", r.id, request_id); r.id == request_id }) .is_some() }) - .inspect(|p| log::debug!("found proof {:?} matching request: {:?}", p.id, request_id)) + .inspect(|p| debug!("found proof {:?} matching request: {:?}", p.id, request_id)) } /// Request a proof to be proven, this doesn't relay the proof to the /// contract, useful for users who don't want to relay + #[tracing::instrument(skip(self))] pub async fn request_proof( &self, circuit: &Circuit, @@ -368,12 +375,13 @@ impl Client { .error_for_status()? .json::() .await - .inspect(|d| log::debug!("requested proof: {:?}", d.proof_id))? + .inspect(|d| debug!("requested proof: {:?}", d.proof_id))? .proof_id) } + #[tracing::instrument(skip(self))] pub async fn get_proof(&self, proof_id: ProofId) -> anyhow::Result { - log::debug!("fetching proof: {}", proof_id.0); + debug!("fetching proof: {}", proof_id.0); Ok(self .inner .get(format!("{}/proof/{}", self.config.rpc_url, proof_id.0)) @@ -382,10 +390,11 @@ impl Client { .error_for_status()? .json::() .await - .inspect(|d| log::debug!("fetched proof: {:?}/{:?}", d.id, d.status))?) + .inspect(|d| debug!("fetched proof: {:?}/{:?}", d.id, d.status))?) } /// Sync the light client + #[tracing::instrument(skip(self))] pub async fn sync(&self, relay: bool) -> anyhow::Result { let circuit = Circuit::Sync; let req = self.build_sync_request(self.fetch_trusted_header_hash().await?); @@ -399,12 +408,13 @@ impl Client { } /// Verify a set of transactions + #[tracing::instrument(skip(self))] pub async fn verify( &self, ids: Vec, relay: bool, ) -> anyhow::Result { - log::trace!("verifying {} ids", ids.len()); + trace!("verifying {} ids", ids.len()); ensure!( ids.len() == self.verify_amt, "wrong number of transactions for verify" @@ -421,6 +431,7 @@ impl Client { } /// Fetch the last synced header from the contract + #[tracing::instrument(skip(self))] async fn fetch_trusted_header_hash(&self) -> anyhow::Result { let mut h = self .contract @@ -429,9 +440,9 @@ impl Client { .await .map(|x| *x._0) .map(CryptoHash)?; - log::debug!("fetched trusted header hash {:?}", h); + debug!("fetched trusted header hash {:?}", h); if h == CryptoHash::default() { - log::info!("no trusted header found, using checkpoint hash"); + info!("no trusted header found, using checkpoint hash"); h = self.genesis; } Ok(h) diff --git a/bin/operator/src/succinct/types.rs b/bin/operator/src/succinct/types.rs index e31246a..605c4c6 100644 --- a/bin/operator/src/succinct/types.rs +++ b/bin/operator/src/succinct/types.rs @@ -9,6 +9,7 @@ use near_light_clientx::plonky2x::backend::{ prover::ProofId, }; use serde::{Deserialize, Serialize}; +use tracing::debug; use uuid::Uuid; use NearX::TransactionOrReceiptId; @@ -26,6 +27,7 @@ sol!( ); /// The circuits we support in this nearxclient +#[derive(Debug)] pub enum Circuit { Sync, Verify, @@ -57,9 +59,9 @@ impl Circuit { /// Filter a deployment from the release list /// Safety: panics when a deployment cannot be found pub fn deployment(&self, releases: &[Deployment]) -> Deployment { - log::debug!("finding deployment in {:?}", releases); + debug!("finding deployment in {:?}", releases); let find = |entrypoint: &str| -> Deployment { - log::debug!("finding deployment for {}", entrypoint); + debug!("finding deployment for {}", entrypoint); releases .iter() .find(|r| r.release_info.release.entrypoint == entrypoint) diff --git a/bin/operator/tests/succinct.rs b/bin/operator/tests/succinct.rs index 6acfdeb..641af94 100644 --- a/bin/operator/tests/succinct.rs +++ b/bin/operator/tests/succinct.rs @@ -1,15 +1,11 @@ #![allow(dead_code)] // Justification: Until we decide on test feature flags use std::str::FromStr; -use near_light_client_rpc::prelude::Itertools; -use nearx_operator::{ - config::Config, succinct::*, types::TransactionOrReceiptIdPrimitive, BaseConfig, -}; -use test_utils::fixture; +use nearx_operator::{config::Config, succinct::*, BaseConfig}; use uuid::Uuid; async fn client() -> Client { - pretty_env_logger::try_init().ok(); + let _ = tracing_subscriber::fmt::try_init(); Client::new(&Config::test_config()).await.unwrap() } diff --git a/config.toml b/config.toml new file mode 100644 index 0000000..b29f628 --- /dev/null +++ b/config.toml @@ -0,0 +1,34 @@ +[default] +catchup = false +state_path = "state.db" + +[default.rpc] +network = "Testnet" + +[default.protocol] +genesis = "6pD4xDH3kRSyBdEacErHF7WTeJ2fUXmzU8zXHHTrCnHY" + +[default.succinct] +api_key = "default" +contract_address = "0x0000000000000000000000000000000000000000" +eth_rpc_url = "https://gateway.tenderly.co/public/sepolia" +version = "latest" + +[default.engine] +cleanup_interval = 60 +drain_interval = 1 +persist_interval = 30 +sync_interval = 3000 + +[testnet] +catchup = true + +[testnet.rpc] +network = "Testnet" + +# Sepolia +[testnet.succinct] +api_key = "testnet" +contract_address = "0x73876e41ca149853160Ed5BFeC22e3C7bABEA67a" +eth_rpc_url = "https://gateway.tenderly.co/public/sepolia" +version = "v0.0.4-rc.1" diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index b9f16c9..1f51657 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -7,7 +7,7 @@ version.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -config.workspace = true +figment.workspace = true log.workspace = true near-primitives-core.workspace = true serde.workspace = true diff --git a/crates/primitives/src/config.rs b/crates/primitives/src/config.rs index 11e7ec9..be1857e 100644 --- a/crates/primitives/src/config.rs +++ b/crates/primitives/src/config.rs @@ -1,6 +1,9 @@ use std::env; -use config::{Config as ConfigTrait, ConfigError, Environment, File}; +use figment::{ + providers::{Env, Format, Toml}, + Error as ConfigError, Figment, +}; use serde::Deserialize; pub fn default_host() -> String { @@ -24,34 +27,55 @@ where T::new(None).expect("Failed to load config") } fn new(path: Option<&str>) -> Result { - let path = path.unwrap_or("."); + let path = path.unwrap_or(""); let env_prefix = "NEAR_LIGHT_CLIENT"; let required = env::var(format!("{env_prefix}_CONFIG_FILE")) - .unwrap_or_else(|_| format!("{path}/default")); + .unwrap_or_else(|_| format!("config.toml")); log::debug!("required file: {required}"); - let mode_path = env::var(format!("{env_prefix}_NETWORK")) - .map(|s| s.to_lowercase()) - .unwrap_or_else(|_| format!("{path}/testnet")); + let mode = env::var(format!("{env_prefix}_MODE")) + .unwrap_or_else(|_| "testnet".into()) + .to_lowercase(); + + let mode_path = format!("{path}{mode}.toml"); log::debug!("mode file: {mode_path}"); - let local_path = format!("{path}/local"); + let local_path = format!("{path}local.toml"); log::debug!("local file: {local_path}"); - let s = ConfigTrait::builder() - .add_source(File::with_name(&required).required(true)) - .add_source(File::with_name(&mode_path).required(false)) - // This file shouldn't be checked in to git - .add_source(File::with_name(&local_path).required(false)) - .add_source(Environment::with_prefix(env_prefix).try_parsing(true)) - .build()?; + let figment = Figment::new() + .select(mode) + .merge(Toml::file(&required).nested()) + .merge(Toml::file(&local_path).nested()) + .merge( + Env::prefixed(&format!("{env_prefix}_")) + .split("__") + .global(), + ); + + println!("figment: {figment:#?}"); - s.try_deserialize() + figment.extract() } fn test_config() -> T { Self::new(Some("../../")).unwrap() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Debug, Deserialize, Clone)] + struct StubConfig; + impl Configurable for StubConfig {} + + #[test] + fn test_default() { + std::env::set_var("NEAR_LIGHT_CLIENT_NETWORK", "fakenet"); + StubConfig::test_config(); + } +} diff --git a/default.toml b/default.toml deleted file mode 100644 index 7850cd8..0000000 --- a/default.toml +++ /dev/null @@ -1,14 +0,0 @@ -catchup = false -state_path = "state.db" - -[rpc] -network = "Testnet" - -[protocol] -genesis = "6pD4xDH3kRSyBdEacErHF7WTeJ2fUXmzU8zXHHTrCnHY" - -[succinct] -api_key = "" -contract_address = "0x0000000000000000000000000000000000000000" -eth_rpc_url = "https://gateway.tenderly.co/public/sepolia" -version = "latest" diff --git a/testnet.toml b/testnet.toml deleted file mode 100644 index 0d9a966..0000000 --- a/testnet.toml +++ /dev/null @@ -1,14 +0,0 @@ -catchup = true -host = "0.0.0.0:3030" -state_path = "state.db" - -[protocol] -genesis = "6pD4xDH3kRSyBdEacErHF7WTeJ2fUXmzU8zXHHTrCnHY" - -[rpc] -network = "Testnet" - -[succinct] -contract_address = "0x73876e41ca149853160Ed5BFeC22e3C7bABEA67a" -eth_rpc_url = "https://gateway.tenderly.co/public/sepolia" -version = "v0.0.4-rc.1"