diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f8af7c2..b746bcf 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -159,7 +159,7 @@ jobs: fi public-push: name: Push Docker Images to Docker hub - if: contains('main holesky mainnet hack-merge-table v1_base 129/merge', github.ref_name) || startsWith(github.ref, 'refs/tags/') + if: contains('main holesky mainnet hack-merge-table v1_base 140/merge', github.ref_name) || startsWith(github.ref, 'refs/tags/') runs-on: ${{needs.base.outputs.RUNNER}} needs: - build diff --git a/Cargo.lock b/Cargo.lock index 2a648cd..c3222fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3136,6 +3136,26 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "glob" version = "0.3.1" @@ -3156,8 +3176,8 @@ dependencies = [ [[package]] name = "gnark-utils" -version = "0.1.0" -source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=v1.1.2#77fa458c7d136cdea2db4637d0031facfe7d38d7" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" dependencies = [ "anyhow", "base64 0.22.1", @@ -3176,8 +3196,8 @@ dependencies = [ [[package]] name = "groth16_framework" -version = "0.1.0" -source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=v1.1.2#77fa458c7d136cdea2db4637d0031facfe7d38d7" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" dependencies = [ "alloy", "anyhow", @@ -3981,6 +4001,7 @@ dependencies = [ "metrics-exporter-prometheus", "miette", "mimalloc", + "mp2_common", "prost", "protox", "rand", @@ -4287,8 +4308,8 @@ checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" [[package]] name = "mp2_common" -version = "0.1.0" -source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=v1.1.2#77fa458c7d136cdea2db4637d0031facfe7d38d7" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" dependencies = [ "alloy", "anyhow", @@ -4296,6 +4317,7 @@ dependencies = [ "derive_more 1.0.0", "eth_trie", "ethereum-types", + "git-version", "hashbrown 0.15.0", "hex", "itertools 0.13.0", @@ -4314,8 +4336,8 @@ dependencies = [ [[package]] name = "mp2_test" -version = "0.1.0" -source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=v1.1.2#77fa458c7d136cdea2db4637d0031facfe7d38d7" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" dependencies = [ "alloy", "anyhow", @@ -4333,8 +4355,8 @@ dependencies = [ [[package]] name = "mp2_v1" -version = "0.1.0" -source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=v1.1.2#77fa458c7d136cdea2db4637d0031facfe7d38d7" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" dependencies = [ "alloy", "anyhow", @@ -4712,8 +4734,8 @@ dependencies = [ [[package]] name = "parsil" -version = "0.1.0" -source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=v1.1.2#77fa458c7d136cdea2db4637d0031facfe7d38d7" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" dependencies = [ "alloy", "anyhow", @@ -5474,8 +5496,8 @@ dependencies = [ [[package]] name = "recursion_framework" -version = "0.1.0" -source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=v1.1.2#77fa458c7d136cdea2db4637d0031facfe7d38d7" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" dependencies = [ "anyhow", "log", @@ -6012,8 +6034,8 @@ dependencies = [ [[package]] name = "ryhope" -version = "0.1.0" -source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=v1.1.2#77fa458c7d136cdea2db4637d0031facfe7d38d7" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" dependencies = [ "anyhow", "bb8", @@ -7489,8 +7511,8 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "verifiable-db" -version = "0.1.0" -source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=v1.1.2#77fa458c7d136cdea2db4637d0031facfe7d38d7" +version = "1.2.1" +source = "git+https://github.com/Lagrange-Labs/mapreduce-plonky2.git?rev=1.2.1#196952fe435d637757dda2c02ac3f694eb57cd18" dependencies = [ "alloy", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 9020b9c..ca2c455 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,11 +3,11 @@ resolver = "2" members = ["lgn-auth", "lgn-messages", "lgn-provers", "lgn-worker"] [workspace.dependencies] -groth16_framework_v1 = { git = "https://github.com/Lagrange-Labs/mapreduce-plonky2.git", rev = "v1.1.3", package = "groth16_framework" } -mp2_common = { git = "https://github.com/Lagrange-Labs/mapreduce-plonky2.git", rev = "v1.1.3" } -mp2_v1 = { git = "https://github.com/Lagrange-Labs/mapreduce-plonky2.git", rev = "v1.1.3" } -parsil = { git = "https://github.com/Lagrange-Labs/mapreduce-plonky2.git", rev = "v1.1.3" } -verifiable-db = { git = "https://github.com/Lagrange-Labs/mapreduce-plonky2.git", rev = "v1.1.3" } +groth16_framework_v1 = { git = "https://github.com/Lagrange-Labs/mapreduce-plonky2.git", rev = "1.2.1", package = "groth16_framework" } +mp2_common = { git = "https://github.com/Lagrange-Labs/mapreduce-plonky2.git", rev = "1.2.1" } +mp2_v1 = { git = "https://github.com/Lagrange-Labs/mapreduce-plonky2.git", rev = "1.2.1" } +parsil = { git = "https://github.com/Lagrange-Labs/mapreduce-plonky2.git", rev = "1.2.1" } +verifiable-db = { git = "https://github.com/Lagrange-Labs/mapreduce-plonky2.git", rev = "1.2.1" } anyhow = { version = "1.0" } bincode = { version = "1.0" } diff --git a/lgn-provers/src/params/mod.rs b/lgn-provers/src/params/mod.rs index 8d797fa..f14b004 100644 --- a/lgn-provers/src/params/mod.rs +++ b/lgn-provers/src/params/mod.rs @@ -18,6 +18,9 @@ use tracing::error; use tracing::info; use tracing::warn; +/// The filename of params checksum hashes +pub const PARAMS_CHECKSUM_FILENAME: &str = "public_params.hash"; + pub struct ParamsLoader; // Could make configurable but 3600 should be enough diff --git a/lgn-worker/Cargo.toml b/lgn-worker/Cargo.toml index 7ce9b9c..80ddac4 100644 --- a/lgn-worker/Cargo.toml +++ b/lgn-worker/Cargo.toml @@ -19,17 +19,30 @@ serde_json = { workspace = true } checksums = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } -tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } -rand = { workspace = true, default-features = false, features = [ "std", "std_rng", "getrandom", "min_const_gen" ] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +rand = { workspace = true, default-features = false, features = [ + "std", + "std_rng", + "getrandom", + "min_const_gen", +] } reqwest = { workspace = true, features = ["blocking"] } # The ethers macro `abigen` needs to import ethers as a crate. backtrace = { workspace = true } -clap = { workspace = true, features = ["derive", "env", "help", "std", "suggestions"] } +clap = { workspace = true, features = [ + "derive", + "env", + "help", + "std", + "suggestions", +] } config = { workspace = true, features = ["toml"] } elliptic-curve = { workspace = true } # The ethers macro `abigen` needs to import ethers as a crate. -ethers = { git = "https://github.com/Lagrange-Labs/ethers-rs", default-features = false, features = [ "rustls" ], branch = "get-proof-0x" } +ethers = { git = "https://github.com/Lagrange-Labs/ethers-rs", default-features = false, features = [ + "rustls", +], branch = "get-proof-0x" } jwt = { workspace = true } k256 = { workspace = true, features = ["ecdsa", "std"] } lazy-static-include = { workspace = true } @@ -44,6 +57,8 @@ tonic = { workspace = true } prost = { workspace = true } tokio-stream = { workspace = true } +mp2_common = { workspace = true } + lgn-auth = { path = "../lgn-auth" } lgn-messages = { path = "../lgn-messages" } lgn-provers = { path = "../lgn-provers" } diff --git a/lgn-worker/src/config/default.toml b/lgn-worker/src/config/default.toml index 1db90a8..4f935fe 100644 --- a/lgn-worker/src/config/default.toml +++ b/lgn-worker/src/config/default.toml @@ -17,8 +17,7 @@ dir = "./zkmr_params" checksum_expected_local_path = "/tmp/expected_checksums.txt" # Parameters common directory # v1.1.x PPs -url = "https://pub-fbb5db8dc9ee4e8da9daf13e07d27c24.r2.dev" -checksum_url = "https://pub-fbb5db8dc9ee4e8da9daf13e07d27c24.r2.dev/public_params.hash" +params_root_url = "https://pub-a894572689a54c008859f232868fc67d.r2.dev" skip_checksum = false skip_store = false diff --git a/lgn-worker/src/config/mod.rs b/lgn-worker/src/config/mod.rs index bd8a948..7d27f3d 100644 --- a/lgn-worker/src/config/mod.rs +++ b/lgn-worker/src/config/mod.rs @@ -1,6 +1,7 @@ use config::FileFormat; use lazy_static_include::*; use lgn_messages::types::TaskDifficulty; +use lgn_provers::params::PARAMS_CHECKSUM_FILENAME; use redact::Secret; use serde_derive::Deserialize; use tracing::debug; @@ -21,8 +22,9 @@ pub(crate) struct Config #[derive(Deserialize, Debug, Clone, PartialEq)] pub(crate) struct PublicParamsConfig { - pub(crate) url: String, - pub(crate) checksum_url: String, + /// the root URL over which we should fetch params. + /// The FULL url is constructed from this one and the mp2 version + pub(crate) params_root_url: String, pub(crate) checksum_expected_local_path: String, pub(crate) skip_checksum: bool, pub(crate) dir: String, @@ -39,16 +41,10 @@ impl PublicParamsConfig { assert!( !self - .url + .params_root_url .is_empty(), "URL is required" ); - assert!( - !self - .checksum_url - .is_empty(), - "Checksum URL is required" - ); assert!( !self .checksum_expected_local_path @@ -68,6 +64,19 @@ impl PublicParamsConfig self.groth16_assets .validate(); } + + /// Build the base URL with path of mp2 version for downloading param files. + pub fn params_base_url(&self) -> String + { + add_mp2_version_path_to_url(&self.params_root_url) + } + + /// Build the URL for downloading the checksum file. + pub fn checksum_file_url(&self) -> String + { + let url = self.params_base_url(); + format!("{url}/{PARAMS_CHECKSUM_FILENAME}") + } } #[derive(Deserialize, Debug, Clone, PartialEq)] @@ -264,3 +273,11 @@ impl Config .validate(); } } + +/// Add mp2 version as a path to the base URL. +/// e.g. https://base.com/MP2_VERSION +fn add_mp2_version_path_to_url(url: &str) -> String +{ + let mp2_ver = mp2_common::git::short_git_version(); + format!("{url}/{mp2_ver}") +} diff --git a/lgn-worker/src/main.rs b/lgn-worker/src/main.rs index 9415a5c..9796ecd 100644 --- a/lgn-worker/src/main.rs +++ b/lgn-worker/src/main.rs @@ -272,7 +272,7 @@ async fn maybe_verify_checksums(config: &Config) -> Result<()> let checksum_url = &config .public_params - .checksum_url; + .checksum_file_url(); let expected_checksums_file = &config .public_params .checksum_expected_local_path; @@ -729,7 +729,7 @@ fn run_with_websocket(config: &Config) -> Result<()> // Always download the checksum files, this is needed by the prover constructor let checksum_url = &config .public_params - .checksum_url; + .checksum_file_url(); let expected_checksums_file = &config .public_params .checksum_expected_local_path; diff --git a/lgn-worker/src/manager/v1.rs b/lgn-worker/src/manager/v1.rs index dd7e60b..5019911 100644 --- a/lgn-worker/src/manager/v1.rs +++ b/lgn-worker/src/manager/v1.rs @@ -57,8 +57,9 @@ fn register_v1_preprocessor( ) -> Result<()> { let params_config = &config.public_params; + let params_base_url = ¶ms_config.params_base_url(); let preprocessing_prover = lgn_provers::provers::v1::preprocessing::create_prover( - ¶ms_config.url, + params_base_url, ¶ms_config.dir, ¶ms_config .preprocessing_params @@ -81,8 +82,9 @@ fn register_v1_query( ) -> Result<()> { let params_config = &config.public_params; + let params_base_url = ¶ms_config.params_base_url(); let query_prover = lgn_provers::provers::v1::query::create_prover( - ¶ms_config.url, + params_base_url, ¶ms_config.dir, ¶ms_config .query_params @@ -105,9 +107,10 @@ fn register_v1_groth16( ) -> Result<()> { let params_config = &config.public_params; + let params_base_url = ¶ms_config.params_base_url(); let assets = ¶ms_config.groth16_assets; let groth16_prover = lgn_provers::provers::v1::groth16::create_prover( - ¶ms_config.url, + params_base_url, ¶ms_config.dir, &assets.circuit_file, ¶ms_config.checksum_expected_local_path, diff --git a/lgn-worker/src/one-shot.rs b/lgn-worker/src/one-shot.rs index 0f066a7..921eff1 100644 --- a/lgn-worker/src/one-shot.rs +++ b/lgn-worker/src/one-shot.rs @@ -109,7 +109,7 @@ fn main() -> Result<()> let checksum_url = &config .public_params - .checksum_url; + .checksum_file_url(); let expected_checksums_file = &config .public_params .checksum_expected_local_path;