feat(contract-verifier): Partial matching & automatic verification

.github/workflows/ci-core-reusable.yml

@@ -72,7 +72,7 @@ jobs:
           ci_run zkstack dev contracts
 
       - name: Download compilers for contract verifier tests
-        run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
+        run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.10 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
 
       - name: Rust unit tests
         run: |
@@ -431,7 +431,7 @@ jobs:
 
       - name: Initialize Contract verifier
         run: |
-          ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
+          ci_run zkstack contract-verifier init --zksolc-version=v1.5.10 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era
           ci_run zkstack contract-verifier run --chain era &> ${{ env.SERVER_LOGS_DIR }}/contract-verifier-rollup.log &
           ci_run zkstack contract-verifier wait --chain era --verbose
 

core/Cargo.toml

@@ -170,6 +170,7 @@ serde = "1"
 serde_json = "1"
 serde_with = "1"
 serde_yaml = "0.9"
+ciborium = "0.2"
 sha2 = "0.10.8"
 sha3 = "0.10.8"
 sqlx = "0.8.1"
@@ -231,7 +232,7 @@ tokio-stream = "0.1.16"
 circuit_encodings = "=0.150.20"
 circuit_sequencer_api = "=0.150.20"
 circuit_definitions = "=0.150.20"
-crypto_codegen = { package = "zksync_solidity_vk_codegen",version = "=0.30.13" }
+crypto_codegen = { package = "zksync_solidity_vk_codegen", version = "=0.30.13" }
 kzg = { package = "zksync_kzg", version = "=0.150.20" }
 zk_evm = { version = "=0.133.0" }
 zk_evm_1_3_1 = { package = "zk_evm", version = "0.131.0-rc.2" }
@@ -240,7 +241,7 @@ zk_evm_1_4_0 = { package = "zk_evm", version = "0.140" }
 zk_evm_1_4_1 = { package = "zk_evm", version = "0.141" }
 zk_evm_1_5_0 = { package = "zk_evm", version = "=0.150.20" }
 fflonk = "=0.30.13"
-bellman = {package = "zksync_bellman", version = "=0.30.13"}
+bellman = { package = "zksync_bellman", version = "=0.30.13" }
 
 # New VM; pinned to a specific commit because of instability
 zksync_vm2 = { git = "https://github.com/matter-labs/vm2.git", rev = "457d8a7eea9093af9440662e33e598c13ba41633" }

core/bin/contract-verifier/src/main.rs

@@ -6,7 +6,7 @@ use tokio::sync::watch;
 use zksync_config::configs::PrometheusConfig;
 use zksync_contract_verifier_lib::ContractVerifier;
 use zksync_core_leftovers::temp_config_store::{load_database_secrets, load_general_config};
-use zksync_dal::{ConnectionPool, Core};
+use zksync_dal::{ConnectionPool, Core, CoreDal};
 use zksync_queued_job_processor::JobProcessor;
 use zksync_utils::wait_for_tasks::ManagedTasks;
 use zksync_vlog::prometheus::PrometheusExporterConfig;
@@ -25,6 +25,32 @@ struct Opt {
     secrets_path: Option<PathBuf>,
 }
 
+async fn perform_storage_migration(pool: &ConnectionPool<Core>) -> anyhow::Result<()> {
+    const BATCH_SIZE: usize = 1000;
+
+    // Make it possible to override just in case.
+    let batch_size = std::env::var("CONTRACT_VERIFIER_MIGRATION_BATCH_SIZE")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(BATCH_SIZE);
+
+    let mut storage = pool.connection().await?;
+    let migration_performed = storage
+        .contract_verification_dal()
+        .is_verification_info_migration_performed()
+        .await?;
+    if !migration_performed {
+        tracing::info!("Running the storage migration for the contract verifier table");
+        storage
+            .contract_verification_dal()
+            .perform_verification_info_migration(batch_size)
+            .await?;
+    } else {
+        tracing::info!("Storage migration is not needed");
+    }
+    Ok(())
+}
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     let opt = Opt::parse();
@@ -51,6 +77,8 @@ async fn main() -> anyhow::Result<()> {
     .build()
     .await?;
 
+    perform_storage_migration(&pool).await?;
+
     let (stop_sender, stop_receiver) = watch::channel(false);
     let contract_verifier = ContractVerifier::new(verifier_config.compilation_timeout(), pool)
         .await

core/bin/verified_sources_fetcher/src/main.rs

@@ -3,7 +3,7 @@ use std::io::Write;
 use zksync_config::configs::DatabaseSecrets;
 use zksync_dal::{ConnectionPool, Core, CoreDal};
 use zksync_env_config::FromEnv;
-use zksync_types::contract_verification_api::SourceCodeData;
+use zksync_types::contract_verification::api::SourceCodeData;
 
 #[tokio::main]
 async fn main() {

core/lib/contract_verifier/src/compilers/mod.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use anyhow::Context as _;
 use serde::{Deserialize, Serialize};
-use zksync_types::contract_verification_api::CompilationArtifacts;
+use zksync_types::contract_verification::api::CompilationArtifacts;
 
 pub(crate) use self::{
     solc::{Solc, SolcInput},

core/lib/contract_verifier/src/compilers/solc.rs

@@ -3,8 +3,12 @@ use std::{collections::HashMap, path::PathBuf, process::Stdio};
 use anyhow::Context;
 use tokio::io::AsyncWriteExt;
 use zksync_queued_job_processor::async_trait;
-use zksync_types::contract_verification_api::{
-    CompilationArtifacts, SourceCodeData, VerificationIncomingRequest,
+use zksync_types::{
+    bytecode::BytecodeMarker,
+    contract_verification::{
+        api::{CompilationArtifacts, SourceCodeData, VerificationIncomingRequest},
+        contract_identifier::ContractIdentifier,
+    },
 };
 
 use super::{parse_standard_json_output, process_contract_name, Settings, Source, StandardJson};
@@ -103,7 +107,7 @@ impl Compiler<SolcInput> for Solc {
     async fn compile(
         self: Box<Self>,
         input: SolcInput,
-    ) -> Result<CompilationArtifacts, ContractVerifierError> {
+    ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> {
         let mut command = tokio::process::Command::new(&self.path);
         let mut child = command
             .arg("--standard-json")
@@ -128,7 +132,11 @@ impl Compiler<SolcInput> for Solc {
         if output.status.success() {
             let output = serde_json::from_slice(&output.stdout)
                 .context("zksolc output is not valid JSON")?;
-            parse_standard_json_output(&output, input.contract_name, input.file_name, true)
+            let output =
+                parse_standard_json_output(&output, input.contract_name, input.file_name, true)?;
+            let id =
+                ContractIdentifier::from_bytecode(BytecodeMarker::Evm, output.deployed_bytecode());
+            Ok((output, id))
         } else {
             Err(ContractVerifierError::CompilerError(
                 "solc",

core/lib/contract_verifier/src/compilers/vyper.rs

@@ -3,8 +3,12 @@ use std::{collections::HashMap, mem, path::PathBuf, process::Stdio};
 use anyhow::Context;
 use tokio::io::AsyncWriteExt;
 use zksync_queued_job_processor::async_trait;
-use zksync_types::contract_verification_api::{
-    CompilationArtifacts, SourceCodeData, VerificationIncomingRequest,
+use zksync_types::{
+    bytecode::BytecodeMarker,
+    contract_verification::{
+        api::{CompilationArtifacts, SourceCodeData, VerificationIncomingRequest},
+        contract_identifier::ContractIdentifier,
+    },
 };
 
 use super::{parse_standard_json_output, process_contract_name, Settings, Source, StandardJson};
@@ -76,7 +80,7 @@ impl Compiler<VyperInput> for Vyper {
     async fn compile(
         self: Box<Self>,
         mut input: VyperInput,
-    ) -> Result<CompilationArtifacts, ContractVerifierError> {
+    ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> {
         let mut command = tokio::process::Command::new(&self.path);
         let mut child = command
             .arg("--standard-json")
@@ -103,7 +107,11 @@ impl Compiler<VyperInput> for Vyper {
         if output.status.success() {
             let output =
                 serde_json::from_slice(&output.stdout).context("vyper output is not valid JSON")?;
-            parse_standard_json_output(&output, input.contract_name, input.file_name, true)
+            let output =
+                parse_standard_json_output(&output, input.contract_name, input.file_name, true)?;
+            let id =
+                ContractIdentifier::from_bytecode(BytecodeMarker::Evm, output.deployed_bytecode());
+            Ok((output, id))
         } else {
             Err(ContractVerifierError::CompilerError(
                 "vyper",

core/lib/contract_verifier/src/compilers/zksolc.rs

@@ -6,8 +6,12 @@ use semver::Version;
 use serde::{Deserialize, Serialize};
 use tokio::io::AsyncWriteExt;
 use zksync_queued_job_processor::async_trait;
-use zksync_types::contract_verification_api::{
-    CompilationArtifacts, SourceCodeData, VerificationIncomingRequest,
+use zksync_types::{
+    bytecode::BytecodeMarker,
+    contract_verification::{
+        api::{CompilationArtifacts, SourceCodeData, VerificationIncomingRequest},
+        contract_identifier::ContractIdentifier,
+    },
 };
 
 use super::{parse_standard_json_output, process_contract_name, Source};
@@ -65,6 +69,7 @@ pub(crate) struct Optimizer {
     /// Whether the optimizer is enabled.
     pub enabled: bool,
     /// The optimization mode string.
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub mode: Option<char>,
 }
 
@@ -144,12 +149,24 @@ impl ZkSolc {
     fn parse_single_file_yul_output(
         output: &str,
     ) -> Result<CompilationArtifacts, ContractVerifierError> {
-        let re = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap();
-        let cap = re
-            .captures(output)
-            .context("Yul output doesn't match regex")?;
+        let cap = if output.contains("Binary:\n") {
+            // Format of the new output
+            // ======= /tmp/input.yul:Empty =======
+            // Binary:
+            // 00000001002 <..>
+            let re = Regex::new(r"Binary:\n([\da-f]+)").unwrap();
+            re.captures(output)
+                .with_context(|| format!("Yul output doesn't match regex. Output: {output}"))?
+        } else {
+            // Old compiler versions
+            let re_old = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap();
+            re_old
+                .captures(output)
+                .with_context(|| format!("Yul output doesn't match regex. Output: {output}"))?
+        };
         let bytecode_str = cap.get(1).context("no matches in Yul output")?.as_str();
         let bytecode = hex::decode(bytecode_str).context("invalid Yul output bytecode")?;
+
         Ok(CompilationArtifacts {
             bytecode,
             deployed_bytecode: None,
@@ -179,7 +196,7 @@ impl Compiler<ZkSolcInput> for ZkSolc {
     async fn compile(
         self: Box<Self>,
         input: ZkSolcInput,
-    ) -> Result<CompilationArtifacts, ContractVerifierError> {
+    ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> {
         let mut command = tokio::process::Command::new(&self.paths.zk);
         command.stdout(Stdio::piped()).stderr(Stdio::piped());
 
@@ -238,7 +255,13 @@ impl Compiler<ZkSolcInput> for ZkSolc {
                 if output.status.success() {
                     let output = serde_json::from_slice(&output.stdout)
                         .context("zksolc output is not valid JSON")?;
-                    parse_standard_json_output(&output, contract_name, file_name, false)
+                    let output =
+                        parse_standard_json_output(&output, contract_name, file_name, false)?;
+                    let id = ContractIdentifier::from_bytecode(
+                        BytecodeMarker::EraVm,
+                        output.deployed_bytecode(),
+                    );
+                    Ok((output, id))
                 } else {
                     Err(ContractVerifierError::CompilerError(
                         "zksolc",
@@ -255,6 +278,9 @@ impl Compiler<ZkSolcInput> for ZkSolc {
                     .context("cannot create temporary Yul file")?;
                 file.write_all(source_code.as_bytes())
                     .context("failed writing Yul file")?;
+
+                // TODO: `zksolc` support standard JSON for `yul` since 1.5.0, so we don't have
+                // to parse `--bin` output.
                 let child = command
                     .arg(file.path().to_str().unwrap())
                     .arg("--optimization")
@@ -267,7 +293,12 @@ impl Compiler<ZkSolcInput> for ZkSolc {
                 if output.status.success() {
                     let output =
                         String::from_utf8(output.stdout).context("zksolc output is not UTF-8")?;
-                    Self::parse_single_file_yul_output(&output)
+                    let output = Self::parse_single_file_yul_output(&output)?;
+                    let id = ContractIdentifier::from_bytecode(
+                        BytecodeMarker::EraVm,
+                        output.deployed_bytecode(),
+                    );
+                    Ok((output, id))
                 } else {
                     Err(ContractVerifierError::CompilerError(
                         "zksolc",

core/lib/contract_verifier/src/compilers/zkvyper.rs

@@ -3,7 +3,10 @@ use std::{ffi::OsString, path, path::Path, process::Stdio};
 use anyhow::Context as _;
 use tokio::{fs, io::AsyncWriteExt};
 use zksync_queued_job_processor::async_trait;
-use zksync_types::contract_verification_api::CompilationArtifacts;
+use zksync_types::{
+    bytecode::BytecodeMarker,
+    contract_verification::{api::CompilationArtifacts, contract_identifier::ContractIdentifier},
+};
 
 use super::VyperInput;
 use crate::{
@@ -95,7 +98,7 @@ impl Compiler<VyperInput> for ZkVyper {
     async fn compile(
         self: Box<Self>,
         input: VyperInput,
-    ) -> Result<CompilationArtifacts, ContractVerifierError> {
+    ) -> Result<(CompilationArtifacts, ContractIdentifier), ContractVerifierError> {
         let mut command = tokio::process::Command::new(&self.paths.zk);
         if let Some(o) = input.optimizer_mode.as_ref() {
             command.arg("-O").arg(o);
@@ -123,7 +126,12 @@ impl Compiler<VyperInput> for ZkVyper {
         if output.status.success() {
             let output = serde_json::from_slice(&output.stdout)
                 .context("zkvyper output is not valid JSON")?;
-            Self::parse_output(&output, input.contract_name)
+            let output = Self::parse_output(&output, input.contract_name)?;
+            let id = ContractIdentifier::from_bytecode(
+                BytecodeMarker::EraVm,
+                output.deployed_bytecode(),
+            );
+            Ok((output, id))
         } else {
             Err(ContractVerifierError::CompilerError(
                 "zkvyper",