feat(zk_toolbox): zk_supervisor prover subcommand (matter-labs#2820)

## What ❔

Add prover subcommand for `zk_supervisor`. Add the following subcommand:
* `zk_supervisor prover info` - Prints information about current prover
setup.
* `zk_supervisor prover insert-version` - Insert new protocol version in
prover database(integration with `prover_cli`).
* `zk_supervisor prover insert-batch` - Insert new batch in prover
database(integration with `prover_cli`).

Add automatic creation of `prover/artifacts/witness_inputs` dirs if the
storage is file backed on init.

## Why ❔

To improve UX of working with provers.

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.

prover/crates/bin/prover_cli/README.md

@@ -9,6 +9,12 @@ git clone [email protected]:matter-labs/zksync-era.git
 cargo install prover_cli
 ```
 
+Or
+
+```
+cargo +nightly-2024-08-01 install --git https://github.com/matter-labs/zksync-era/ --locked prover_cli --force
+```
+
 ## Usage
 
 ```

zk_toolbox/crates/common/src/lib.rs

@@ -15,8 +15,8 @@ pub mod server;
 pub mod wallets;
 
 pub use prerequisites::{
-    check_general_prerequisites, check_prerequisites, GCLOUD_PREREQUISITES, GPU_PREREQUISITES,
-    WGET_PREREQUISITES,
+    check_general_prerequisites, check_prerequisites, GCLOUD_PREREQUISITE, GPU_PREREQUISITES,
+    PROVER_CLI_PREREQUISITE, WGET_PREREQUISITE,
 };
 pub use prompt::{init_prompt_theme, Prompt, PromptConfirm, PromptSelect};
 pub use term::{error, logger, spinner};

zk_toolbox/crates/common/src/prerequisites.rs

@@ -45,16 +45,22 @@ pub const GPU_PREREQUISITES: [Prerequisite; 3] = [
     }, // CUDA GPU driver
 ];
 
-pub const WGET_PREREQUISITES: [Prerequisite; 1] = [Prerequisite {
+pub const WGET_PREREQUISITE: [Prerequisite; 1] = [Prerequisite {
     name: "wget",
     download_link: "https://www.gnu.org/software/wget/",
 }];
 
-pub const GCLOUD_PREREQUISITES: [Prerequisite; 1] = [Prerequisite {
+pub const GCLOUD_PREREQUISITE: [Prerequisite; 1] = [Prerequisite {
     name: "gcloud",
     download_link: "https://cloud.google.com/sdk/docs/install",
 }];
 
+pub const PROVER_CLI_PREREQUISITE: [Prerequisite; 1] = [Prerequisite {
+    name: "prover_cli",
+    download_link:
+        "https://github.com/matter-labs/zksync-era/tree/main/prover/crates/bin/prover_cli",
+}];
+
 pub struct Prerequisite {
     name: &'static str,
     download_link: &'static str,

zk_toolbox/crates/config/src/ecosystem.rs

@@ -290,3 +290,10 @@ fn find_file(shell: &Shell, path_buf: PathBuf, file_name: &str) -> Result<PathBu
         find_file(shell, path.to_path_buf(), file_name)
     }
 }
+
+pub fn get_link_to_prover(config: &EcosystemConfig) -> PathBuf {
+    let link_to_code = config.link_to_code.clone();
+    let mut link_to_prover = link_to_code.into_os_string();
+    link_to_prover.push("/prover");
+    link_to_prover.into()
+}

zk_toolbox/crates/zk_inception/src/commands/prover/compressor_keys.rs

@@ -1,11 +1,11 @@
 use anyhow::Context;
 use common::{
-    check_prerequisites, cmd::Cmd, config::global_config, spinner::Spinner, WGET_PREREQUISITES,
+    check_prerequisites, cmd::Cmd, config::global_config, spinner::Spinner, WGET_PREREQUISITE,
 };
-use config::{EcosystemConfig, GeneralConfig};
+use config::{get_link_to_prover, EcosystemConfig, GeneralConfig};
 use xshell::{cmd, Shell};
 
-use super::{args::compressor_keys::CompressorKeysArgs, utils::get_link_to_prover};
+use super::args::compressor_keys::CompressorKeysArgs;
 use crate::messages::{
     MSG_CHAIN_NOT_FOUND_ERR, MSG_DOWNLOADING_SETUP_COMPRESSOR_KEY_SPINNER,
     MSG_PROOF_COMPRESSOR_CONFIG_NOT_FOUND_ERR, MSG_SETUP_KEY_PATH_ERROR,
@@ -37,7 +37,7 @@ pub(crate) fn download_compressor_key(
     general_config: &mut GeneralConfig,
     path: &str,
 ) -> anyhow::Result<()> {
-    check_prerequisites(shell, &WGET_PREREQUISITES, false);
+    check_prerequisites(shell, &WGET_PREREQUISITE, false);
     let spinner = Spinner::new(MSG_DOWNLOADING_SETUP_COMPRESSOR_KEY_SPINNER);
     let mut compressor_config: zksync_config::configs::FriProofCompressorConfig = general_config
         .proof_compressor_config

zk_toolbox/crates/zk_inception/src/commands/prover/gcs.rs

@@ -1,4 +1,4 @@
-use common::{check_prerequisites, cmd::Cmd, logger, spinner::Spinner, GCLOUD_PREREQUISITES};
+use common::{check_prerequisites, cmd::Cmd, logger, spinner::Spinner, GCLOUD_PREREQUISITE};
 use xshell::{cmd, Shell};
 use zksync_config::{configs::object_store::ObjectStoreMode, ObjectStoreConfig};
 
@@ -14,7 +14,7 @@ pub(crate) fn create_gcs_bucket(
     shell: &Shell,
     config: ProofStorageGCSCreateBucket,
 ) -> anyhow::Result<ObjectStoreConfig> {
-    check_prerequisites(shell, &GCLOUD_PREREQUISITES, false);
+    check_prerequisites(shell, &GCLOUD_PREREQUISITE, false);
 
     let bucket_name = config.bucket_name;
     let location = config.location;

zk_toolbox/crates/zk_inception/src/commands/prover/init.rs

@@ -2,13 +2,17 @@ use std::path::PathBuf;
 
 use anyhow::Context;
 use common::{
+    cmd::Cmd,
     config::global_config,
     db::{drop_db_if_exists, init_db, migrate_db, DatabaseConfig},
     logger,
     spinner::Spinner,
 };
-use config::{copy_configs, set_prover_database, traits::SaveConfigWithBasePath, EcosystemConfig};
-use xshell::Shell;
+use config::{
+    copy_configs, get_link_to_prover, set_prover_database, traits::SaveConfigWithBasePath,
+    EcosystemConfig,
+};
+use xshell::{cmd, Shell};
 use zksync_config::{configs::object_store::ObjectStoreMode, ObjectStoreConfig};
 
 use super::{
@@ -19,6 +23,7 @@ use super::{
     setup_keys,
 };
 use crate::{
+    commands::prover::args::init::ProofStorageFileBacked,
     consts::{PROVER_MIGRATIONS, PROVER_STORE_MAX_RETRIES},
     messages::{
         MSG_CHAIN_NOT_FOUND_ERR, MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR,
@@ -105,13 +110,11 @@ fn get_object_store_config(
     config: Option<ProofStorageConfig>,
 ) -> anyhow::Result<Option<ObjectStoreConfig>> {
     let object_store = match config {
-        Some(ProofStorageConfig::FileBacked(config)) => Some(ObjectStoreConfig {
-            mode: ObjectStoreMode::FileBacked {
-                file_backed_base_path: config.proof_store_dir,
-            },
-            max_retries: PROVER_STORE_MAX_RETRIES,
-            local_mirror_path: None,
-        }),
+        Some(ProofStorageConfig::FileBacked(config)) => Some(init_file_backed_proof_storage(
+            shell,
+            &EcosystemConfig::from_file(shell)?,
+            config,
+        )?),
         Some(ProofStorageConfig::GCS(config)) => Some(ObjectStoreConfig {
             mode: ObjectStoreMode::GCSWithCredentialFile {
                 bucket_base_url: config.bucket_base_url,
@@ -154,3 +157,27 @@ async fn initialize_prover_database(
 
     Ok(())
 }
+
+fn init_file_backed_proof_storage(
+    shell: &Shell,
+    ecosystem_config: &EcosystemConfig,
+    config: ProofStorageFileBacked,
+) -> anyhow::Result<ObjectStoreConfig> {
+    let proof_store_dir = config.proof_store_dir;
+    let prover_path = get_link_to_prover(ecosystem_config);
+
+    let proof_store_dir = prover_path.join(proof_store_dir).join("witness_inputs");
+
+    let cmd = Cmd::new(cmd!(shell, "mkdir -p {proof_store_dir}"));
+    cmd.run()?;
+
+    let object_store_config = ObjectStoreConfig {
+        mode: ObjectStoreMode::FileBacked {
+            file_backed_base_path: proof_store_dir.into_os_string().into_string().unwrap(),
+        },
+        max_retries: PROVER_STORE_MAX_RETRIES,
+        local_mirror_path: None,
+    };
+
+    Ok(object_store_config)
+}

zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs

@@ -14,7 +14,6 @@ mod init;
 mod init_bellman_cuda;
 mod run;
 mod setup_keys;
-mod utils;
 
 #[derive(Subcommand, Debug)]
 pub enum ProverCommands {

zk_toolbox/crates/zk_inception/src/commands/prover/run.rs

@@ -2,13 +2,10 @@ use std::path::PathBuf;
 
 use anyhow::{anyhow, Context};
 use common::{check_prerequisites, cmd::Cmd, config::global_config, logger, GPU_PREREQUISITES};
-use config::EcosystemConfig;
+use config::{get_link_to_prover, EcosystemConfig};
 use xshell::{cmd, Shell};
 
-use super::{
-    args::run::{ProverComponent, ProverRunArgs},
-    utils::get_link_to_prover,
-};
+use super::args::run::{ProverComponent, ProverRunArgs};
 use crate::messages::{
     MSG_BELLMAN_CUDA_DIR_ERR, MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR,
     MSG_RUNNING_COMPRESSOR, MSG_RUNNING_COMPRESSOR_ERR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_ERR,

zk_toolbox/crates/zk_inception/src/commands/prover/setup_keys.rs

@@ -1,12 +1,10 @@
 use anyhow::Ok;
 use common::{
-    check_prerequisites, cmd::Cmd, logger, spinner::Spinner, GCLOUD_PREREQUISITES,
-    GPU_PREREQUISITES,
+    check_prerequisites, cmd::Cmd, logger, spinner::Spinner, GCLOUD_PREREQUISITE, GPU_PREREQUISITES,
 };
-use config::EcosystemConfig;
+use config::{get_link_to_prover, EcosystemConfig};
 use xshell::{cmd, Shell};
 
-use super::utils::get_link_to_prover;
 use crate::{
     commands::prover::args::setup_keys::{Mode, Region, SetupKeysArgs},
     messages::{MSG_GENERATING_SK_SPINNER, MSG_SK_GENERATED},
@@ -33,7 +31,7 @@ pub(crate) async fn run(args: SetupKeysArgs, shell: &Shell) -> anyhow::Result<()
         spinner.finish();
         logger::outro(MSG_SK_GENERATED);
     } else {
-        check_prerequisites(shell, &GCLOUD_PREREQUISITES, false);
+        check_prerequisites(shell, &GCLOUD_PREREQUISITE, false);
 
         let link_to_setup_keys = get_link_to_prover(&ecosystem_config).join("data/keys");
         let path_to_keys_buckets =

zk_toolbox/crates/zk_supervisor/README.md

@@ -32,7 +32,9 @@ This document contains the help content for the `zk_supervisor` command-line pro
 - [`zk_supervisor fmt rustfmt`↴](#zk_supervisor-fmt-rustfmt)
 - [`zk_supervisor fmt contract`↴](#zk_supervisor-fmt-contract)
 - [`zk_supervisor fmt prettier`↴](#zk_supervisor-fmt-prettier)
-- [`zk_supervisor prover-version`↴](#zk_supervisor-prover-version)
+- [`zk_supervisor prover info`↴](#zk_supervisor-prover-info)
+- [`zk_supervisor prover insert-version`↴](#zk_supervisor-prover-insert-version)
+- [`zk_supervisor prover insert-batch`↴](#zk_supervisor-prover-insert-batch)
 
 ## `zk_supervisor`
 
@@ -348,11 +350,35 @@ Format code
 
   Possible values: `md`, `sol`, `js`, `ts`, `rs`
 
-## `zk_supervisor prover-version`
+## `zk_supervisor prover info`
 
-Protocol version used by provers
+Prints prover protocol version, snark wrapper and prover database URL
 
-**Usage:** `zk_supervisor prover-version`
+**Usage:** `zk_supervisor prover info`
+
+## `zk_supervisor prover insert-version`
+
+Inserts protocol version into prover database.
+
+**Usage:** `zk_supervisor prover insert-version [OPTIONS]`
+
+###### **Options:**
+
+- `--version <VERSION>` — Protocol version in semantic format(`x.y.z`). Major version should be 0.
+- `--snark-wrapper <SNARK_WRAPPER>` — Snark wrapper hash.
+- `--default` - use default values for protocol version and snark wrapper hash (the ones found in zksync-era).
+
+## `zk_supervisor prover insert-batch`
+
+Inserts batch into prover database.
+
+**Usage:** `zk_supervisor prover insert-batch`
+
+###### **Options:**
+
+- `--number <BATCH_NUMBER>` — Number of the batch to insert.
+- `--version <VERSION>` — Protocol version in semantic format(`x.y.z`). Major version should be 0.
+- `--default` - use default value for protocol version (the one found in zksync-era).
 
 <hr/>
 

zk_toolbox/crates/zk_supervisor/src/commands/database/reset.rs

@@ -20,14 +20,14 @@ pub async fn run(shell: &Shell, args: DatabaseCommonArgs) -> anyhow::Result<()>
         return Ok(());
     }
 
-    let ecoseystem_config = EcosystemConfig::from_file(shell)?;
+    let ecosystem_config = EcosystemConfig::from_file(shell)?;
 
     logger::info(msg_database_info(MSG_DATABASE_RESET_GERUND));
 
     let dals = get_dals(shell, &args.selected_dals, &args.urls)?;
     for dal in dals {
         logger::info(msg_database_loading(MSG_DATABASE_RESET_GERUND, &dal.path));
-        reset_database(shell, ecoseystem_config.link_to_code.clone(), dal).await?;
+        reset_database(shell, ecosystem_config.link_to_code.clone(), dal).await?;
     }
 
     logger::outro(msg_database_success(MSG_DATABASE_RESET_PAST));

zk_toolbox/crates/zk_supervisor/src/commands/mod.rs

@@ -4,6 +4,6 @@ pub mod database;
 pub mod fmt;
 pub mod lint;
 pub(crate) mod lint_utils;
-pub mod prover_version;
+pub mod prover;
 pub mod snapshot;
 pub mod test;

zk_toolbox/crates/zk_supervisor/src/commands/prover/args/insert_batch.rs

@@ -0,0 +1,40 @@
+use clap::Parser;
+
+#[derive(Debug, Parser)]
+pub struct InsertBatchArgs {
+    #[clap(long)]
+    pub number: Option<u32>,
+    #[clap(long, default_value = "false")]
+    pub default: bool,
+    #[clap(long)]
+    pub version: Option<String>,
+}
+
+#[derive(Debug)]
+pub struct InsertBatchArgsFinal {
+    pub number: u32,
+    pub version: String,
+}
+
+impl InsertBatchArgs {
+    pub(crate) fn fill_values_with_prompts(self, era_version: String) -> InsertBatchArgsFinal {
+        let number = self.number.unwrap_or_else(|| {
+            common::Prompt::new("Enter the number of the batch to insert").ask()
+        });
+
+        if self.default {
+            return InsertBatchArgsFinal {
+                number,
+                version: era_version,
+            };
+        }
+
+        let version = self.version.unwrap_or_else(|| {
+            common::Prompt::new("Enter the version of the batch to insert")
+                .default(&era_version)
+                .ask()
+        });
+
+        InsertBatchArgsFinal { number, version }
+    }
+}

zk_toolbox/crates/zk_supervisor/src/commands/prover/args/insert_version.rs

@@ -0,0 +1,49 @@
+use clap::Parser;
+
+#[derive(Debug, Parser)]
+pub struct InsertVersionArgs {
+    #[clap(long, default_value = "false")]
+    pub default: bool,
+    #[clap(long)]
+    pub version: Option<String>,
+    #[clap(long)]
+    pub snark_wrapper: Option<String>,
+}
+
+#[derive(Debug)]
+pub struct InsertVersionArgsFinal {
+    pub snark_wrapper: String,
+    pub version: String,
+}
+
+impl InsertVersionArgs {
+    pub(crate) fn fill_values_with_prompts(
+        self,
+        era_version: String,
+        snark_wrapper: String,
+    ) -> InsertVersionArgsFinal {
+        if self.default {
+            return InsertVersionArgsFinal {
+                snark_wrapper,
+                version: era_version,
+            };
+        }
+
+        let version = self.version.unwrap_or_else(|| {
+            common::Prompt::new("Enter the version of the protocol to insert")
+                .default(&era_version)
+                .ask()
+        });
+
+        let snark_wrapper = self.snark_wrapper.unwrap_or_else(|| {
+            common::Prompt::new("Enter the snark wrapper of the protocol to insert")
+                .default(&snark_wrapper)
+                .ask()
+        });
+
+        InsertVersionArgsFinal {
+            snark_wrapper,
+            version,
+        }
+    }
+}