feat: Move prover data to /home/popzxc/workspace/current/zksync-era/p…

…rover/data (#2778)

⚠️ this change is non-destructive globally (e.g. no changes in
deployments/configs are needed), but for local deployment if you have
setup keys locally, you need to move them to the new folder:

```
cd $ZKSYNC_HOME
mv prover/crates/bin/vk_setup_data_generator_server_fri/data/setup* prover/data/keys
rmdir prover/crates/bin/vk_setup_data_generator_server_fri/data
```

## What

- Moves prover data (`data`/`historical_data`) from
`prover/crates/bin/vk_setup_data_generator_server_fri` to `prover/data`
- Updates all the relevant paths
- Adds some minimal documentation to `prover/data/README.md`

## Why

- More intuitive
- Simplifies refactoring of the workspace: no need to worry about layout
of data when renaming crates/changing structure/etc.

.dockerignore

@@ -39,7 +39,7 @@ contracts/.git
 !etc/multivm_bootloaders
 !cargo
 !bellman-cuda
-!prover/crates/bin/vk_setup_data_generator_server_fri/data/
+!prover/data/
 !.github/release-please/manifest.json
 
 !etc/env/file_based

.gitignore

@@ -110,6 +110,7 @@ hyperchain-*.yml
 
 # Prover keys that should not be commited
 prover/crates/bin/vk_setup_data_generator_server_fri/data/setup_*
+prover/data/keys/setup_*
 
 # Zk Toolbox
 chains/era/configs/*

core/lib/env_config/src/fri_prover.rs

@@ -32,7 +32,7 @@ mod tests {
 
     fn expected_config() -> FriProverConfig {
         FriProverConfig {
-            setup_data_path: "vk_setup_data_generator_server_fri/data".to_string(),
+            setup_data_path: "prover/data/keys".to_string(),
             prometheus_port: 3315,
             max_attempts: 10,
             generation_timeout_in_secs: 300,
@@ -68,7 +68,7 @@ mod tests {
     fn from_env() {
         let mut lock = MUTEX.lock();
         let config = r#"
-            FRI_PROVER_SETUP_DATA_PATH="vk_setup_data_generator_server_fri/data"
+            FRI_PROVER_SETUP_DATA_PATH="prover/data/keys"
             FRI_PROVER_PROMETHEUS_PORT="3315"
             FRI_PROVER_MAX_ATTEMPTS="10"
             FRI_PROVER_GENERATION_TIMEOUT_IN_SECS="300"

docker/proof-fri-gpu-compressor/Dockerfile

@@ -37,7 +37,7 @@ FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04
 RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
 
 # copy VK required for proof wrapping
-COPY prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY prover/data/keys/ /prover/data/keys
 
 COPY setup_2\^24.key /setup_2\^24.key
 

docker/prover-fri-gateway/Dockerfile

@@ -11,7 +11,7 @@ FROM debian:bookworm-slim
 RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
 
 # copy VK required for proof wrapping
-COPY prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY prover/data/keys/ /prover/data/keys/
 
 COPY --from=builder /usr/src/zksync/prover/target/release/zksync_prover_fri_gateway /usr/bin/
 

docker/prover-gpu-fri-gar/Dockerfile

@@ -9,7 +9,7 @@ COPY *.bin /
 RUN apt-get update && apt-get install -y libpq5 ca-certificates openssl && rm -rf /var/lib/apt/lists/*
 
 # copy finalization hints required for assembly generation
-COPY --from=prover prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY --from=prover prover/data/keys/ /prover/data/keys/
 COPY --from=prover /usr/bin/zksync_prover_fri /usr/bin/
 
 ENTRYPOINT ["zksync_prover_fri"]

docker/prover-gpu-fri/Dockerfile

@@ -31,7 +31,7 @@ FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04
 RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
 
 # copy finalization hints required for assembly generation
-COPY prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY prover/data/keys/ /prover/data/keys/
 
 COPY --from=builder /usr/src/zksync/prover/target/release/zksync_prover_fri /usr/bin/
 

docker/witness-generator/Dockerfile

@@ -13,7 +13,7 @@ FROM debian:bookworm-slim
 
 RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
 
-COPY prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY prover/data/keys/ /prover/data/keys/
 
 COPY --from=builder /usr/src/zksync/prover/target/release/zksync_witness_generator /usr/bin/
 

docker/witness-vector-generator/Dockerfile

@@ -12,7 +12,7 @@ FROM debian:bookworm-slim
 RUN apt-get update && apt-get install -y curl libpq5 ca-certificates && rm -rf /var/lib/apt/lists/*
 
 # copy finalization hints required for witness vector generation
-COPY prover/crates/bin/vk_setup_data_generator_server_fri/data/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
+COPY prover/data/keys/ /prover/crates/bin/vk_setup_data_generator_server_fri/data/
 
 COPY --from=builder /usr/src/zksync/prover/target/release/zksync_witness_vector_generator /usr/bin/
 

docs/guides/advanced/15_prover_keys.md

@@ -118,9 +118,9 @@ friendly hash function (currently Poseidon2).
 [recursive_circuit_list]:
   https://github.com/matter-labs/era-zkevm_test_harness/blob/3cd647aa57fc2e1180bab53f7a3b61ec47502a46/circuit_definitions/src/circuit_definitions/recursion_layer/mod.rs#L29
 [verification_key_list]:
-  https://github.com/matter-labs/zksync-era/tree/boojum-integration/prover/vk_setup_data_generator_server_fri/data
+  https://github.com/matter-labs/zksync-era/tree/6d18061df4a18803d3c6377305ef711ce60317e1/prover/data/keys
 [env_variables_for_hash]:
-  https://github.com/matter-labs/zksync-era/blob/boojum-integration/etc/env/base/contracts.toml#L44
+  https://github.com/matter-labs/zksync-era/blob/6d18061df4a18803d3c6377305ef711ce60317e1/etc/env/base/contracts.toml#L61
 [prover_setup_data]:
   https://github.com/matter-labs/zksync-era/blob/d2ca29bf20b4ec2d9ec9e327b4ba6b281d9793de/prover/vk_setup_data_generator_server_fri/src/lib.rs#L61
 [verifier_computation]:

etc/env/base/fri_prover.toml

@@ -1,5 +1,5 @@
 [fri_prover]
-setup_data_path = "crates/bin/vk_setup_data_generator_server_fri/data"
+setup_data_path = "data/keys"
 prometheus_port = 3315
 max_attempts = 10
 generation_timeout_in_secs = 600

etc/env/file_based/general.yaml

@@ -140,7 +140,7 @@ prover:
     file_backed:
       file_backed_base_path: artifacts
     max_retries: 10
-  setup_data_path: crates/bin/vk_setup_data_generator_server_fri/data
+  setup_data_path: data/keys
   prometheus_port: 3315
   max_attempts: 10
   generation_timeout_in_secs: 600

infrastructure/zk/src/prover_setup.ts

@@ -30,8 +30,7 @@ export async function setupProver(proverType: ProverType) {
         } else {
             env.modify(
                 'FRI_PROVER_SETUP_DATA_PATH',
-                `${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${process.env.ZKSYNC_ENV}/${
-                    proverType === ProverType.GPU ? 'gpu' : 'cpu'
+                `${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${process.env.ZKSYNC_ENV}/${proverType === ProverType.GPU ? 'gpu' : 'cpu'
                 }/`,
                 process.env.ENV_FILE!
             );
@@ -98,8 +97,7 @@ async function setupProverKeys(proverType: ProverType) {
 
     env.modify(
         'FRI_PROVER_SETUP_DATA_PATH',
-        `${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${process.env.ZKSYNC_ENV}/${
-            proverType === ProverType.GPU ? 'gpu' : 'cpu'
+        `${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${process.env.ZKSYNC_ENV}/${proverType === ProverType.GPU ? 'gpu' : 'cpu'
         }/`,
         process.env.ENV_FILE!
     );
@@ -204,7 +202,7 @@ async function downloadDefaultSetupKeys(proverType: ProverType, region: string)
     );
 
     await utils.spawn(
-        `cp -r ${process.env.ZKSYNC_HOME}/prover/vk_setup_data_generator_server_fri/data/* ${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${currentEnv}/${proverType}/`
+        `cp -r ${process.env.ZKSYNC_HOME}/prover/data/keys/* ${process.env.ZKSYNC_HOME}/etc/hyperchains/prover-keys/${currentEnv}/${proverType}/`
     );
 }
 

prover/crates/bin/vk_setup_data_generator_server_fri/src/keystore.rs

@@ -47,16 +47,14 @@ pub struct Keystore {
 fn get_base_path() -> PathBuf {
     let path = core_workspace_dir_or_current_dir();
 
-    let new_path = path.join("prover/crates/bin/vk_setup_data_generator_server_fri/data");
+    let new_path = path.join("data/keys");
     if new_path.exists() {
         return new_path;
     }
 
     let mut components = path.components();
     components.next_back().unwrap();
-    components
-        .as_path()
-        .join("prover/crates/bin/vk_setup_data_generator_server_fri/data")
+    components.as_path().join("data/keys")
 }
 
 impl Default for Keystore {

prover/crates/bin/vk_setup_data_generator_server_fri/src/utils.rs

@@ -132,7 +132,7 @@ mod tests {
     #[test]
     fn test_keyhash_generation() {
         let mut path_to_input = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap());
-        path_to_input.push("historical_data");
+        path_to_input.push("../../../data/historical_data");
 
         for entry in std::fs::read_dir(path_to_input.clone()).unwrap().flatten() {
             if entry.metadata().unwrap().is_dir() {

prover/data/README.md

@@ -0,0 +1,23 @@
+# Prover data directory
+
+This directory contains the data required to run provers.
+
+Currently, it has the following sub-directories:
+
+- [keys](./keys/): Data required for proof generation. This data is mapped to a single protocol version.
+- [historical_data](./historical_data/) Descriptors for the protocol versions used in the past.
+
+## Keys directory
+
+`keys` directory is used by various components in the prover subsystem, and it generally can contain two kinds of data:
+
+- Small static files, like commitments, finalization hints, or verification keys.
+- Big generated blobs, like setup keys.
+
+Small static files are committed to the repository. Big files are expected to be downloaded or generated on demand. Two
+important notices as of Sep 2024:
+
+- Path to setup keys can be overridden via configuration.
+- Proof compressor requires an universal setup file, named, for example, `setup_2^24.bin` or `setup_2^26.bin`. It's
+  handled separately from the rest of the keys, e.g. it has separate configuration variables, and can naturally occur in
+  the `$ZKSYNC_HOME/keys/setup` during development.

zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs

@@ -18,8 +18,8 @@ pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> {
         shell,
         "cargo run --features gpu --release --bin key_generator -- 
             generate-sk-gpu all --recompute-if-missing
-            --setup-path=crates/bin/vk_setup_data_generator_server_fri/data
-            --path={link_to_prover}/crates/bin/vk_setup_data_generator_server_fri/data"
+            --setup-path=data/keys
+            --path={link_to_prover}/data/keys"
     ));
     cmd.run()?;
     spinner.finish();

zk_toolbox/crates/zk_supervisor/src/commands/prover_version.rs

@@ -27,8 +27,7 @@ async fn get_protocol_version(shell: &Shell, link_to_prover: &Path) -> anyhow::R
 }
 
 async fn get_snark_wrapper(link_to_prover: &Path) -> anyhow::Result<String> {
-    let path =
-        link_to_prover.join("crates/bin/vk_setup_data_generator_server_fri/data/commitments.json");
+    let path = link_to_prover.join("data/keys/commitments.json");
     let file = fs::File::open(path).expect("Could not find commitments file in zksync-era");
     let json: serde_json::Value =
         serde_json::from_reader(file).expect("Could not parse commitments.json");