From 8b283feba8bd6f7b420d40877cadc8dc087d1938 Mon Sep 17 00:00:00 2001 From: "juan.mv" Date: Fri, 20 Oct 2023 10:15:56 -0300 Subject: [PATCH 1/4] Make sed run on mac and linux --- prover/setup.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/prover/setup.sh b/prover/setup.sh index d2ae3ce1edb..c9fcf9f6227 100755 --- a/prover/setup.sh +++ b/prover/setup.sh @@ -6,10 +6,13 @@ if [[ -z "${ZKSYNC_HOME}" ]]; then exit 1 fi -sed -i '' 's/^proof_sending_mode=.*$/proof_sending_mode="OnlyRealProofs"/' ../etc/env/base/eth_sender.toml -sed -i '' 's/^proof_loading_mode=.*$/proof_loading_mode="FriProofFromGcs"/' ../etc/env/base/eth_sender.toml -sed -i '' 's/^setup_data_path=.*$/setup_data_path="vk_setup_data_generator_server_fri\/data\/"/' ../etc/env/base/fri_prover.toml -sed -i '' 's/^universal_setup_path=.*$/universal_setup_path="..\/keys\/setup\/setup_2^26.key"/' ../etc/env/base/fri_proof_compressor.toml +sed -i.backup 's/^proof_sending_mode=.*$/proof_sending_mode="OnlyRealProofs"/' ../etc/env/base/eth_sender.toml +sed -i.backup 's/^proof_loading_mode=.*$/proof_loading_mode="FriProofFromGcs"/' ../etc/env/base/eth_sender.toml +rm ../etc/env/base/eth_sender.toml.backup +sed -i.backup 's/^setup_data_path=.*$/setup_data_path="vk_setup_data_generator_server_fri\/data\/"/' ../etc/env/base/fri_prover.toml +rm ../etc/env/base/fri_prover.toml +sed -i.backup 's/^universal_setup_path=.*$/universal_setup_path="..\/keys\/setup\/setup_2^26.key"/' ../etc/env/base/fri_proof_compressor.toml +rm ../etc/env/base/fri_proof_compressor.toml zk config compile dev From 8c2b1038cb9d59830e5a0ce7bfb29f1f6872a284 Mon Sep 17 00:00:00 2001 From: "juan.mv" Date: Fri, 20 Oct 2023 10:41:52 -0300 Subject: [PATCH 2/4] Add pipeline to readme; Move steps --- prover/prover_fri/README.md | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/prover/prover_fri/README.md b/prover/prover_fri/README.md index 5fbcfe24ecc..cd9ace0e6b9 100644 --- a/prover/prover_fri/README.md +++ b/prover/prover_fri/README.md @@ -10,6 +10,18 @@ ## Proving a block using CPU prover locally +### Overview of the pipeline + +These are the main components to this process: + - Sequencer + - Prover gateway + - Witness + - Prover + - Compressor +All of them will be sharing information through a database. +The general idea is that the sequencer will produce blocks and the gateway will place them into the database to be proven. Then, the rest of the components will pull jobs from the database and do their part of the pipeline. + +### Block proving Below steps can be used to prove a block on local machine using CPU prover. This is useful for debugging and testing Machine specs: @@ -27,16 +39,16 @@ The whole setup below will NOT work if you don't have this environment variable depends on it. 1. Install the correct nightly version using command: `rustup install nightly-2023-07-21` -2. Generate the cpu setup data (no need to regenerate if it's already there). This will consume around 300Gb of disk. - For this, run - +2. Initialize DB and run migrations. Go into the root of the repository, then run ``` - ./setup.sh + zk init ``` -3. Initialize DB and run migrations. Go into the root of the repository, then run +3. Generate the cpu setup data (no need to regenerate if it's already there). This will consume around 300Gb of disk. + For this, run + ``` - zk init + ./setup.sh ``` For the following steps, we recommend using `tmux` to run every command on a separate session, so you can attach to and From b82763aff712466ae189f96c49ed2fe1a0156225 Mon Sep 17 00:00:00 2001 From: igamigo Date: Fri, 20 Oct 2023 13:56:31 -0300 Subject: [PATCH 3/4] Update README.md --- prover/prover_fri/README.md | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/prover/prover_fri/README.md b/prover/prover_fri/README.md index cd9ace0e6b9..415d69d975c 100644 --- a/prover/prover_fri/README.md +++ b/prover/prover_fri/README.md @@ -21,22 +21,25 @@ These are the main components to this process: All of them will be sharing information through a database. The general idea is that the sequencer will produce blocks and the gateway will place them into the database to be proven. Then, the rest of the components will pull jobs from the database and do their part of the pipeline. -### Block proving -Below steps can be used to prove a block on local machine using CPU prover. This is useful for debugging and testing -Machine specs: - -- CPU: At least 8 physical cores -- RAM: 60GB of RAM(if you have lower RAM machine enable swap) -- Disk: 400GB of free disk +### Prerequisites -Before starting, make sure you go into the root of the repository, then run +Make sure these dependencies are installed and available on your machine: [Installing dependencies](./setup-dev.md) +Once that is done, before starting, make sure you go into the root of the repository, then run ``` export ZKSYNC_HOME=$(pwd) ``` -The whole setup below will NOT work if you don't have this environment variable properly set, as the entirety of `zk` -depends on it. +The whole setup below will NOT work if you don't have this environment variable properly set, as the entirety of the `zk` +tool depends on it. + +### Block proving with CPU +Below steps can be used to prove a block on local machine using CPU prover. This is useful for debugging and testing +Machine specs: + +- CPU: At least 8 physical cores +- RAM: 60GB of RAM(if you have lower RAM machine enable swap) +- Disk: 400GB of free disk 1. Install the correct nightly version using command: `rustup install nightly-2023-07-21` 2. Initialize DB and run migrations. Go into the root of the repository, then run @@ -130,14 +133,16 @@ which means the proof for the block was verified on L1. ## Proving a block using GPU prover locally -Below steps can be used to prove a block on local machine using GPU prover, It requires Cuda 12.0 installation as -pre-requisite. This is useful for debugging and testing Machine specs: +Below steps can be used to prove a block on local machine using GPU prover. +Running a GPU prover requires a Cuda 12.0 installation as a pre-requisite, alongside these machine specs: - CPU: At least 8 physical cores - RAM: 16GB of RAM(if you have lower RAM machine enable swap) - Disk: 30GB of free disk - GPU: 1x Nvidia L4/T4 with 16GB of GPU RAM + + 1. Install the correct nightly version using command: `rustup install nightly-2023-07-21` 2. Generate the gpu setup data (no need to regenerate if it's already there). This will consume around 300Gb of disk. Use these commands: From a91dbd3daa3c9e0afb0f6e03bf786e2e2b7e6190 Mon Sep 17 00:00:00 2001 From: Nacho Date: Fri, 20 Oct 2023 14:21:08 -0300 Subject: [PATCH 4/4] gpu prover docs --- prover/prover_fri/README.md | 138 ++++++++++++++++++------------------ prover/setup_gpu.sh | 27 +++++++ 2 files changed, 95 insertions(+), 70 deletions(-) create mode 100755 prover/setup_gpu.sh diff --git a/prover/prover_fri/README.md b/prover/prover_fri/README.md index 415d69d975c..30f3cd60fad 100644 --- a/prover/prover_fri/README.md +++ b/prover/prover_fri/README.md @@ -18,8 +18,9 @@ These are the main components to this process: - Witness - Prover - Compressor -All of them will be sharing information through a database. -The general idea is that the sequencer will produce blocks and the gateway will place them into the database to be proven. Then, the rest of the components will pull jobs from the database and do their part of the pipeline. + +All of them will be sharing information through a SQL database. +The general idea is that the sequencer will produce blocks and the gateway will place them into the database to be proven. Then, the rest of the components will pull jobs from the database and do their part of the pipeline. ### Prerequisites @@ -31,14 +32,14 @@ export ZKSYNC_HOME=$(pwd) ``` The whole setup below will NOT work if you don't have this environment variable properly set, as the entirety of the `zk` -tool depends on it. +CLI tool depends on it. ### Block proving with CPU Below steps can be used to prove a block on local machine using CPU prover. This is useful for debugging and testing Machine specs: - CPU: At least 8 physical cores -- RAM: 60GB of RAM(if you have lower RAM machine enable swap) +- RAM: 60GB of RAM (if you have lower RAM machine enable swap) - Disk: 400GB of free disk 1. Install the correct nightly version using command: `rustup install nightly-2023-07-21` @@ -51,7 +52,7 @@ Machine specs: For this, run ``` - ./setup.sh + ./prover/setup.sh ``` For the following steps, we recommend using `tmux` to run every command on a separate session, so you can attach to and @@ -89,7 +90,67 @@ monitor logs for each one. zk f cargo run --release --bin zksync_proof_fri_compressor ``` -After this is done, the server should have at least three blocks, you can see the first one by running +## Proving a block using GPU prover locally + +Below steps can be used to prove a block on local machine using GPU prover. +Running a GPU prover requires a Cuda 12.0 installation as a pre-requisite, alongside these machine specs: + +- CPU: At least 8 physical cores +- RAM: 16GB of RAM(if you have lower RAM machine enable swap) +- Disk: 30GB of free disk +- GPU: 1x Nvidia L4/T4 with 16GB of GPU RAM + +1. Install the correct nightly version using command: `rustup install nightly-2023-07-21` +2. Initialize DB and run migrations: `zk init` +3. Generate the GPU setup data (no need to regenerate if it's already there). This will consume around 300Gb of disk. + For this, run + + ``` + ./prover/setup_gpu.sh + ``` + +4. Run the sequencer/operator. In the root of the repository: + + ``` + zk server --components=api,eth,tree,state_keeper,housekeeper,proof_data_handler + ``` + + to produce blocks to be proven + +5. Run prover gateway to fetch blocks to be proven from server: + + ``` + zk f cargo run --release --bin zksync_prover_fri_gateway + ``` + +6. Run 4 witness generators to generate witness for each round: + + ``` + API_PROMETHEUS_LISTENER_PORT=3116 zk f cargo run --release --bin zksync_witness_generator -- --round=basic_circuits + API_PROMETHEUS_LISTENER_PORT=3117 zk f cargo run --release --bin zksync_witness_generator -- --round=leaf_aggregation + API_PROMETHEUS_LISTENER_PORT=3118 zk f cargo run --release --bin zksync_witness_generator -- --round=node_aggregation + API_PROMETHEUS_LISTENER_PORT=3119 zk f cargo run --release --bin zksync_witness_generator -- --round=scheduler + ``` + +7. Run prover to perform actual proving: `zk f cargo run --features "gpu" --release --bin zksync_prover_fri` + +8. Run 5 witness vector generators to feed jobs to GPU prover: + + ``` + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3416 zk f cargo run --release --bin zksync_witness_vector_generator + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3417 zk f cargo run --release --bin zksync_witness_vector_generator + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3418 zk f cargo run --release --bin zksync_witness_vector_generator + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3419 zk f cargo run --release --bin zksync_witness_vector_generator + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3420 zk f cargo run --release --bin zksync_witness_vector_generator + ``` + +9. Finally, run proof compressor to compress the proof to be sent on L1: + `zk f cargo run --release --bin zksync_proof_fri_compressor` + + +## Checking the status of the prover + +Once everything is running (either with the CPU or GPU prover), the server should have at least three blocks, and you can see the first one by running ``` curl -X POST -H 'content-type: application/json' localhost:3050 -d '{"jsonrpc": "2.0", "id": 1, "method": "zks_getBlockDetails", "params": [0]}' @@ -106,7 +167,7 @@ You can follow the status of this pipeline by running zk status prover ``` -This will take a while (around an hour and a half on my machine), you can check on it once in a while. A succesful flow +This might take a while (around an hour and a half on my machine using the CPU prover), you can check on it once in a while. A succesful flow should output something like ``` @@ -131,69 +192,6 @@ L1 state: block verified: 1, block committed: 1 which means the proof for the block was verified on L1. -## Proving a block using GPU prover locally - -Below steps can be used to prove a block on local machine using GPU prover. -Running a GPU prover requires a Cuda 12.0 installation as a pre-requisite, alongside these machine specs: - -- CPU: At least 8 physical cores -- RAM: 16GB of RAM(if you have lower RAM machine enable swap) -- Disk: 30GB of free disk -- GPU: 1x Nvidia L4/T4 with 16GB of GPU RAM - - - -1. Install the correct nightly version using command: `rustup install nightly-2023-07-21` -2. Generate the gpu setup data (no need to regenerate if it's already there). This will consume around 300Gb of disk. - Use these commands: - - ```markdown - for i in {1..13}; do zk f cargo run --features "gpu" --release --bin zksync_setup_data_generator_fri -- - --numeric-circuit $i --is_base_layer done - - for i in {1..15}; do zk f cargo run --features "gpu" --release --bin zksync_setup_data_generator_fri -- - --numeric-circuit $i done - ``` - -3. Initialize DB and run migrations: `zk init` - -4. Override the following configuration in your `dev.env`: - - ``` - ETH_SENDER_SENDER_PROOF_SENDING_MODE=OnlyRealProofs - ETH_SENDER_SENDER_PROOF_LOADING_MODE=FriProofFromGcs - OBJECT_STORE_FILE_BACKED_BASE_PATH=/path/to/server/artifacts - PROVER_OBJECT_STORE_FILE_BACKED_BASE_PATH=/path/to/prover/artifacts - FRI_PROVER_SETUP_DATA_PATH=/path/to/above-generated/gpu-setup-data - ``` - -5. Run server `zk server --components=api,eth,tree,state_keeper,housekeeper,proof_data_handler` to produce blocks to be - proven -6. Run prover gateway to fetch blocks to be proven from server : - `zk f cargo run --release --bin zksync_prover_fri_gateway` -7. Run 4 witness generators to generate witness for each round: - - ``` - API_PROMETHEUS_LISTENER_PORT=3116 zk f cargo run --release --bin zksync_witness_generator -- --round=basic_circuits - API_PROMETHEUS_LISTENER_PORT=3117 zk f cargo run --release --bin zksync_witness_generator -- --round=leaf_aggregation - API_PROMETHEUS_LISTENER_PORT=3118 zk f cargo run --release --bin zksync_witness_generator -- --round=node_aggregation - API_PROMETHEUS_LISTENER_PORT=3119 zk f cargo run --release --bin zksync_witness_generator -- --round=scheduler - ``` - -8. Run prover to perform actual proving: `zk f cargo run --features "gpu" --release --bin zksync_prover_fri` -9. Run 5 witness vector generators to feed jobs to GPU prover: - - ``` - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3416 zk f cargo run --release --bin zksync_witness_vector_generator - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3417 zk f cargo run --release --bin zksync_witness_vector_generator - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3418 zk f cargo run --release --bin zksync_witness_vector_generator - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3419 zk f cargo run --release --bin zksync_witness_vector_generator - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3420 zk f cargo run --release --bin zksync_witness_vector_generator - ``` - -10. Finally, run proof compressor to compress the proof to be sent on L1: - `zk f cargo run --release --bin zksync_proof_fri_compressor` - ## Performing circuit upgrade Performing circuit upgrade requires crypto library to be updated and generating new setup data, verification key, diff --git a/prover/setup_gpu.sh b/prover/setup_gpu.sh new file mode 100755 index 00000000000..e1f31158b27 --- /dev/null +++ b/prover/setup_gpu.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# This script sets up the necessary data needed by the GPU FRI prover to be used locally. + +if [[ -z "${ZKSYNC_HOME}" ]]; then + echo "Environment variable ZKSYNC_HOME is not set. Make sure it's set and pointing to the root of this repository" + exit 1 +fi + +sed -i.backup 's/^proof_sending_mode=.*$/proof_sending_mode="OnlyRealProofs"/' ../etc/env/base/eth_sender.toml +sed -i.backup 's/^proof_loading_mode=.*$/proof_loading_mode="FriProofFromGcs"/' ../etc/env/base/eth_sender.toml +rm ../etc/env/base/eth_sender.toml.backup +sed -i.backup 's/^setup_data_path=.*$/setup_data_path="vk_setup_data_generator_server_fri\/data\/"/' ../etc/env/base/fri_prover.toml +rm ../etc/env/base/fri_prover.toml +sed -i.backup 's/^universal_setup_path=.*$/universal_setup_path="..\/keys\/setup\/setup_2^26.key"/' ../etc/env/base/fri_proof_compressor.toml +rm ../etc/env/base/fri_proof_compressor.toml + +zk config compile dev + +for i in {1..13} +do + zk f cargo run --features "gpu" --release --bin zksync_setup_data_generator_fri -- --numeric-circuit $i --is_base_layer +done + +for i in {1..15} +do + zk f cargo run --features "gpu" --release --bin zksync_setup_data_generator_fri -- --numeric-circuit $i +done