From a91dbd3daa3c9e0afb0f6e03bf786e2e2b7e6190 Mon Sep 17 00:00:00 2001 From: Nacho Date: Fri, 20 Oct 2023 14:21:08 -0300 Subject: [PATCH] gpu prover docs --- prover/prover_fri/README.md | 138 ++++++++++++++++++------------------ prover/setup_gpu.sh | 27 +++++++ 2 files changed, 95 insertions(+), 70 deletions(-) create mode 100755 prover/setup_gpu.sh diff --git a/prover/prover_fri/README.md b/prover/prover_fri/README.md index 415d69d975c1..30f3cd60fade 100644 --- a/prover/prover_fri/README.md +++ b/prover/prover_fri/README.md @@ -18,8 +18,9 @@ These are the main components to this process: - Witness - Prover - Compressor -All of them will be sharing information through a database. -The general idea is that the sequencer will produce blocks and the gateway will place them into the database to be proven. Then, the rest of the components will pull jobs from the database and do their part of the pipeline. + +All of them will be sharing information through a SQL database. +The general idea is that the sequencer will produce blocks and the gateway will place them into the database to be proven. Then, the rest of the components will pull jobs from the database and do their part of the pipeline. ### Prerequisites @@ -31,14 +32,14 @@ export ZKSYNC_HOME=$(pwd) ``` The whole setup below will NOT work if you don't have this environment variable properly set, as the entirety of the `zk` -tool depends on it. +CLI tool depends on it. ### Block proving with CPU Below steps can be used to prove a block on local machine using CPU prover. This is useful for debugging and testing Machine specs: - CPU: At least 8 physical cores -- RAM: 60GB of RAM(if you have lower RAM machine enable swap) +- RAM: 60GB of RAM (if you have lower RAM machine enable swap) - Disk: 400GB of free disk 1. Install the correct nightly version using command: `rustup install nightly-2023-07-21` @@ -51,7 +52,7 @@ Machine specs: For this, run ``` - ./setup.sh + ./prover/setup.sh ``` For the following steps, we recommend using `tmux` to run every command on a separate session, so you can attach to and @@ -89,7 +90,67 @@ monitor logs for each one. zk f cargo run --release --bin zksync_proof_fri_compressor ``` -After this is done, the server should have at least three blocks, you can see the first one by running +## Proving a block using GPU prover locally + +Below steps can be used to prove a block on local machine using GPU prover. +Running a GPU prover requires a Cuda 12.0 installation as a pre-requisite, alongside these machine specs: + +- CPU: At least 8 physical cores +- RAM: 16GB of RAM(if you have lower RAM machine enable swap) +- Disk: 30GB of free disk +- GPU: 1x Nvidia L4/T4 with 16GB of GPU RAM + +1. Install the correct nightly version using command: `rustup install nightly-2023-07-21` +2. Initialize DB and run migrations: `zk init` +3. Generate the GPU setup data (no need to regenerate if it's already there). This will consume around 300Gb of disk. + For this, run + + ``` + ./prover/setup_gpu.sh + ``` + +4. Run the sequencer/operator. In the root of the repository: + + ``` + zk server --components=api,eth,tree,state_keeper,housekeeper,proof_data_handler + ``` + + to produce blocks to be proven + +5. Run prover gateway to fetch blocks to be proven from server: + + ``` + zk f cargo run --release --bin zksync_prover_fri_gateway + ``` + +6. Run 4 witness generators to generate witness for each round: + + ``` + API_PROMETHEUS_LISTENER_PORT=3116 zk f cargo run --release --bin zksync_witness_generator -- --round=basic_circuits + API_PROMETHEUS_LISTENER_PORT=3117 zk f cargo run --release --bin zksync_witness_generator -- --round=leaf_aggregation + API_PROMETHEUS_LISTENER_PORT=3118 zk f cargo run --release --bin zksync_witness_generator -- --round=node_aggregation + API_PROMETHEUS_LISTENER_PORT=3119 zk f cargo run --release --bin zksync_witness_generator -- --round=scheduler + ``` + +7. Run prover to perform actual proving: `zk f cargo run --features "gpu" --release --bin zksync_prover_fri` + +8. Run 5 witness vector generators to feed jobs to GPU prover: + + ``` + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3416 zk f cargo run --release --bin zksync_witness_vector_generator + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3417 zk f cargo run --release --bin zksync_witness_vector_generator + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3418 zk f cargo run --release --bin zksync_witness_vector_generator + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3419 zk f cargo run --release --bin zksync_witness_vector_generator + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3420 zk f cargo run --release --bin zksync_witness_vector_generator + ``` + +9. Finally, run proof compressor to compress the proof to be sent on L1: + `zk f cargo run --release --bin zksync_proof_fri_compressor` + + +## Checking the status of the prover + +Once everything is running (either with the CPU or GPU prover), the server should have at least three blocks, and you can see the first one by running ``` curl -X POST -H 'content-type: application/json' localhost:3050 -d '{"jsonrpc": "2.0", "id": 1, "method": "zks_getBlockDetails", "params": [0]}' @@ -106,7 +167,7 @@ You can follow the status of this pipeline by running zk status prover ``` -This will take a while (around an hour and a half on my machine), you can check on it once in a while. A succesful flow +This might take a while (around an hour and a half on my machine using the CPU prover), you can check on it once in a while. A succesful flow should output something like ``` @@ -131,69 +192,6 @@ L1 state: block verified: 1, block committed: 1 which means the proof for the block was verified on L1. -## Proving a block using GPU prover locally - -Below steps can be used to prove a block on local machine using GPU prover. -Running a GPU prover requires a Cuda 12.0 installation as a pre-requisite, alongside these machine specs: - -- CPU: At least 8 physical cores -- RAM: 16GB of RAM(if you have lower RAM machine enable swap) -- Disk: 30GB of free disk -- GPU: 1x Nvidia L4/T4 with 16GB of GPU RAM - - - -1. Install the correct nightly version using command: `rustup install nightly-2023-07-21` -2. Generate the gpu setup data (no need to regenerate if it's already there). This will consume around 300Gb of disk. - Use these commands: - - ```markdown - for i in {1..13}; do zk f cargo run --features "gpu" --release --bin zksync_setup_data_generator_fri -- - --numeric-circuit $i --is_base_layer done - - for i in {1..15}; do zk f cargo run --features "gpu" --release --bin zksync_setup_data_generator_fri -- - --numeric-circuit $i done - ``` - -3. Initialize DB and run migrations: `zk init` - -4. Override the following configuration in your `dev.env`: - - ``` - ETH_SENDER_SENDER_PROOF_SENDING_MODE=OnlyRealProofs - ETH_SENDER_SENDER_PROOF_LOADING_MODE=FriProofFromGcs - OBJECT_STORE_FILE_BACKED_BASE_PATH=/path/to/server/artifacts - PROVER_OBJECT_STORE_FILE_BACKED_BASE_PATH=/path/to/prover/artifacts - FRI_PROVER_SETUP_DATA_PATH=/path/to/above-generated/gpu-setup-data - ``` - -5. Run server `zk server --components=api,eth,tree,state_keeper,housekeeper,proof_data_handler` to produce blocks to be - proven -6. Run prover gateway to fetch blocks to be proven from server : - `zk f cargo run --release --bin zksync_prover_fri_gateway` -7. Run 4 witness generators to generate witness for each round: - - ``` - API_PROMETHEUS_LISTENER_PORT=3116 zk f cargo run --release --bin zksync_witness_generator -- --round=basic_circuits - API_PROMETHEUS_LISTENER_PORT=3117 zk f cargo run --release --bin zksync_witness_generator -- --round=leaf_aggregation - API_PROMETHEUS_LISTENER_PORT=3118 zk f cargo run --release --bin zksync_witness_generator -- --round=node_aggregation - API_PROMETHEUS_LISTENER_PORT=3119 zk f cargo run --release --bin zksync_witness_generator -- --round=scheduler - ``` - -8. Run prover to perform actual proving: `zk f cargo run --features "gpu" --release --bin zksync_prover_fri` -9. Run 5 witness vector generators to feed jobs to GPU prover: - - ``` - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3416 zk f cargo run --release --bin zksync_witness_vector_generator - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3417 zk f cargo run --release --bin zksync_witness_vector_generator - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3418 zk f cargo run --release --bin zksync_witness_vector_generator - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3419 zk f cargo run --release --bin zksync_witness_vector_generator - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3420 zk f cargo run --release --bin zksync_witness_vector_generator - ``` - -10. Finally, run proof compressor to compress the proof to be sent on L1: - `zk f cargo run --release --bin zksync_proof_fri_compressor` - ## Performing circuit upgrade Performing circuit upgrade requires crypto library to be updated and generating new setup data, verification key, diff --git a/prover/setup_gpu.sh b/prover/setup_gpu.sh new file mode 100755 index 000000000000..e1f31158b27a --- /dev/null +++ b/prover/setup_gpu.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# This script sets up the necessary data needed by the GPU FRI prover to be used locally. + +if [[ -z "${ZKSYNC_HOME}" ]]; then + echo "Environment variable ZKSYNC_HOME is not set. Make sure it's set and pointing to the root of this repository" + exit 1 +fi + +sed -i.backup 's/^proof_sending_mode=.*$/proof_sending_mode="OnlyRealProofs"/' ../etc/env/base/eth_sender.toml +sed -i.backup 's/^proof_loading_mode=.*$/proof_loading_mode="FriProofFromGcs"/' ../etc/env/base/eth_sender.toml +rm ../etc/env/base/eth_sender.toml.backup +sed -i.backup 's/^setup_data_path=.*$/setup_data_path="vk_setup_data_generator_server_fri\/data\/"/' ../etc/env/base/fri_prover.toml +rm ../etc/env/base/fri_prover.toml +sed -i.backup 's/^universal_setup_path=.*$/universal_setup_path="..\/keys\/setup\/setup_2^26.key"/' ../etc/env/base/fri_proof_compressor.toml +rm ../etc/env/base/fri_proof_compressor.toml + +zk config compile dev + +for i in {1..13} +do + zk f cargo run --features "gpu" --release --bin zksync_setup_data_generator_fri -- --numeric-circuit $i --is_base_layer +done + +for i in {1..15} +do + zk f cargo run --features "gpu" --release --bin zksync_setup_data_generator_fri -- --numeric-circuit $i +done