From 49eb312744cf238e7b082dabc12b5213343bf448 Mon Sep 17 00:00:00 2001 From: Vasiliy Olekhov Date: Fri, 20 Sep 2024 15:28:04 +0300 Subject: [PATCH] Added save option for updated commitment state file updated naming, added test scripts rm grinding --- proof-producer/README.md | 3 +- .../nil/proof-generator/arg_parser.hpp | 1 + .../include/nil/proof-generator/prover.hpp | 36 +++++++++--------- .../bin/proof-producer/src/arg_parser.cpp | 1 + .../bin/proof-producer/src/main.cpp | 5 ++- proof-producer/tests/00-preprocessor.sh | 22 +++++++++++ proof-producer/tests/01-partial-proof.sh | 26 +++++++++++++ proof-producer/tests/02-gen-challenges.sh | 11 ++++++ proof-producer/tests/03-compute-combined-Q.sh | 25 +++++++++++++ proof-producer/tests/04-aggregated-FRI.sh | 18 +++++++++ proof-producer/tests/05-consistency-checks.sh | 17 +++++++++ proof-producer/tests/06-merge-proofs.sh | 19 ++++++++++ proof-producer/tests/99-run-all.sh | 37 +++++++++++++++++++ 13 files changed, 200 insertions(+), 21 deletions(-) create mode 100755 proof-producer/tests/00-preprocessor.sh create mode 100755 proof-producer/tests/01-partial-proof.sh create mode 100755 proof-producer/tests/02-gen-challenges.sh create mode 100755 proof-producer/tests/03-compute-combined-Q.sh create mode 100755 proof-producer/tests/04-aggregated-FRI.sh create mode 100755 proof-producer/tests/05-consistency-checks.sh create mode 100755 proof-producer/tests/06-merge-proofs.sh create mode 100755 proof-producer/tests/99-run-all.sh diff --git a/proof-producer/README.md b/proof-producer/README.md index b6c4228a61..c21b8b52d5 100644 --- a/proof-producer/README.md +++ b/proof-producer/README.md @@ -118,7 +118,7 @@ Verify generated proof: Partial proof, ran on each prover. ```bash ./build/bin/proof-producer/proof-producer-single-threaded \ - --stage partial-prove \ + --stage generate-partial-proof \ --grind-param 16 \ --max-quotient-chunks 10 \ --circuit circuits-and-assignments/$CIRCUIT/circuit.crct \ @@ -126,6 +126,7 @@ Partial proof, ran on each prover. --common-data $CIRCUIT-common_data.dat \ --preprocessed-data $CIRCUIT-preprocessed.dat \ --commitment-state-file $CIRCUIT-commitment_state.dat \ + --updated-commitment-state-file $CIRCUIT-updated_commitment_state.dat \ --assignment-description-file $CIRCUIT-assignment-description.dat \ --challenge-file $CIRCUIT-challenge.dat \ --theta-power-file $CIRCUIT-theta-power.txt \ diff --git a/proof-producer/bin/proof-producer/include/nil/proof-generator/arg_parser.hpp b/proof-producer/bin/proof-producer/include/nil/proof-generator/arg_parser.hpp index 6d5733d2f0..e36dca7862 100644 --- a/proof-producer/bin/proof-producer/include/nil/proof-generator/arg_parser.hpp +++ b/proof-producer/bin/proof-producer/include/nil/proof-generator/arg_parser.hpp @@ -41,6 +41,7 @@ namespace nil { boost::filesystem::path preprocessed_common_data_path = "preprocessed_common_data.dat"; boost::filesystem::path preprocessed_public_data_path = "preprocessed_data.dat"; boost::filesystem::path commitment_scheme_state_path = "commitment_scheme_state.dat"; + boost::filesystem::path updated_commitment_scheme_state_path = "updated_commitment_scheme_state.dat"; boost::filesystem::path circuit_file_path; boost::filesystem::path assignment_table_file_path; boost::filesystem::path assignment_description_file_path; diff --git a/proof-producer/bin/proof-producer/include/nil/proof-generator/prover.hpp b/proof-producer/bin/proof-producer/include/nil/proof-generator/prover.hpp index b30b8cc07e..dec4e7ec0d 100644 --- a/proof-producer/bin/proof-producer/include/nil/proof-generator/prover.hpp +++ b/proof-producer/bin/proof-producer/include/nil/proof-generator/prover.hpp @@ -107,7 +107,7 @@ namespace nil { PROVE = 2, VERIFY = 3, GENERATE_AGGREGATED_CHALLENGE = 4, - PARTIAL_PROVE = 5, + GENERATE_PARTIAL_PROOF = 5, COMPUTE_COMBINED_Q = 6, GENERATE_AGGREGATED_FRI_PROOF = 7, GENERATE_CONSISTENCY_CHECKS_PROOF = 8, @@ -121,7 +121,7 @@ namespace nil { {"prove", ProverStage::PROVE}, {"verify", ProverStage::VERIFY}, {"generate-aggregated-challenge", ProverStage::GENERATE_AGGREGATED_CHALLENGE}, - {"partial-prove", ProverStage::PARTIAL_PROVE}, + {"generate-partial-proof", ProverStage::GENERATE_PARTIAL_PROOF}, {"compute-combined-Q", ProverStage::COMPUTE_COMBINED_Q}, {"merge-proofs", ProverStage::MERGE_PROOFS}, {"aggregated-FRI", ProverStage::GENERATE_AGGREGATED_FRI_PROOF}, @@ -345,7 +345,7 @@ namespace nil { } bool save_preprocessed_common_data_to_file(boost::filesystem::path preprocessed_common_data_file) { - BOOST_LOG_TRIVIAL(info) << "Writing preprocessed common data to " << preprocessed_common_data_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Writing preprocessed common data to " << preprocessed_common_data_file; auto marshalled_common_data = nil::crypto3::marshalling::types::fill_placeholder_common_data( public_preprocessed_data_->common_data @@ -361,7 +361,7 @@ namespace nil { } bool read_preprocessed_common_data_from_file(boost::filesystem::path preprocessed_common_data_file) { - BOOST_LOG_TRIVIAL(info) << "Read preprocessed common data from " << preprocessed_common_data_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Read preprocessed common data from " << preprocessed_common_data_file; using CommonDataMarshalling = nil::crypto3::marshalling::types::placeholder_common_data; @@ -383,7 +383,7 @@ namespace nil { using namespace nil::crypto3::marshalling::types; BOOST_LOG_TRIVIAL(info) << "Writing all preprocessed public data to " << - preprocessed_data_file << std::endl; + preprocessed_data_file; using PreprocessedPublicDataType = typename PublicPreprocessedData::preprocessed_data_type; auto marshalled_preprocessed_public_data = @@ -401,7 +401,7 @@ namespace nil { } bool read_public_preprocessed_data_from_file(boost::filesystem::path preprocessed_data_file) { - BOOST_LOG_TRIVIAL(info) << "Read preprocessed data from " << preprocessed_data_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Read preprocessed data from " << preprocessed_data_file; using namespace nil::crypto3::marshalling::types; @@ -424,7 +424,7 @@ namespace nil { using namespace nil::crypto3::marshalling::types; BOOST_LOG_TRIVIAL(info) << "Writing commitment_state to " << - commitment_scheme_state_file << std::endl; + commitment_scheme_state_file; auto marshalled_lpc_state = fill_commitment_scheme( *lpc_scheme_); @@ -439,7 +439,7 @@ namespace nil { } bool read_commitment_scheme_from_file(boost::filesystem::path commitment_scheme_state_file) { - BOOST_LOG_TRIVIAL(info) << "Read commitment scheme from " << commitment_scheme_state_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Read commitment scheme from " << commitment_scheme_state_file; using namespace nil::crypto3::marshalling::types; @@ -475,7 +475,7 @@ namespace nil { } bool read_circuit(const boost::filesystem::path& circuit_file_) { - BOOST_LOG_TRIVIAL(info) << "Read circuit from " << circuit_file_ << std::endl; + BOOST_LOG_TRIVIAL(info) << "Read circuit from " << circuit_file_; using ConstraintMarshalling = nil::crypto3::marshalling::types::plonk_constraint_system; @@ -493,7 +493,7 @@ namespace nil { } bool read_assignment_table(const boost::filesystem::path& assignment_table_file_) { - BOOST_LOG_TRIVIAL(info) << "Read assignment table from " << assignment_table_file_ << std::endl; + BOOST_LOG_TRIVIAL(info) << "Read assignment table from " << assignment_table_file_; using TableValueMarshalling = nil::crypto3::marshalling::types::plonk_assignment_table; @@ -513,7 +513,7 @@ namespace nil { } bool save_assignment_description(const boost::filesystem::path& assignment_description_file) { - BOOST_LOG_TRIVIAL(info) << "Writing assignment description to " << assignment_description_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Writing assignment description to " << assignment_description_file; auto marshalled_assignment_description = nil::crypto3::marshalling::types::fill_assignment_table_description( @@ -530,7 +530,7 @@ namespace nil { } bool read_assignment_description(const boost::filesystem::path& assignment_description_file_) { - BOOST_LOG_TRIVIAL(info) << "Read assignment description from " << assignment_description_file_ << std::endl; + BOOST_LOG_TRIVIAL(info) << "Read assignment description from " << assignment_description_file_; using TableDescriptionMarshalling = nil::crypto3::marshalling::types::plonk_assignment_table_description; @@ -572,7 +572,7 @@ namespace nil { using challenge_marshalling_type = nil::crypto3::marshalling::types::field_element< TTypeBase, typename BlueprintField::value_type>; - BOOST_LOG_TRIVIAL(info) << "Writing challenge to " << challenge_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Writing challenge to " << challenge_file; // marshall the challenge challenge_marshalling_type marshalled_challenge(challenge); @@ -658,7 +658,7 @@ namespace nil { using polynomial_marshalling_type = typename nil::crypto3::marshalling::types::polynomial< TTypeBase, PolynomialType>::type; - BOOST_LOG_TRIVIAL(info) << "Writing polynomial to " << output_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Writing polynomial to " << output_file; polynomial_marshalling_type marshalled_poly = nil::crypto3::marshalling::types::fill_polynomial(poly); @@ -791,7 +791,7 @@ namespace nil { using fri_proof_marshalling_type = nil::crypto3::marshalling::types::initial_fri_proof_type< TTypeBase, LpcScheme>; - BOOST_LOG_TRIVIAL(info) << "Writing aggregated FRI proof to " << output_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Writing aggregated FRI proof to " << output_file; fri_proof_marshalling_type marshalled_proof = nil::crypto3::marshalling::types::fill_initial_fri_proof(fri_proof); @@ -803,7 +803,7 @@ namespace nil { const typename FriType::grinding_type::output_type &proof_of_work, const boost::filesystem::path &output_file) { using POW_marshalling_type = nil::marshalling::types::integral; - BOOST_LOG_TRIVIAL(info) << "Writing proof of work to " << output_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Writing proof of work to " << output_file; POW_marshalling_type marshalled_pow(proof_of_work); @@ -818,7 +818,7 @@ namespace nil { using challenge_vector_marshalling_type = nil::crypto3::marshalling::types::field_element_vector< typename BlueprintField::value_type, TTypeBase>; - BOOST_LOG_TRIVIAL(info) << "Writing challenges to " << consistency_checks_challenges_output_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Writing challenges to " << consistency_checks_challenges_output_file; challenge_vector_marshalling_type marshalled_challenges = nil::crypto3::marshalling::types::fill_field_element_vector( @@ -898,7 +898,7 @@ namespace nil { using lpc_consistency_proof_marshalling_type = nil::crypto3::marshalling::types::inital_eval_proof< TTypeBase, LpcScheme>; - BOOST_LOG_TRIVIAL(info) << "Writing LPC consistency proof to " << output_file << std::endl; + BOOST_LOG_TRIVIAL(info) << "Writing LPC consistency proof to " << output_file; lpc_consistency_proof_marshalling_type marshalled_proof = nil::crypto3::marshalling::types::fill_initial_eval_proof(lpc_consistency_proof); diff --git a/proof-producer/bin/proof-producer/src/arg_parser.cpp b/proof-producer/bin/proof-producer/src/arg_parser.cpp index 82eaae0ad9..1e74678edc 100644 --- a/proof-producer/bin/proof-producer/src/arg_parser.cpp +++ b/proof-producer/bin/proof-producer/src/arg_parser.cpp @@ -79,6 +79,7 @@ namespace nil { ("common-data", make_defaulted_option(prover_options.preprocessed_common_data_path), "Preprocessed common data file") ("preprocessed-data", make_defaulted_option(prover_options.preprocessed_public_data_path), "Preprocessed public data file") ("commitment-state-file", make_defaulted_option(prover_options.commitment_scheme_state_path), "Commitment state data file") + ("updated-commitment-state-file", make_defaulted_option(prover_options.updated_commitment_scheme_state_path), "Updated commitment state data file") ("circuit", po::value(&prover_options.circuit_file_path), "Circuit input file") ("assignment-table,t", po::value(&prover_options.assignment_table_file_path), "Assignment table input file") ("assignment-description-file", po::value(&prover_options.assignment_description_file_path), "Assignment description file") diff --git a/proof-producer/bin/proof-producer/src/main.cpp b/proof-producer/bin/proof-producer/src/main.cpp index 7fa6d524de..ab22d77ea2 100644 --- a/proof-producer/bin/proof-producer/src/main.cpp +++ b/proof-producer/bin/proof-producer/src/main.cpp @@ -78,7 +78,7 @@ int run_prover(const nil::proof_generator::ProverOptions& prover_options) { prover_options.json_file_path, true/*skip verification*/); break; - case nil::proof_generator::detail::ProverStage::PARTIAL_PROVE: + case nil::proof_generator::detail::ProverStage::GENERATE_PARTIAL_PROOF: // Load preprocessed data from file and generate the proof. prover_result = prover.read_circuit(prover_options.circuit_file_path) && @@ -90,7 +90,8 @@ int run_prover(const nil::proof_generator::ProverOptions& prover_options) { prover.generate_partial_proof_to_file( prover_options.proof_file_path, prover_options.challenge_file_path, - prover_options.theta_power_file_path); + prover_options.theta_power_file_path) && + prover.save_commitment_state_to_file(prover_options.updated_commitment_scheme_state_path); break; case nil::proof_generator::detail::ProverStage::VERIFY: prover_result = diff --git a/proof-producer/tests/00-preprocessor.sh b/proof-producer/tests/00-preprocessor.sh new file mode 100755 index 0000000000..77724efefe --- /dev/null +++ b/proof-producer/tests/00-preprocessor.sh @@ -0,0 +1,22 @@ +#!/bin/sh + + +if [ "x$1" = "x" ] ; then + echo "Circuit not defined" + exit 1 +fi + +CIRCUIT=$1 + +echo "Preprocessing circuit: $CIRCUIT" + +bin/proof-producer/proof-producer-single-threaded \ + --circuit circuits-and-assignments/$CIRCUIT/circuit.crct \ + --assignment-table circuits-and-assignments/$CIRCUIT/assignment.tbl \ + --common-data $CIRCUIT-common_data.dat \ + --preprocessed-data $CIRCUIT-preprocessed.dat \ + --commitment-state-file $CIRCUIT-commitment_state.dat \ + --assignment-description-file $CIRCUIT-assignment-description.dat \ + --stage preprocess \ + --max-quotient-chunks 10 + diff --git a/proof-producer/tests/01-partial-proof.sh b/proof-producer/tests/01-partial-proof.sh new file mode 100755 index 0000000000..d158dcc1da --- /dev/null +++ b/proof-producer/tests/01-partial-proof.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +if [ "x$1" = "x" ] ; then + echo "Circuit not defined" + exit 1 +fi + +CIRCUIT=$1 + +echo "Partial proof for circuit: $CIRCUIT" + +bin/proof-producer/proof-producer-single-threaded \ + --stage generate-partial-proof \ + --max-quotient-chunks 10 \ + --circuit circuits-and-assignments/$CIRCUIT/circuit.crct \ + --assignment-table circuits-and-assignments/$CIRCUIT/assignment.tbl \ + --common-data $CIRCUIT-common_data.dat \ + --preprocessed-data $CIRCUIT-preprocessed.dat \ + --commitment-state-file $CIRCUIT-commitment_state.dat \ + --updated-commitment-state-file $CIRCUIT-updated_commitment_state.dat \ + --assignment-description-file $CIRCUIT-assignment-description.dat \ + --challenge-file $CIRCUIT-challenge.dat \ + --theta-power-file $CIRCUIT-theta-power.txt \ + --proof $CIRCUIT-proof.dat \ + --json $CIRCUIT-proof.json + diff --git a/proof-producer/tests/02-gen-challenges.sh b/proof-producer/tests/02-gen-challenges.sh new file mode 100755 index 0000000000..bfa50908bf --- /dev/null +++ b/proof-producer/tests/02-gen-challenges.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +echo "Generating challenges" + +bin/proof-producer/proof-producer-single-threaded \ + --stage generate-aggregated-challenge \ + -u "fri_array_swap-challenge.dat" \ + -u "merkle_tree_poseidon_cpp_example-challenge.dat" \ + --aggregated-challenge-file "challenge-aggregated.dat" + + diff --git a/proof-producer/tests/03-compute-combined-Q.sh b/proof-producer/tests/03-compute-combined-Q.sh new file mode 100755 index 0000000000..29b340fd72 --- /dev/null +++ b/proof-producer/tests/03-compute-combined-Q.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +if [ "x$1" = "x" ] ; then + echo "Circuit not defined" + exit 1 +fi + +if [ "x$2" = "x" ] ; then + echo "Starting power not defined" + exit 1 +fi + + +CIRCUIT=$1 + +echo "Computing combined Q for circuit: $CIRCUIT, starting power: $2" + +bin/proof-producer/proof-producer-single-threaded \ + --stage compute-combined-Q \ + --aggregated-challenge-file "challenge-aggregated.dat" \ + --combined-Q-starting-power=$2 \ + --commitment-state-file $CIRCUIT-commitment_state.dat \ + --combined-Q-polynomial-file $CIRCUIT-combined-Q.dat + + diff --git a/proof-producer/tests/04-aggregated-FRI.sh b/proof-producer/tests/04-aggregated-FRI.sh new file mode 100755 index 0000000000..6c3aa4a078 --- /dev/null +++ b/proof-producer/tests/04-aggregated-FRI.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +CIRCUIT1=fri_array_swap +CIRCUIT2=merkle_tree_poseidon_cpp_example + +echo "Computing aggregated FRI" + +bin/proof-producer/proof-producer-single-threaded \ + --stage aggregated-FRI \ + --assignment-description-file $CIRCUIT1-assignment-description.dat \ + --aggregated-challenge-file "challenge-aggregated.dat" \ + --input-combined-Q-polynomial-files "$CIRCUIT1-combined-Q.dat" \ + --input-combined-Q-polynomial-files "$CIRCUIT2-combined-Q.dat" \ + --proof="aggregated_FRI_proof.bin" \ + --proof-of-work-file="POW.dat" \ + --consistency-checks-challenges-file="challenges.dat" + + diff --git a/proof-producer/tests/05-consistency-checks.sh b/proof-producer/tests/05-consistency-checks.sh new file mode 100755 index 0000000000..40de7538ee --- /dev/null +++ b/proof-producer/tests/05-consistency-checks.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +if [ "x$1" = "x" ] ; then + echo "Circuit not defined" + exit 1 +fi + +CIRCUIT=$1 + +echo "Consistency chacks for circuit: $CIRCUIT" + +bin/proof-producer/proof-producer-single-threaded \ + --stage consistency-checks \ + --commitment-state-file $CIRCUIT-updated_commitment_state.dat \ + --combined-Q-polynomial-file $CIRCUIT-combined-Q.dat \ + --consistency-checks-challenges-file "challenges.dat" \ + --proof $CIRCUIT-LPC_consistency_check_proof.bin diff --git a/proof-producer/tests/06-merge-proofs.sh b/proof-producer/tests/06-merge-proofs.sh new file mode 100755 index 0000000000..b3ccf6585a --- /dev/null +++ b/proof-producer/tests/06-merge-proofs.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +CIRCUIT1=fri_array_swap +CIRCUIT2=merkle_tree_poseidon_cpp_example + +echo "Merging proofs" + +bin/proof-producer/proof-producer-single-threaded \ + --stage merge-proofs \ + --partial-proof $CIRCUIT1-proof.dat \ + --partial-proof $CIRCUIT2-proof.dat \ + --initial-proof $CIRCUIT1-LPC_consistency_check_proof.bin \ + --initial-proof $CIRCUIT2-LPC_consistency_check_proof.bin \ + --aggregated-FRI-proof aggregated_FRI_proof.bin \ + --proof final-proof.dat + + + + diff --git a/proof-producer/tests/99-run-all.sh b/proof-producer/tests/99-run-all.sh new file mode 100755 index 0000000000..adcaca3956 --- /dev/null +++ b/proof-producer/tests/99-run-all.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +# This suite expects that circuits are extracted into "circuits-and-assignments" directory +# You can get these from https://github.com/NilFoundation/zkLLVM/actions +# Pick latest pipeline run from master, scroll down to "Artifacts" section and +# download zip. The size (10Gb) is of unpacked data, downloaded archive is 100Mb or so. + +CIRCUIT1=fri_array_swap +CIRCUIT2=merkle_tree_poseidon_cpp_example + +echo " === STAGE 00 === " +./00-preprocessor.sh $CIRCUIT1 +./00-preprocessor.sh $CIRCUIT2 + +echo " === STAGE 01 === " +./01-partial-proof.sh $CIRCUIT1 +./01-partial-proof.sh $CIRCUIT2 + +echo " === STAGE 02 === " +./02-gen-challenges.sh + +echo " === STAGE 03 === " +./03-compute-combined-Q.sh $CIRCUIT1 0 +./03-compute-combined-Q.sh $CIRCUIT2 `cat $CIRCUIT1-theta-power.txt` + +echo " === STAGE 04 === " +./04-aggregated-FRI.sh + +echo " === STAGE 05 === " +./05-consistency-checks.sh $CIRCUIT1 +./05-consistency-checks.sh $CIRCUIT2 + +echo " === STAGE 06 === " +./06-merge-proofs.sh + + +