Added save option for updated commitment state file

updated naming, added test scripts

rm grinding

proof-producer/README.md

@@ -118,14 +118,15 @@ Verify generated proof:
 Partial proof, ran on each prover.
 ```bash
 ./build/bin/proof-producer/proof-producer-single-threaded \
-    --stage partial-prove \
+    --stage generate-partial-proof \
     --grind-param 16 \
     --max-quotient-chunks 10 \
     --circuit           circuits-and-assignments/$CIRCUIT/circuit.crct \
     --assignment-table  circuits-and-assignments/$CIRCUIT/assignment.tbl \
     --common-data                  $CIRCUIT-common_data.dat \
     --preprocessed-data            $CIRCUIT-preprocessed.dat \
     --commitment-state-file        $CIRCUIT-commitment_state.dat \
+    --updated-commitment-state-file $CIRCUIT-updated_commitment_state.dat \
     --assignment-description-file  $CIRCUIT-assignment-description.dat \
     --challenge-file               $CIRCUIT-challenge.dat \
     --theta-power-file             $CIRCUIT-theta-power.txt \

proof-producer/bin/proof-producer/include/nil/proof-generator/arg_parser.hpp

@@ -41,6 +41,7 @@ namespace nil {
             boost::filesystem::path preprocessed_common_data_path = "preprocessed_common_data.dat";
             boost::filesystem::path preprocessed_public_data_path = "preprocessed_data.dat";
             boost::filesystem::path commitment_scheme_state_path = "commitment_scheme_state.dat";
+            boost::filesystem::path updated_commitment_scheme_state_path = "updated_commitment_scheme_state.dat";
             boost::filesystem::path circuit_file_path;
             boost::filesystem::path assignment_table_file_path;
             boost::filesystem::path assignment_description_file_path;

proof-producer/bin/proof-producer/include/nil/proof-generator/prover.hpp

@@ -107,7 +107,7 @@ namespace nil {
                 PROVE = 2,
                 VERIFY = 3,
                 GENERATE_AGGREGATED_CHALLENGE = 4,
-                PARTIAL_PROVE = 5,
+                GENERATE_PARTIAL_PROOF = 5,
                 COMPUTE_COMBINED_Q = 6,
                 GENERATE_AGGREGATED_FRI_PROOF = 7,
                 GENERATE_CONSISTENCY_CHECKS_PROOF = 8,
@@ -121,7 +121,7 @@ namespace nil {
                     {"prove", ProverStage::PROVE},
                     {"verify", ProverStage::VERIFY},
                     {"generate-aggregated-challenge", ProverStage::GENERATE_AGGREGATED_CHALLENGE},
-                    {"partial-prove", ProverStage::PARTIAL_PROVE},
+                    {"generate-partial-proof", ProverStage::GENERATE_PARTIAL_PROOF},
                     {"compute-combined-Q", ProverStage::COMPUTE_COMBINED_Q},
                     {"merge-proofs", ProverStage::MERGE_PROOFS},
                     {"aggregated-FRI", ProverStage::GENERATE_AGGREGATED_FRI_PROOF},
@@ -345,7 +345,7 @@ namespace nil {
             }
 
             bool save_preprocessed_common_data_to_file(boost::filesystem::path preprocessed_common_data_file) {
-                BOOST_LOG_TRIVIAL(info) << "Writing preprocessed common data to " << preprocessed_common_data_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Writing preprocessed common data to " << preprocessed_common_data_file;
                 auto marshalled_common_data =
                     nil::crypto3::marshalling::types::fill_placeholder_common_data<Endianness, CommonData>(
                         public_preprocessed_data_->common_data
@@ -361,7 +361,7 @@ namespace nil {
             }
 
             bool read_preprocessed_common_data_from_file(boost::filesystem::path preprocessed_common_data_file) {
-                BOOST_LOG_TRIVIAL(info) << "Read preprocessed common data from " << preprocessed_common_data_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Read preprocessed common data from " << preprocessed_common_data_file;
 
                 using CommonDataMarshalling = nil::crypto3::marshalling::types::placeholder_common_data<TTypeBase, CommonData>;
 
@@ -383,7 +383,7 @@ namespace nil {
                 using namespace nil::crypto3::marshalling::types;
 
                 BOOST_LOG_TRIVIAL(info) << "Writing all preprocessed public data to " <<
-                    preprocessed_data_file << std::endl;
+                    preprocessed_data_file;
                 using PreprocessedPublicDataType = typename PublicPreprocessedData::preprocessed_data_type;
 
                 auto marshalled_preprocessed_public_data =
@@ -401,7 +401,7 @@ namespace nil {
             }
 
             bool read_public_preprocessed_data_from_file(boost::filesystem::path preprocessed_data_file) {
-                BOOST_LOG_TRIVIAL(info) << "Read preprocessed data from " << preprocessed_data_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Read preprocessed data from " << preprocessed_data_file;
 
                 using namespace nil::crypto3::marshalling::types;
 
@@ -424,7 +424,7 @@ namespace nil {
                 using namespace nil::crypto3::marshalling::types;
 
                 BOOST_LOG_TRIVIAL(info) << "Writing commitment_state to " <<
-                    commitment_scheme_state_file << std::endl;
+                    commitment_scheme_state_file;
 
                 auto marshalled_lpc_state = fill_commitment_scheme<Endianness, LpcScheme>(
                     *lpc_scheme_);
@@ -439,7 +439,7 @@ namespace nil {
             }
 
             bool read_commitment_scheme_from_file(boost::filesystem::path commitment_scheme_state_file) {
-                BOOST_LOG_TRIVIAL(info) << "Read commitment scheme from " << commitment_scheme_state_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Read commitment scheme from " << commitment_scheme_state_file;
 
                 using namespace nil::crypto3::marshalling::types;
 
@@ -475,7 +475,7 @@ namespace nil {
             }
 
             bool read_circuit(const boost::filesystem::path& circuit_file_) {
-                BOOST_LOG_TRIVIAL(info) << "Read circuit from " << circuit_file_ << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Read circuit from " << circuit_file_;
 
                 using ConstraintMarshalling =
                     nil::crypto3::marshalling::types::plonk_constraint_system<TTypeBase, ConstraintSystem>;
@@ -493,7 +493,7 @@ namespace nil {
             }
 
             bool read_assignment_table(const boost::filesystem::path& assignment_table_file_) {
-                BOOST_LOG_TRIVIAL(info) << "Read assignment table from " << assignment_table_file_ << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Read assignment table from " << assignment_table_file_;
 
                 using TableValueMarshalling =
                     nil::crypto3::marshalling::types::plonk_assignment_table<TTypeBase, AssignmentTable>;
@@ -513,7 +513,7 @@ namespace nil {
             }
 
             bool save_assignment_description(const boost::filesystem::path& assignment_description_file) {
-                BOOST_LOG_TRIVIAL(info) << "Writing assignment description to " << assignment_description_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Writing assignment description to " << assignment_description_file;
 
                 auto marshalled_assignment_description =
                     nil::crypto3::marshalling::types::fill_assignment_table_description<Endianness, BlueprintField>(
@@ -530,7 +530,7 @@ namespace nil {
             }
 
             bool read_assignment_description(const boost::filesystem::path& assignment_description_file_) {
-                BOOST_LOG_TRIVIAL(info) << "Read assignment description from " << assignment_description_file_ << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Read assignment description from " << assignment_description_file_;
 
                 using TableDescriptionMarshalling =
                     nil::crypto3::marshalling::types::plonk_assignment_table_description<TTypeBase>;
@@ -572,7 +572,7 @@ namespace nil {
                 using challenge_marshalling_type = nil::crypto3::marshalling::types::field_element<
                     TTypeBase, typename BlueprintField::value_type>;
 
-                BOOST_LOG_TRIVIAL(info) << "Writing challenge to " << challenge_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Writing challenge to " << challenge_file;
 
                 // marshall the challenge
                 challenge_marshalling_type marshalled_challenge(challenge);
@@ -658,7 +658,7 @@ namespace nil {
                 using polynomial_marshalling_type = typename nil::crypto3::marshalling::types::polynomial<
                     TTypeBase, PolynomialType>::type;
 
-                BOOST_LOG_TRIVIAL(info) << "Writing polynomial to " << output_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Writing polynomial to " << output_file;
 
                 polynomial_marshalling_type marshalled_poly = nil::crypto3::marshalling::types::fill_polynomial<Endianness, PolynomialType>(poly);
 
@@ -791,7 +791,7 @@ namespace nil {
                 using fri_proof_marshalling_type = nil::crypto3::marshalling::types::initial_fri_proof_type<
                     TTypeBase, LpcScheme>;
 
-                BOOST_LOG_TRIVIAL(info) << "Writing aggregated FRI proof to " << output_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Writing aggregated FRI proof to " << output_file;
 
                 fri_proof_marshalling_type marshalled_proof = nil::crypto3::marshalling::types::fill_initial_fri_proof<Endianness, LpcScheme>(fri_proof);
 
@@ -803,7 +803,7 @@ namespace nil {
                 const typename FriType::grinding_type::output_type &proof_of_work, 
                 const boost::filesystem::path &output_file) {
                 using POW_marshalling_type = nil::marshalling::types::integral<TTypeBase, typename FriType::grinding_type::output_type>;
-                BOOST_LOG_TRIVIAL(info) << "Writing proof of work to " << output_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Writing proof of work to " << output_file;
 
                 POW_marshalling_type marshalled_pow(proof_of_work);
 
@@ -818,7 +818,7 @@ namespace nil {
                 using challenge_vector_marshalling_type = nil::crypto3::marshalling::types::field_element_vector<
                     typename BlueprintField::value_type, TTypeBase>;
 
-                BOOST_LOG_TRIVIAL(info) << "Writing challenges to " << consistency_checks_challenges_output_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Writing challenges to " << consistency_checks_challenges_output_file;
 
                 challenge_vector_marshalling_type marshalled_challenges = 
                     nil::crypto3::marshalling::types::fill_field_element_vector<typename BlueprintField::value_type, Endianness>(
@@ -898,7 +898,7 @@ namespace nil {
                 using lpc_consistency_proof_marshalling_type = nil::crypto3::marshalling::types::inital_eval_proof<
                     TTypeBase, LpcScheme>;
 
-                BOOST_LOG_TRIVIAL(info) << "Writing LPC consistency proof to " << output_file << std::endl;
+                BOOST_LOG_TRIVIAL(info) << "Writing LPC consistency proof to " << output_file;
 
                 lpc_consistency_proof_marshalling_type marshalled_proof = nil::crypto3::marshalling::types::fill_initial_eval_proof<Endianness, LpcScheme>(lpc_consistency_proof);
 

proof-producer/bin/proof-producer/src/arg_parser.cpp

@@ -79,6 +79,7 @@ namespace nil {
                 ("common-data", make_defaulted_option(prover_options.preprocessed_common_data_path), "Preprocessed common data file")
                 ("preprocessed-data", make_defaulted_option(prover_options.preprocessed_public_data_path), "Preprocessed public data file")
                 ("commitment-state-file", make_defaulted_option(prover_options.commitment_scheme_state_path), "Commitment state data file")
+                ("updated-commitment-state-file", make_defaulted_option(prover_options.updated_commitment_scheme_state_path), "Updated commitment state data file")
                 ("circuit", po::value(&prover_options.circuit_file_path), "Circuit input file")
                 ("assignment-table,t", po::value(&prover_options.assignment_table_file_path), "Assignment table input file")
                 ("assignment-description-file", po::value(&prover_options.assignment_description_file_path), "Assignment description file")

proof-producer/bin/proof-producer/src/main.cpp

@@ -78,7 +78,7 @@ int run_prover(const nil::proof_generator::ProverOptions& prover_options) {
                             prover_options.json_file_path,
                             true/*skip verification*/);
                     break;
-                case nil::proof_generator::detail::ProverStage::PARTIAL_PROVE:
+                case nil::proof_generator::detail::ProverStage::GENERATE_PARTIAL_PROOF:
                     // Load preprocessed data from file and generate the proof.
                     prover_result =
                         prover.read_circuit(prover_options.circuit_file_path) &&
@@ -90,7 +90,8 @@ int run_prover(const nil::proof_generator::ProverOptions& prover_options) {
                         prover.generate_partial_proof_to_file(
                             prover_options.proof_file_path,
                             prover_options.challenge_file_path,
-                            prover_options.theta_power_file_path);
+                            prover_options.theta_power_file_path) &&
+                        prover.save_commitment_state_to_file(prover_options.updated_commitment_scheme_state_path);
                     break;
                 case nil::proof_generator::detail::ProverStage::VERIFY:
                     prover_result =

proof-producer/tests/00-preprocessor.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+
+if [ "x$1" = "x" ] ; then
+    echo "Circuit not defined"
+    exit 1
+fi
+
+CIRCUIT=$1
+
+echo "Preprocessing circuit: $CIRCUIT"
+
+bin/proof-producer/proof-producer-single-threaded \
+    --circuit           circuits-and-assignments/$CIRCUIT/circuit.crct \
+    --assignment-table  circuits-and-assignments/$CIRCUIT/assignment.tbl \
+    --common-data $CIRCUIT-common_data.dat \
+    --preprocessed-data $CIRCUIT-preprocessed.dat \
+    --commitment-state-file $CIRCUIT-commitment_state.dat \
+    --assignment-description-file $CIRCUIT-assignment-description.dat \
+    --stage preprocess \
+    --max-quotient-chunks 10
+

proof-producer/tests/01-partial-proof.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+if [ "x$1" = "x" ] ; then
+    echo "Circuit not defined"
+    exit 1
+fi
+
+CIRCUIT=$1
+
+echo "Partial proof for circuit: $CIRCUIT"
+
+bin/proof-producer/proof-producer-single-threaded \
+    --stage generate-partial-proof \
+    --max-quotient-chunks 10 \
+    --circuit           circuits-and-assignments/$CIRCUIT/circuit.crct \
+    --assignment-table  circuits-and-assignments/$CIRCUIT/assignment.tbl \
+    --common-data                  $CIRCUIT-common_data.dat \
+    --preprocessed-data            $CIRCUIT-preprocessed.dat \
+    --commitment-state-file        $CIRCUIT-commitment_state.dat \
+    --updated-commitment-state-file $CIRCUIT-updated_commitment_state.dat \
+    --assignment-description-file  $CIRCUIT-assignment-description.dat \
+    --challenge-file               $CIRCUIT-challenge.dat \
+    --theta-power-file             $CIRCUIT-theta-power.txt \
+    --proof                        $CIRCUIT-proof.dat \
+    --json                         $CIRCUIT-proof.json
+

proof-producer/tests/02-gen-challenges.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+echo "Generating challenges"
+
+bin/proof-producer/proof-producer-single-threaded \
+    --stage generate-aggregated-challenge  \
+    -u "fri_array_swap-challenge.dat" \
+    -u "merkle_tree_poseidon_cpp_example-challenge.dat" \
+    --aggregated-challenge-file "challenge-aggregated.dat"
+
+

proof-producer/tests/03-compute-combined-Q.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+if [ "x$1" = "x" ] ; then
+    echo "Circuit not defined"
+    exit 1
+fi
+
+if [ "x$2" = "x" ] ; then
+    echo "Starting power not defined"
+    exit 1
+fi
+
+
+CIRCUIT=$1
+
+echo "Computing combined Q for circuit: $CIRCUIT, starting power: $2"
+
+bin/proof-producer/proof-producer-single-threaded \
+    --stage compute-combined-Q  \
+    --aggregated-challenge-file           "challenge-aggregated.dat" \
+    --combined-Q-starting-power=$2  \
+    --commitment-state-file               $CIRCUIT-commitment_state.dat \
+    --combined-Q-polynomial-file          $CIRCUIT-combined-Q.dat
+
+

proof-producer/tests/04-aggregated-FRI.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+CIRCUIT1=fri_array_swap
+CIRCUIT2=merkle_tree_poseidon_cpp_example
+
+echo "Computing aggregated FRI"
+
+bin/proof-producer/proof-producer-single-threaded \
+    --stage aggregated-FRI  \
+    --assignment-description-file $CIRCUIT1-assignment-description.dat \
+    --aggregated-challenge-file "challenge-aggregated.dat" \
+    --input-combined-Q-polynomial-files "$CIRCUIT1-combined-Q.dat" \
+    --input-combined-Q-polynomial-files "$CIRCUIT2-combined-Q.dat" \
+    --proof="aggregated_FRI_proof.bin" \
+    --proof-of-work-file="POW.dat" \
+    --consistency-checks-challenges-file="challenges.dat"
+
+

proof-producer/tests/05-consistency-checks.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+if [ "x$1" = "x" ] ; then
+    echo "Circuit not defined"
+    exit 1
+fi
+
+CIRCUIT=$1
+
+echo "Consistency chacks for circuit: $CIRCUIT"
+
+bin/proof-producer/proof-producer-single-threaded \
+    --stage consistency-checks  \
+    --commitment-state-file $CIRCUIT-updated_commitment_state.dat \
+    --combined-Q-polynomial-file $CIRCUIT-combined-Q.dat \
+    --consistency-checks-challenges-file "challenges.dat" \
+    --proof $CIRCUIT-LPC_consistency_check_proof.bin

proof-producer/tests/06-merge-proofs.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+CIRCUIT1=fri_array_swap
+CIRCUIT2=merkle_tree_poseidon_cpp_example
+
+echo "Merging proofs"
+
+bin/proof-producer/proof-producer-single-threaded \
+    --stage merge-proofs  \
+    --partial-proof $CIRCUIT1-proof.dat \
+    --partial-proof $CIRCUIT2-proof.dat \
+    --initial-proof $CIRCUIT1-LPC_consistency_check_proof.bin \
+    --initial-proof $CIRCUIT2-LPC_consistency_check_proof.bin \
+    --aggregated-FRI-proof aggregated_FRI_proof.bin \
+    --proof final-proof.dat
+
+
+
+

proof-producer/tests/99-run-all.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+# This suite expects that circuits are extracted into "circuits-and-assignments" directory
+# You can get these from https://github.com/NilFoundation/zkLLVM/actions
+# Pick latest pipeline run from master, scroll down to "Artifacts" section and
+# download zip. The size (10Gb) is of unpacked data, downloaded archive is 100Mb or so.
+
+CIRCUIT1=fri_array_swap
+CIRCUIT2=merkle_tree_poseidon_cpp_example
+
+echo " === STAGE 00 === "
+./00-preprocessor.sh $CIRCUIT1
+./00-preprocessor.sh $CIRCUIT2
+
+echo " === STAGE 01 === "
+./01-partial-proof.sh $CIRCUIT1
+./01-partial-proof.sh $CIRCUIT2
+
+echo " === STAGE 02 === "
+./02-gen-challenges.sh
+
+echo " === STAGE 03 === "
+./03-compute-combined-Q.sh $CIRCUIT1 0
+./03-compute-combined-Q.sh $CIRCUIT2 `cat $CIRCUIT1-theta-power.txt`
+
+echo " === STAGE 04 === "
+./04-aggregated-FRI.sh
+
+echo " === STAGE 05 === "
+./05-consistency-checks.sh $CIRCUIT1
+./05-consistency-checks.sh $CIRCUIT2
+
+echo " === STAGE 06 === "
+./06-merge-proofs.sh
+
+
+