diff --git a/.github/workflows/check-proof-producer.yml b/.github/workflows/check-proof-producer.yml new file mode 100644 index 0000000000..7bf071e314 --- /dev/null +++ b/.github/workflows/check-proof-producer.yml @@ -0,0 +1,256 @@ +name: Check Proof producer + +on: + workflow_call: + +env: + CAA_ARTIFACT_NAME: circuits-and-assignments + TO_ARTIFACT_NAME: transpiler-output + INTEGRATION_TESTING_TARGETS: | + arithmetics_cpp_example + polynomial_cpp_example + poseidon_cpp_example + merkle_tree_poseidon_cpp_example + uint_remainder_cpp + compare_eq_cpp + private_input_cpp + # uint_shift_left + # uint_bit_decomposition + # uint_bit_composition + +jobs: + prepare-targets: + name: Prepare targets strings + runs-on: ubuntu-22.04 + if: | + always() && !cancelled() + outputs: + evm-targets: ${{ steps.get-targets.outputs.evm-targets }} + prover-targets: ${{ steps.get-targets.outputs.prover-targets }} + steps: + - name: Set targets for integration testing + id: get-targets + run: | + targets_str=$(echo "${{ env.INTEGRATION_TESTING_TARGETS }}" | awk 'NF {print "transpiler_output_" $1}') + echo "evm-targets<> $GITHUB_OUTPUT + echo "${targets_str}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + echo "prover-targets<> $GITHUB_OUTPUT + echo "${{ env.INTEGRATION_TESTING_TARGETS }}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + + get-zkllvm-run: + name: Get zkLLVM run to use artifacts from + needs: + - handle-syncwith + if: | + always() && !cancelled() && + (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') + runs-on: ubuntu-22.04 + outputs: + run-id: ${{ steps.get-run-id.outputs.run-id }} + steps: + - name: Get run ID of zkLLVM + id: get-run-id + env: + GH_TOKEN: ${{ github.token }} + run: | + prs_refs="${{ needs.handle-syncwith.outputs.prs-refs }}" + zkllvm_ref="master" + zkllvm_repo="NilFoundation/zkLLVM" + + while read -r line; do + echo "$line" + if [[ $line == "${zkllvm_repo}:"* ]]; then + zkllvm_ref=$(echo "$line" | cut -d ' ' -f 2) + break + fi + done <<< "$prs_refs" + + if [[ $zkllvm_ref == refs/pull/* ]]; then + echo "Considering reference ${zkllvm_ref} a pr ref" + pr_number=${zkllvm_ref#refs/pull/} + pr_number=${pr_number%/merge} + sha=$(gh api repos/${zkllvm_repo}/pulls/$pr_number --jq '.head.sha') + elif [[ $zkllvm_ref == refs/tags/* ]]; then + echo "Considering reference ${zkllvm_ref} a tag" + tag=${zkllvm_ref#refs/tags/} + sha=$(gh api repos/${zkllvm_repo}/git/ref/tags/$tag --jq '.object.sha') + else + echo "Considering reference ${zkllvm_ref} a branch" + branch=${zkllvm_ref#refs/heads/} + # We can already fetch run_id here, but better fit to common approach with extra query by sha + sha=$(gh api "repos/${zkllvm_repo}/actions/workflows/nix_build_linux.yml/runs?branch=${branch}&status=completed&per_page=1" \ + --jq '.workflow_runs[0].head_sha') + fi + + echo "Using head sha: ${sha}" + run_id=$(gh api "repos/${zkllvm_repo}/actions/workflows/nix_build_linux.yml/runs?head_sha=${sha}&status=completed&per_page=1" \ + --jq '.workflow_runs[0].id') + if [ -z "${run_id}" ]; then + echo no run ID fetched + exit 1 + fi + echo "Run ID: ${run_id}" + for artifact in "${{ env.CAA_ARTIFACT_NAME }}" "${{ env.TO_ARTIFACT_NAME }}"; do + # Check if the artifact exists in the run + if ! gh run view ${run_id} --repo ${zkllvm_repo} | grep "$artifact"; then + echo "Artifact '$artifact' not found in run ${run_id}" + exit 1 + fi + done + echo "run-id=${run_id}" >> $GITHUB_OUTPUT + + build-and-generate-proofs: + name: Build prover, generate proofs for circuits + runs-on: [self-hosted, Linux, X64, aws_autoscaling] + needs: + - handle-syncwith + - get-zkllvm-run + - prepare-targets + if: | + always() && !cancelled() && + (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') && + (needs.get-zkllvm-run.result == 'success' || needs.get-zkllvm-run.result == 'skipped') && + (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped') + outputs: + artifact-name: ${{ steps.artifact-name.outputs.merged }} + + steps: + # https://github.com/actions/checkout/issues/1552 + - name: Clean up after previous checkout + run: chmod +w -R ${GITHUB_WORKSPACE}; rm -rf ${GITHUB_WORKSPACE}/*; + + - name: Checkout Proof Producer + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + + - name: Checkout submodules to specified refs + if: inputs.submodules-refs != '' + uses: NilFoundation/ci-cd/actions/recursive-checkout@v1.2.1 + with: + refs: ${{ needs.handle-syncwith.outputs.prs-refs }} + paths: | + ${{ github.workspace }}/** + !${{ github.workspace }}/ + !${{ github.workspace }}/**/.git/** + + - name: Run checks + run: nix -L build '.?submodules=1#' + env: + NIX_CONFIG: | + cores = 2 + max-jobs = 1 + + - name: Set usefull strings + id: strings + run: | + echo "artifact-dir=$(realpath ${{ github.workspace }}/../artifacts)" >> $GITHUB_OUTPUT + + - name: Download circuits and assignments artifact + uses: dawidd6/action-download-artifact@v3 + with: + repo: NilFoundation/zkLLVM + name: ${{ env.CAA_ARTIFACT_NAME }} + path: ${{ steps.strings.outputs.artifact-dir }} + run_id: ${{ needs.get-zkllvm-run.outputs.run-id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + skip_unpack: true # It can't unpack such large files (>2Gb for some circuits) + + - name: Extract circuits and assignments artifact + working-directory: ${{ steps.strings.outputs.artifact-dir }} + run: | + unzip -o ${{ env.CAA_ARTIFACT_NAME }}.zip + + - name: List artifacts + working-directory: ${{ steps.strings.outputs.artifact-dir }} + run: find . + + - name: Make proofs for pairs + working-directory: ${{ steps.strings.outputs.artifact-dir }} + run: | + extra_args="" + if [[ "true" == "true" ]]; then + extra_args="--multithreaded " + fi + targets_str=$(echo "${{ needs.prepare-targets.outputs.prover-targets }}" | awk '{$1=$1};1' | sed '/^$/d' | tr '\n' ' ' | sed 's/ $//') + echo "targets from input: ${targets_str}" + ${{ github.workspace }}/tests/make_proof_for_pairs.sh --use-nix ${extra_args} ${targets_str} + + - name: Download transpiler output artifact + id: download-to-artifact + uses: dawidd6/action-download-artifact@v3 + with: + repo: NilFoundation/zkLLVM + name: ${{ env.TO_ARTIFACT_NAME }} + path: ${{ steps.strings.outputs.artifact-dir }} + run_id: ${{ needs.get-zkllvm-run.outputs.run-id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Merge proofs into transpiler output + working-directory: ${{ steps.strings.outputs.artifact-dir }} + run: | + copy_failed=0 + while read dir; do + base_name=${dir#./transpiler_output_} + + if [[ -d "$base_name" ]]; then + if ! cp "${base_name}/proof.bin" "${dir}/"; then + # fails if artifact contains dirs targets we didn't produce proofs for + echo "Failed to copy proof.bin to ${dir}" >&2 + copy_failed=1 + else + echo "proof.bin added to ${dir}" + fi + else + echo "Error: No matching directory found for ${dir}" >&2 + fi + done < <(find . -type d -name "transpiler_output_*") + + if [ $copy_failed -eq 1 ]; then + echo "One or more copy operations failed." >&2 + fi + + - name: Set aritfact name + id: artifact-name + run: | + echo "merged=transpiler-output-merged-proofs" >> $GITHUB_OUTPUT + + - name: Upload merged artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.artifact-name.outputs.merged }} + path: | + ${{ steps.strings.outputs.artifact-dir }}/transpiler_output_* + + # TODO: add package derivation to nix, upload its result + # - name: Upload .deb package + # if: matrix.cpp-compiler == 'clang++' && matrix.build-type == 'Release' + # uses: actions/upload-artifact@v4 + # with: + # name: proof-producer.deb + # path: | + # ${{ steps.strings.outputs.build-dir }}/proof-producer*.deb + + + verify-proof-producer-proofs: + name: Verify proof-producer proofs with EVM-placeholder + needs: + - handle-syncwith + - build-and-generate-proofs + - prepare-targets + if: | + always() && !cancelled() && + (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') && + (needs.build-and-generate-proofs.result == 'success' || needs.build-and-generate-proofs.result == 'skipped') && + (needs.prepare-targets.result == 'success' || needs.prepare-targets.result == 'skipped') + uses: NilFoundation/evm-placeholder-verification/.github/workflows/reusable-verify-proofs.yml@2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d + with: + artifact-name: ${{ needs.build-and-generate-proofs.outputs.artifact-name }} + evm-placeholder-verification-ref: 2a0ef5fc67e97be8e3c7b59338cf9eecd030c57d + refs: ${{ needs.handle-syncwith.outputs.prs-refs }} + test-names: ${{ needs.prepare-targets.outputs.evm-targets }} diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 7f2f09e15c..093b840c31 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -29,3 +29,10 @@ jobs: always() && !cancelled() secrets: inherit + check-proof-producer: + name: Check Proof Producer + uses: ./.github/workflows/check-proof-producer.yml + if: | + always() && !cancelled() + secrets: inherit +