From 706d4bc8882731cf9a560d826ff3b7a16ff55dd7 Mon Sep 17 00:00:00 2001 From: Ratan Kaliani Date: Wed, 18 Sep 2024 17:19:50 -0700 Subject: [PATCH] feat: succinct 1.81.0 --- .github/workflows/ci.yml | 265 +++--------------- .github/workflows/release.yml | 39 +++ compiler/rustc_target/src/spec/mod.rs | 8 +- .../spec/targets/riscv32im_risc0_zkvm_elf.rs | 2 +- library/std/src/sys/pal/zkvm/abi.rs | 10 +- library/std/src/sys/pal/zkvm/mod.rs | 11 +- .../riscv32im-risc0-zkvm-elf.md | 19 +- src/tools/build-manifest/src/main.rs | 2 +- tests/assembly/targets/targets-elf.rs | 6 +- 9 files changed, 107 insertions(+), 255 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8032154a7365..fb0fd6c3c94a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,240 +1,57 @@ -# This file defines our primary CI workflow that runs on pull requests -# and also on pushes to special branches (auto, try). -# -# The actual definition of the executed jobs is calculated by a Python -# script located at src/ci/github-actions/calculate-job-matrix.py, which -# uses job definition data from src/ci/github-actions/jobs.yml. -# You should primarily modify the `jobs.yml` file if you want to modify -# what jobs are executed in CI. - name: CI -on: - push: - branches: - - auto - - try - - try-perf - - automation/bors/try - pull_request: - branches: - - "**" -permissions: - contents: read - packages: write - -defaults: - run: - # On Linux, macOS, and Windows, use the system-provided bash as the default - # shell. (This should only make a difference on Windows, where the default - # shell is PowerShell.) - shell: bash +on: + workflow_call: + workflow_dispatch: -concurrency: - # For a given workflow, if we push to the same branch, cancel all previous builds on that branch. - # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which - # are all triggered on the same branch, but which should be able to run concurrently. - group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }} - cancel-in-progress: true -env: - TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" - # This will be empty in PR jobs. - TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }} jobs: - # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml. - # It calculates which jobs should be executed, based on the data of the ${{ github }} context. - # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml. - calculate_matrix: - name: Calculate job matrix - runs-on: ubuntu-latest - outputs: - jobs: ${{ steps.jobs.outputs.jobs }} - run_type: ${{ steps.jobs.outputs.run_type }} - steps: - - name: Checkout the source code - uses: actions/checkout@v4 - - name: Calculate the CI job matrix - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT - id: jobs - job: - name: ${{ matrix.name }} - needs: [ calculate_matrix ] - runs-on: "${{ matrix.os }}" - defaults: - run: - shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }} - timeout-minutes: 240 - env: - CI_JOB_NAME: ${{ matrix.image }} - CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse - # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs. - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SCCACHE_BUCKET: rust-lang-ci-sccache2 - CACHE_DOMAIN: ci-caches.rust-lang.org - continue-on-error: ${{ matrix.continue_on_error || false }} + build: strategy: + fail-fast: false matrix: - # Check the `calculate_matrix` job to see how is the matrix defined. - include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} + include: + - os: macos-14 + triple: aarch64-apple-darwin + - os: macos-13 + triple: x86_64-apple-darwin + - os: buildjet-32vcpu-ubuntu-2004 + triple: x86_64-unknown-linux-gnu + - os: buildjet-32vcpu-ubuntu-2204-arm + triple: aarch64-unknown-linux-gnu + runs-on: ${{ matrix.os }} steps: - - if: contains(matrix.os, 'windows') - uses: msys2/setup-msys2@v2.22.0 + - name: Install Rust + uses: actions-rs/toolchain@v1 with: - # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64. - msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }} - # don't try to download updates for already installed packages - update: false - # don't try to use the msys that comes built-in to the github runner, - # so we can control what is installed (i.e. not python) - release: true - # Inherit the full path from the Windows environment, with MSYS2's */bin/ - # dirs placed in front. This lets us run Windows-native Python etc. - path-type: inherit - install: > - make + toolchain: stable + - uses: lukka/get-cmake@v3.27.4 - - name: disable git crlf conversion - run: git config --global core.autocrlf false + - name: Show rust version + run: | + cargo version + rustup toolchain list - - name: checkout the source code - uses: actions/checkout@v4 + - name: Check out succinctlabs/rust + uses: actions/checkout@v3 with: - fetch-depth: 2 - - # Rust Log Analyzer can't currently detect the PR number of a GitHub - # Actions build on its own, so a hint in the log message is needed to - # point it in the right direction. - - name: configure the PR in which the error message will be posted - run: echo "[CI_PR_NUMBER=$num]" - env: - num: ${{ github.event.number }} - if: needs.calculate_matrix.outputs.run_type == 'pr' - - - name: add extra environment variables - run: src/ci/scripts/setup-environment.sh - env: - # Since it's not possible to merge `${{ matrix.env }}` with the other - # variables in `job..env`, the variables defined in the matrix - # are passed to the `setup-environment.sh` script encoded in JSON, - # which then uses log commands to actually set them. - EXTRA_VARIABLES: ${{ toJson(matrix.env) }} - - - name: ensure the channel matches the target branch - run: src/ci/scripts/verify-channel.sh - - - name: collect CPU statistics - run: src/ci/scripts/collect-cpu-stats.sh - - - name: show the current environment - run: src/ci/scripts/dump-environment.sh - - - name: install awscli - run: src/ci/scripts/install-awscli.sh - - - name: install sccache - run: src/ci/scripts/install-sccache.sh - - - name: select Xcode - run: src/ci/scripts/select-xcode.sh - - - name: install clang - run: src/ci/scripts/install-clang.sh + submodules: "recursive" + path: rust + fetch-depth: 0 - - name: install tidy - run: src/ci/scripts/install-tidy.sh - - - name: install WIX - run: src/ci/scripts/install-wix.sh - - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - - - name: checkout submodules - run: src/ci/scripts/checkout-submodules.sh - - - name: install MinGW - run: src/ci/scripts/install-mingw.sh - - - name: install ninja - run: src/ci/scripts/install-ninja.sh - - - name: enable ipv6 on Docker - run: src/ci/scripts/enable-docker-ipv6.sh - - # Disable automatic line ending conversion (again). On Windows, when we're - # installing dependencies, something switches the git configuration directory or - # re-enables autocrlf. We've not tracked down the exact cause -- and there may - # be multiple -- but this should ensure submodules are checked out with the - # appropriate line endings. - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - - - name: ensure line endings are correct - run: src/ci/scripts/verify-line-endings.sh - - - name: ensure backported commits are in upstream branches - run: src/ci/scripts/verify-backported-commits.sh - - - name: ensure the stable version number is correct - run: src/ci/scripts/verify-stable-version-number.sh - - - name: run the build - # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs. - run: src/ci/scripts/run-build-from-ci.sh 2>&1 - env: - AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} - - - name: create github artifacts - run: src/ci/scripts/create-doc-artifacts.sh - - - name: upload artifacts to github - uses: actions/upload-artifact@v4 + - name: Check out succinctlabs/sp1 + uses: actions/checkout@v3 with: - # name is set in previous step - name: ${{ env.DOC_ARTIFACT_NAME }} - path: obj/artifacts/doc - if-no-files-found: ignore - retention-days: 5 + repository: succinctlabs/sp1 + ref: dev + path: sp1 - - name: upload artifacts to S3 - run: src/ci/scripts/upload-artifacts.sh - env: - AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }} - # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy - # builders *should* have the AWS credentials available. Still, explicitly - # adding the condition is helpful as this way CI will not silently skip - # deploying artifacts from a dist builder if the variables are misconfigured, - # erroring about invalid credentials instead. - if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1' + - name: Build + run: | + cd sp1/cli + GITHUB_ACTIONS=false SP1_BUILD_DIR=$GITHUB_WORKSPACE cargo run --bin cargo-prove -- prove build-toolchain - # This job isused to tell bors the final status of the build, as there is no practical way to detect - # when a workflow is successful listening to webhooks only in our current bors implementation (homu). - outcome: - name: bors build finished - runs-on: ubuntu-latest - needs: [ calculate_matrix, job ] - # !cancelled() executes the job regardless of whether the previous jobs passed or failed - if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }} - steps: - - name: checkout the source code - uses: actions/checkout@v4 + - name: Archive build output + uses: actions/upload-artifact@v3 with: - fetch-depth: 2 - # Calculate the exit status of the whole CI workflow. - # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully). - # If a some dependent job has failed, this exits with 1. - - name: calculate the correct exit status - run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}' - # Publish the toolstate if an auto build succeeds (just before push to master) - - name: publish toolstate - run: src/ci/publish_toolstate.sh - shell: bash - if: needs.calculate_matrix.outputs.run_type == 'auto' - env: - TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues - TOOLSTATE_PUBLISH: 1 + name: rust-toolchain-${{ matrix.triple }} + path: sp1/cli/rust-toolchain-${{ matrix.triple }}.tar.gz \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000000..c4b2c96059dc --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,39 @@ +name: Release +on: + push: + tags: + - "*" +jobs: + build: + uses: ./.github/workflows/ci.yml + release: + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + - name: List artifact contents + run: | + echo "Listing artifacts directory:" + ls -R artifacts + - name: Create release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "Installing gh CLI..." + curl -L https://github.com/cli/cli/releases/download/v2.17.0/gh_2.17.0_linux_amd64.tar.gz | \ + tar xvz --strip-components=2 --exclude=man + chmod +x ./gh + mkdir tars + find artifacts -name "*.tar.gz" -exec mv {} tars/ \; + if [ "$(ls -A tars)" ]; then + ./gh release create --repo "$GITHUB_REPOSITORY" "$GITHUB_REF_NAME" ./tars/* || \ + ./gh release upload --repo "$GITHUB_REPOSITORY" "$GITHUB_REF_NAME" ./tars/* + else + echo "No files to release. Exiting." + exit 1 + fi \ No newline at end of file diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index 607eeac7ccdc..442921ef2d66 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -1251,9 +1251,9 @@ impl StackProbeType { .and_then(|o| o.as_array()) .ok_or_else(|| "expected `min-llvm-version-for-inline` to be an array")?; let mut iter = min_version.into_iter().map(|v| { - let int = v.as_u64().ok_or_else( - || "expected `min-llvm-version-for-inline` values to be integers", - )?; + let int = v.as_u64().ok_or_else(|| { + "expected `min-llvm-version-for-inline` values to be integers" + })?; u32::try_from(int) .map_err(|_| "`min-llvm-version-for-inline` values don't convert to u32") }); @@ -1747,7 +1747,7 @@ supported_targets! { ("x86_64-unikraft-linux-musl", x86_64_unikraft_linux_musl), ("riscv32i-unknown-none-elf", riscv32i_unknown_none_elf), - ("riscv32im-risc0-zkvm-elf", riscv32im_risc0_zkvm_elf), + ("riscv32im-succinct-zkvm-elf", riscv32im_succinct_zkvm_elf), ("riscv32im-unknown-none-elf", riscv32im_unknown_none_elf), ("riscv32ima-unknown-none-elf", riscv32ima_unknown_none_elf), ("riscv32imc-unknown-none-elf", riscv32imc_unknown_none_elf), diff --git a/compiler/rustc_target/src/spec/targets/riscv32im_risc0_zkvm_elf.rs b/compiler/rustc_target/src/spec/targets/riscv32im_risc0_zkvm_elf.rs index 726778aca0d8..d6228d415f1b 100644 --- a/compiler/rustc_target/src/spec/targets/riscv32im_risc0_zkvm_elf.rs +++ b/compiler/rustc_target/src/spec/targets/riscv32im_risc0_zkvm_elf.rs @@ -16,7 +16,7 @@ pub fn target() -> Target { options: TargetOptions { os: "zkvm".into(), - vendor: "risc0".into(), + vendor: "succinct".into(), linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes), linker: Some("rust-lld".into()), cpu: "generic-rv32".into(), diff --git a/library/std/src/sys/pal/zkvm/abi.rs b/library/std/src/sys/pal/zkvm/abi.rs index 53332d90e02c..2f94ec4b19be 100644 --- a/library/std/src/sys/pal/zkvm/abi.rs +++ b/library/std/src/sys/pal/zkvm/abi.rs @@ -1,10 +1,10 @@ -//! ABI definitions for symbols exported by risc0-zkvm-platform. +//! ABI definitions for symbols exported by sp1-zkvm. -// Included here so we don't have to depend on risc0-zkvm-platform. +// Included here so we don't have to depend on sp1-zkvm. // // FIXME: Should we move this to the "libc" crate? It seems like other // architectures put a lot of this kind of stuff there. But there's -// currently no risc0 fork of the libc crate, so we'd either have to +// currently no succinct fork of the libc crate, so we'd either have to // fork it or upstream it. #![allow(dead_code)] @@ -19,7 +19,7 @@ pub mod fileno { } extern "C" { - // Wrappers around syscalls provided by risc0-zkvm-platform: + // Wrappers around syscalls provided by sp1-zkvm: pub fn sys_halt(); pub fn sys_output(output_id: u32, output_value: u32); pub fn sys_sha_compress( @@ -34,7 +34,7 @@ extern "C" { buf: *const u8, count: u32, ); - pub fn sys_rand(recv_buf: *mut u32, words: usize); + pub fn sys_rand(recv_buf: *mut u8, words: usize); pub fn sys_panic(msg_ptr: *const u8, len: usize) -> !; pub fn sys_log(msg_ptr: *const u8, len: usize); pub fn sys_cycle_count() -> usize; diff --git a/library/std/src/sys/pal/zkvm/mod.rs b/library/std/src/sys/pal/zkvm/mod.rs index 651f25d66236..518e882275f1 100644 --- a/library/std/src/sys/pal/zkvm/mod.rs +++ b/library/std/src/sys/pal/zkvm/mod.rs @@ -1,4 +1,4 @@ -//! System bindings for the risc0 zkvm platform +//! System bindings for sp1-zkvm //! //! This module contains the facade (aka platform-specific) implementations of //! OS level functionality for zkvm. @@ -65,9 +65,12 @@ pub fn abort_internal() -> ! { } pub fn hashmap_random_keys() -> (u64, u64) { - let mut buf = [0u32; 4]; + let mut buf = [0u8; 16]; unsafe { - abi::sys_rand(buf.as_mut_ptr(), 4); + abi::sys_rand(buf.as_mut_ptr(), buf.len()); }; - ((buf[0] as u64) << 32 + buf[1] as u64, (buf[2] as u64) << 32 + buf[3] as u64) + + let a = u64::from_le_bytes(buf[0..8].try_into().unwrap()); + let b = u64::from_le_bytes(buf[8..16].try_into().unwrap()); + (a, b) } diff --git a/src/doc/rustc/src/platform-support/riscv32im-risc0-zkvm-elf.md b/src/doc/rustc/src/platform-support/riscv32im-risc0-zkvm-elf.md index 1fdd594012c2..3aa5d1099813 100644 --- a/src/doc/rustc/src/platform-support/riscv32im-risc0-zkvm-elf.md +++ b/src/doc/rustc/src/platform-support/riscv32im-risc0-zkvm-elf.md @@ -2,13 +2,7 @@ **Tier: 3** -RISC Zero's Zero Knowledge Virtual Machine (zkVM) implementing the RV32IM instruction set. - -## Target maintainers - -- Frank Laub, `frank@risczero.com`, https://github.com/flaub -- Jeremy Bruestle, `jeremy@risczero.com`, https://github.com/jbruestle -- Erik Kaneda, `erik@risczero.com`, https://github.com/SchmErik +Succinct's Zero Knowledge Virtual Machine (zkVM) implementing the RV32IM instruction set. ## Background @@ -20,7 +14,7 @@ cryptographic seal. This receipt can be verified to ensure the integrity of the computation and its result. This target is implemented as software only; it has no hardware implementation. -We have a cargo extension called [cargo-risczero] that allow users to generate +We have a cargo extension called [cargo-prove] that allow users to generate project templates, install tools for improved user experience, build the binary using a docker environment and test programs. @@ -44,7 +38,7 @@ Calling `extern "C"` on the target uses the C calling convention outlined in the Programs for the zkVM could be built by adding it to the `target` list in `config.toml`. However, we recommend building programs in our starter template -generated by the [cargo-risczero] utility and the [risc0-build] crate. This +generated by the [cargo-prove] utility and the [sp1-build] crate. This crate calls `rustc` with `-C "link-arg=-Ttext=` so that it maps the text in the appropriate location as well as generating variables that represent the ELF and a unique ID associated with the ELF. The starter template provides developers @@ -57,7 +51,7 @@ Rust does not yet ship pre-compiled artifacts for this target. To compile for this target, you will either need to build Rust with the target enabled (see "Building the target" above). We do not recommend using `build-std` as we have run into issues building core in the past on our starter template. An alternate -solution is to download the risc0 tool chain by running `cargo risczero install`. +solution is to download the SP1 tool chain by running `cargo prove install`. ## Testing @@ -65,7 +59,7 @@ Note: the target is implemented as a software emulator called the zkVM and there is no hardware implementation of the target. The most practical way to test the target program is to use our starter template -that can be generated by using the `cargo risczero new` command. The template +that can be generated by using the `cargo prove new` command. The template generates a sample "host" and "guest" code. The guest code compiled to the target (which is RV32IM) whereas the "host" code is compiled to run on the programmer's machine running either a Linux distribution or macOS. The host @@ -82,5 +76,4 @@ target. On clang and ld.lld linker, it can be generated using the target `riscv32-unknown-none`. [RISC-V specification]: https://riscv.org/wp-content/uploads/2015/01/riscv-calling.pdf -[cargo-risczero]: https://docs.rs/cargo-risczero/latest/cargo_risczero/ -[risc0-build]: https://crates.io/crates/risc0-build +[sp1-build]: https://crates.io/crates/sp1-build diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index a2e7907b532e..a231f2a32d01 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -130,7 +130,7 @@ static TARGETS: &[&str] = &[ "powerpc64-unknown-linux-gnu", "powerpc64le-unknown-linux-gnu", "riscv32i-unknown-none-elf", - "riscv32im-risc0-zkvm-elf", + "riscv32im-succinct-zkvm-elf", "riscv32im-unknown-none-elf", "riscv32ima-unknown-none-elf", "riscv32imc-unknown-none-elf", diff --git a/tests/assembly/targets/targets-elf.rs b/tests/assembly/targets/targets-elf.rs index 32cce3839dc2..f9148704f8d8 100644 --- a/tests/assembly/targets/targets-elf.rs +++ b/tests/assembly/targets/targets-elf.rs @@ -366,9 +366,9 @@ //@ revisions: riscv32i_unknown_none_elf //@ [riscv32i_unknown_none_elf] compile-flags: --target riscv32i-unknown-none-elf //@ [riscv32i_unknown_none_elf] needs-llvm-components: riscv -//@ revisions: riscv32im_risc0_zkvm_elf -//@ [riscv32im_risc0_zkvm_elf] compile-flags: --target riscv32im-risc0-zkvm-elf -//@ [riscv32im_risc0_zkvm_elf] needs-llvm-components: riscv +//@ revisions: riscv32im_succinct_zkvm_elf +//@ [riscv32im_succinct_zkvm_elf] compile-flags: --target riscv32im-succinct-zkvm-elf +//@ [riscv32im_succinct_zkvm_elf] needs-llvm-components: riscv //@ revisions: riscv32im_unknown_none_elf //@ [riscv32im_unknown_none_elf] compile-flags: --target riscv32im-unknown-none-elf //@ [riscv32im_unknown_none_elf] needs-llvm-components: riscv