CI #66
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| workflow_call: | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| packages: write | |
| defaults: | |
| run: | |
| shell: bash | |
| concurrency: | |
| group: "${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}" | |
| cancel-in-progress: true | |
| jobs: | |
| calculate_matrix: | |
| name: Calculate job matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| jobs: "${{ steps.jobs.outputs.jobs }}" | |
| steps: | |
| - name: Checkout the source code | |
| uses: actions/checkout@v4 | |
| - name: Calculate the CI job matrix | |
| run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT | |
| id: jobs | |
| job: | |
| name: "${{ matrix.name }}" | |
| needs: [calculate_matrix] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}" | |
| if: "fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null" | |
| defaults: | |
| run: | |
| shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}" | |
| timeout-minutes: 600 | |
| runs-on: "${{ matrix.os }}" | |
| steps: | |
| - if: "contains(matrix.os, 'windows')" | |
| uses: msys2/[email protected] | |
| with: | |
| msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}" | |
| update: false | |
| release: true | |
| path-type: inherit | |
| install: "make dos2unix diffutils\n" | |
| - name: disable git crlf conversion | |
| run: git config --global core.autocrlf false | |
| - name: checkout the source code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - name: configure the PR in which the error message will be posted | |
| run: 'echo "[CI_PR_NUMBER=$num]"' | |
| env: | |
| num: "${{ github.event.number }}" | |
| if: "success() && github.event_name == 'pull_request'" | |
| - name: add extra environment variables | |
| run: src/ci/scripts/setup-environment.sh | |
| env: | |
| EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" | |
| - name: ensure the channel matches the target branch | |
| run: src/ci/scripts/verify-channel.sh | |
| - name: collect CPU statistics | |
| run: src/ci/scripts/collect-cpu-stats.sh | |
| - name: show the current environment | |
| run: src/ci/scripts/dump-environment.sh | |
| - name: install awscli | |
| run: src/ci/scripts/install-awscli.sh | |
| - name: install sccache | |
| run: src/ci/scripts/install-sccache.sh | |
| - name: select Xcode | |
| run: src/ci/scripts/select-xcode.sh | |
| - name: install clang | |
| run: src/ci/scripts/install-clang.sh | |
| - name: install tidy | |
| run: src/ci/scripts/install-tidy.sh | |
| - name: install WIX | |
| run: src/ci/scripts/install-wix.sh | |
| - name: disable git crlf conversion | |
| run: src/ci/scripts/disable-git-crlf-conversion.sh | |
| - name: checkout submodules | |
| run: src/ci/scripts/checkout-submodules.sh | |
| - name: install MSYS2 | |
| run: src/ci/scripts/install-msys2.sh | |
| - name: install MinGW | |
| run: src/ci/scripts/install-mingw.sh | |
| - name: install ninja | |
| run: src/ci/scripts/install-ninja.sh | |
| - name: enable ipv6 on Docker | |
| run: src/ci/scripts/enable-docker-ipv6.sh | |
| - name: disable git crlf conversion | |
| run: src/ci/scripts/disable-git-crlf-conversion.sh | |
| - name: ensure line endings are correct | |
| run: src/ci/scripts/verify-line-endings.sh | |
| - name: ensure backported commits are in upstream branches | |
| run: src/ci/scripts/verify-backported-commits.sh | |
| - name: ensure the stable version number is correct | |
| run: src/ci/scripts/verify-stable-version-number.sh | |
| - name: run the build | |
| run: src/ci/scripts/run-build-from-ci.sh 2>&1 | |
| env: | |
| AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" | |
| AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" | |
| TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" | |
| - name: create github artifacts | |
| run: src/ci/scripts/create-doc-artifacts.sh | |
| - name: upload artifacts to github | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: "${{ env.DOC_ARTIFACT_NAME }}" | |
| path: obj/artifacts/doc | |
| if-no-files-found: ignore | |
| retention-days: 5 | |
| - name: upload artifacts to S3 | |
| run: src/ci/scripts/upload-artifacts.sh | |
| env: | |
| AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" | |
| AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" | |
| if: "success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" | |
| master: | |
| name: master | |
| runs-on: ubuntu-latest | |
| env: | |
| SCCACHE_BUCKET: rust-lang-ci-sccache2 | |
| DEPLOY_BUCKET: rust-lang-ci2 | |
| TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" | |
| TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues" | |
| TOOLSTATE_PUBLISH: 1 | |
| CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL | |
| ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 | |
| AWS_REGION: us-west-1 | |
| CACHE_DOMAIN: ci-caches.rust-lang.org | |
| if: "github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'" | |
| steps: | |
| - name: checkout the source code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - name: publish toolstate | |
| run: src/ci/publish_toolstate.sh | |
| shell: bash | |
| env: | |
| TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" | |
| try-success: | |
| needs: [job] | |
| if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" | |
| steps: | |
| - name: mark the job as a success | |
| run: exit 0 | |
| shell: bash | |
| name: bors build finished | |
| runs-on: ubuntu-latest | |
| try-failure: | |
| needs: [job] | |
| if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" | |
| steps: | |
| - name: mark the job as a failure | |
| run: exit 1 | |
| shell: bash | |
| name: bors build finished | |
| runs-on: ubuntu-latest | |
| auto-success: | |
| needs: [job] | |
| if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" | |
| steps: | |
| - name: mark the job as a success | |
| run: exit 0 | |
| shell: bash | |
| name: bors build finished | |
| runs-on: ubuntu-latest | |
| auto-failure: | |
| needs: [job] | |
| if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" | |
| steps: | |
| - name: mark the job as a failure | |
| run: exit 1 | |
| shell: bash | |
| name: bors build finished | |
| runs-on: ubuntu-latest | |
| build: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: macos-14 | |
| triple: aarch64-apple-darwin | |
| - os: macos-13 | |
| triple: x86_64-apple-darwin | |
| - os: buildjet-32vcpu-ubuntu-2004 | |
| triple: x86_64-unknown-linux-gnu | |
| - os: buildjet-32vcpu-ubuntu-2204-arm | |
| triple: aarch64-unknown-linux-gnu | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Install nightly toolchain | |
| id: rustc-toolchain | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-01-25 | |
| default: true | |
| - uses: lukka/[email protected] | |
| - name: Show rust version | |
| run: | | |
| cargo version | |
| rustup toolchain list | |
| - name: Check out succinctlabs/rust | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: "recursive" | |
| path: rust | |
| fetch-depth: 0 | |
| - name: Check out succinctlabs/sp1 | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: succinctlabs/sp1 | |
| ref: dev | |
| path: sp1 | |
| - name: Install RISC-V Toolchain | |
| run: | | |
| mkdir -p ~/.sp1/bin | |
| cat << 'EOF' > install_toolchain.sh | |
| #!/bin/bash | |
| set -e | |
| SP1_DIR=~/.sp1 | |
| SP1_BIN_DIR=$SP1_DIR/bin | |
| determine_platform() { | |
| case "$(uname -s)" in | |
| Linux*) PLATFORM=linux ;; | |
| Darwin*) PLATFORM=darwin ;; | |
| MINGW*|MSYS*|CYGWIN*) PLATFORM=win32 ;; | |
| *) echo "Unsupported platform" >&2; exit 1 ;; | |
| esac | |
| } | |
| determine_architecture() { | |
| case "$(uname -m)" in | |
| x86_64) ARCHITECTURE=amd64 ;; | |
| arm64|aarch64) ARCHITECTURE=arm64 ;; | |
| *) echo "Unsupported architecture" >&2; exit 1 ;; | |
| esac | |
| } | |
| download() { | |
| if command -v curl &>/dev/null; then | |
| curl -LsSf "$1" -o "$2" | |
| elif command -v wget &>/dev/null; then | |
| wget -q "$1" -O "$2" | |
| else | |
| echo "Neither curl nor wget is available" >&2 | |
| exit 1 | |
| fi | |
| } | |
| ensure() { | |
| "$@" | |
| if [ $? -ne 0 ]; then | |
| echo "Command failed: $*" >&2 | |
| exit 1 | |
| fi | |
| } | |
| say() { | |
| echo "$@" >&2 | |
| } | |
| err() { | |
| say "$@" | |
| exit 1 | |
| } | |
| install_c_toolchain() { | |
| say "installing C++ toolchain for RISC-V" | |
| determine_platform | |
| determine_architecture | |
| case $PLATFORM in | |
| linux) | |
| case $ARCHITECTURE in | |
| amd64) TOOLCHAIN_URL="https://github.com/risc0/toolchain/releases/download/2022.03.25/riscv32im-linux-x86_64.tar.xz" ;; | |
| *) err "unsupported architecture for Linux: $ARCHITECTURE" ;; | |
| esac | |
| ;; | |
| darwin) | |
| case $ARCHITECTURE in | |
| arm64) TOOLCHAIN_URL="https://github.com/risc0/toolchain/releases/download/2022.03.25/riscv32im-osx-arm64.tar.xz" ;; | |
| amd64) TOOLCHAIN_URL="https://github.com/risc0/toolchain/releases/download/2022.03.25/riscv32im-osx-x86_64.tar.xz" ;; | |
| *) err "unsupported architecture for macOS: $ARCHITECTURE" ;; | |
| esac | |
| ;; | |
| win32) | |
| case $ARCHITECTURE in | |
| amd64) TOOLCHAIN_URL="https://github.com/risc0/toolchain/releases/download/2022.03.25/riscv32im-windows-x86_64.tar.xz" ;; | |
| *) err "unsupported architecture for Windows: $ARCHITECTURE" ;; | |
| esac | |
| ;; | |
| *) | |
| err "unsupported platform: $PLATFORM" | |
| ;; | |
| esac | |
| TOOLCHAIN_ARCHIVE="/tmp/$(basename $TOOLCHAIN_URL)" | |
| TOOLCHAIN_INSTALL_DIR="$SP1_DIR/riscv" | |
| ensure download "$TOOLCHAIN_URL" "$TOOLCHAIN_ARCHIVE" | |
| mkdir -p "$TOOLCHAIN_INSTALL_DIR" | |
| if [ "$PLATFORM" = "win32" ]; then | |
| ensure unzip "$TOOLCHAIN_ARCHIVE" -d "$TOOLCHAIN_INSTALL_DIR" >/dev/null 2>&1 | |
| else | |
| ensure tar -xvf "$TOOLCHAIN_ARCHIVE" -C "$TOOLCHAIN_INSTALL_DIR" >/dev/null 2>&1 | |
| fi | |
| rm -f "$TOOLCHAIN_ARCHIVE" | |
| GCC_PATH=$(find "$TOOLCHAIN_INSTALL_DIR" -name 'riscv32-unknown-elf-gcc' -type f | head -n 1) | |
| if [ -z "$GCC_PATH" ]; then | |
| err "riscv32-unknown-elf-gcc not found in extracted archive" | |
| fi | |
| ln -sf "$GCC_PATH" "$SP1_BIN_DIR/riscv32-unknown-elf-gcc" | |
| say "installed C++ toolchain for RISC-V" | |
| } | |
| install_c_toolchain | |
| EOF | |
| chmod +x install_toolchain.sh | |
| ./install_toolchain.sh | |
| - name: Setup CC for RISC-V | |
| run: | | |
| echo "PATH=$HOME/.sp1/bin:$PATH" >> $GITHUB_ENV | |
| echo "CC_riscv32im_succinct_zkvm_elf=$HOME/.sp1/bin/riscv32-unknown-elf-gcc" >> $GITHUB_ENV | |
| echo "CXX_riscv32im_succinct_zkvm_elf=$HOME/.sp1/bin/riscv32-unknown-elf-g++" >> $GITHUB_ENV | |
| echo "AR_riscv32im_succinct_zkvm_elf=$HOME/.sp1/bin/riscv32-unknown-elf-ar" >> $GITHUB_ENV | |
| - name: Configure Rust build | |
| run: | | |
| cd rust | |
| cp config.toml.example config.toml | |
| echo "[target.riscv32im-succinct-zkvm-elf]" >> config.toml | |
| echo "cc = \"$HOME/.sp1/bin/riscv32-unknown-elf-gcc\"" >> config.toml | |
| echo "cxx = \"$HOME/.sp1/bin/riscv32-unknown-elf-g++\"" >> config.toml | |
| echo "ar = \"$HOME/.sp1/bin/riscv32-unknown-elf-ar\"" >> config.toml | |
| - name: Build | |
| run: | | |
| cd sp1/cli | |
| GITHUB_ACTIONS=false SP1_BUILD_DIR=$GITHUB_WORKSPACE cargo run --bin cargo-prove -- prove build-toolchain | |
| - name: Archive build output | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: rust-toolchain-${{ matrix.triple }} | |
| path: sp1/cli/rust-toolchain-${{ matrix.triple }}.tar.gz |