From f83b286d83d85bfbf641bb32de9a9ecb99e75ae2 Mon Sep 17 00:00:00 2001 From: yuluolout <107767955+yuluolout@users.noreply.github.com> Date: Thu, 18 Apr 2024 00:55:47 +1000 Subject: [PATCH] Add .circleci/config.yml --- .circleci/config.yml | 438 +++---------------------------------------- 1 file changed, 21 insertions(+), 417 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a8db8655..62291703e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,427 +1,31 @@ +# Use the latest 2.1 version of CircleCI pipeline process engine. +# See: https://circleci.com/docs/configuration-reference version: 2.1 -# Validate using `circleci config validate` -# TODO: Push relevant packages to crates.io when a version of master is tagged. - -executors: - minimal: - docker: - - image: circleci/buildpack-deps - # TODO: Add a container with fixed versions for reproducible builds from master. - docker-builder: - docker: - - image: docker:stable-git - environment: - # Necessary for `docker manifest` subcommand - DOCKER_CLI_EXPERIMENTAL: enabled - # Bump the tag version whenever changes are made. - IMAGE: &build_image "0xchain/rust-build-env:11" - docker-rust: - docker: - - image: *build_image - environment: - # See also environment variables declared in rust-build-env.Dockerfile - RUST_BACKTRACE: "1" - SCCACHE_CACHE_SIZE: 1G - NO_STD_PACKAGES: | - utils/macros-decl - algebra/u256 - algebra/primefield - algebra/elliptic-curve - crypto/hash - crypto/merkle-tree - crypto/elliptic-curve-crypto - crypto/stark - docker-npm: +# Define a job to be invoked later in a workflow. +# See: https://circleci.com/docs/jobs-steps/#jobs-overview & https://circleci.com/docs/configuration-reference/#jobs +jobs: + say-hello: + # Specify the execution environment. You can specify an image from Docker Hub or use one of our convenience images from CircleCI's Developer Hub. + # See: https://circleci.com/docs/executor-intro/ & https://circleci.com/docs/configuration-reference/#executor-job docker: - - image: node:latest -commands: - stop-if-image-exists: - steps: - - run: - name: Stop if image exists - command: | - docker manifest inspect $IMAGE && ( - # See https://support.circleci.com/hc/en-us/articles/360015562253-Conditionally-end-a-running-job-gracefully - circleci-agent step halt - ) || true - publish-image: - steps: - - run: - name: Build and publish docker image - command: | - # Switch to CI Docker ignore file. - rm .dockerignore - cp .dockerignore-ci .dockerignore - echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - docker build -t $IMAGE . -f .circleci/images/rust-build-env.Dockerfile - docker push $IMAGE - enable-sccache: - steps: - - restore_cache: - name: Restore sccache cache - keys: - - sccache-cache-v4-{{ arch }}-{{ .Environment.CIRCLE_JOB }} - - run: - name: Enable sccache - command: | - echo 'export RUSTC_WRAPPER=sccache' >> $BASH_ENV - source $BASH_ENV - save-sccache-cache: - steps: - - run: - name: Show sccache statistics - command: | - sccache --show-stats - - save_cache: - name: Save sccache cache - # We use {{ epoch }} to always upload a fresh cache: - # Of course, restore_cache will not find this exact key, - # but it will fall back to the closest key (aka the most recent). - # See https://discuss.circleci.com/t/add-mechanism-to-update-existing-cache-key/9014/13 - key: sccache-cache-v4-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ epoch }} - paths: - - "~/.cache/sccache" + # Specify the version you desire here + # See: https://circleci.com/developer/images/image/cimg/base + - image: cimg/base:current -jobs: - rebuild-docker: - executor: docker-builder - # We need more resources to build the image - resource_class: xlarge - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - publish-image - build-docker: - executor: docker-builder - # We need more resources to build the image - resource_class: xlarge - steps: - - stop-if-image-exists - - setup_remote_docker: - docker_layer_caching: true - - checkout - - publish-image - tests: - # See https://github.com/mozilla/grcov#grcov-with-travis - # Requires nightly because of: https://github.com/rust-lang/rust/issues/42524 - # Requires non-incremental build. - executor: docker-rust - # We need more resources to build the tests - resource_class: xlarge - environment: - GRCOVFLAGS: -s . --llvm --branch --ignore-not-existing --ignore "/*" - steps: - - checkout - - enable-sccache - - run: - name: Build tests with coverage - command: | - # Build with coverage settings from Dockerfile - # TODO: Build and run tests from all targets - CARGO_INCREMENTAL=0 RUSTFLAGS="$COVFLAGS" cargo +$NIGHTLY t --no-run - - save-sccache-cache - - run: - name: Run tests - command: | - # TODO: Store test results in json `--format json` - # TODO: Fix coverage on zkp-macros-lib - # TODO: Enable --nocapture - CARGO_INCREMENTAL=0 RUSTFLAGS="$COVFLAGS" cargo +$NIGHTLY t --workspace --exclude zkp-macros-lib - cargo +$NIGHTLY t --package zkp-macros-lib - - run: - name: Combine coverage - command: | - zip -0 target/ccov.zip `find . \( -name "*.gc*" \) -print` - grcov target/ccov.zip $GRCOVFLAGS -t lcov -o target/lcov.info - - run: - name: Submit to codecov - command: | - bash <(curl -s https://codecov.io/bash) - - store_artifacts: - path: target/lcov.info - destination: lcov.info - benchmarks: - executor: docker-rust - steps: - - checkout - - enable-sccache - - run: - name: Build benchmarks in release mode on stable - command: | - # Clean out old builds (if any) - rm target/release/deps/benchmark-* || true - cargo perf_all --no-run - - save-sccache-cache - - run: - name: Test benchmarks - command: | - # Actually running the benchmarks on CI is too noisy. The results - # would be meaningless. Instead we just run it to make sure they work. - # Exclude benchmarks that compare against external crates (pulls in a lot of deps) - cargo perf_all -- --test - - run: - # The artifact glob would have picked these up - name: Cleanup target folder for wokspace glob - command: rm target/release/deps/benchmark-*.d - - persist_to_workspace: - root: target/release/deps - paths: - - benchmark-* - benchmark-ec2: - # Run benchmarks on AWS EC2 - # See https://engineering.mongodb.com/post/reducing-variability-in-performance-tests-on-ec2-setup-and-key-results - executor: minimal - environment: - SSH_OPT: -o StrictHostKeyChecking=no - HOST: ec2-user@ec2-18-208-129-239.compute-1.amazonaws.com - steps: - - restore_cache: - name: Restore master benchmark results - keys: - - benchmark-criterion-v1 - - attach_workspace: - # Restoring benchmark executables. - at: /tmp/workspace - - run: - name: Execute benchmarks on ECS - command: | - # TODO: Fix concurrency. We don't want multiple benchmarks in parallel. - mkdir $CIRCLE_BUILD_NUM - cp /tmp/workspace/benchmark-* ./$CIRCLE_BUILD_NUM - scp $SSH_OPT -r ./$CIRCLE_BUILD_NUM $HOST:~/ - scp $SSH_OPT -r ./criterion.tgz $HOST:~/ && - ssh $SSH_OPT $HOST tar xzf ./criterion.tgz || - echo "No previous results cached" - for bench in ./$CIRCLE_BUILD_NUM/*; do - # Note: works better if gnuplot is installed on the machine - ssh $SSH_OPT $HOST $bench --bench --color always - done - ssh $SSH_OPT $HOST tar czf ./criterion.tgz ./target/criterion - scp $SSH_OPT -r $HOST:~/criterion.tgz . - ssh $SSH_OPT $HOST rm -r ./$CIRCLE_BUILD_NUM ./target ./criterion.tgz - - persist_to_workspace: - root: . - paths: - - criterion.tgz - - store_artifacts: - path: criterion.tgz - destination: criterion.tgz - benchmark-store: - # Stores criterion results as baseline. - executor: minimal - steps: - - attach_workspace: - # Restoring benchmark criterion result folder - at: . - - save_cache: - name: Save sccache cache - key: benchmark-criterion-v1-{{ epoch }} - paths: - - ./criterion.tgz - doc: - executor: docker-rust - steps: - - checkout - - run: - name: Build docs - command: cargo d - - store_artifacts: - path: ./target/doc - destination: doc - - run: - name: Test examples - command: cargo d --test - book: - executor: docker-rust - steps: - - checkout - - run: - # TODO: Move to docker image - name: Install mdbook - command: cargo install mdbook - - run: - name: Build book - working_directory: ./book - command: mdbook build - - store_artifacts: - path: ./book/book - destination: book - - run: - name: Compile examples - working_directory: ./book - command: mdbook test - rustformat: - executor: docker-rust - steps: - - checkout - - run: - name: Lint with rustformat nightly - command: cargo +$NIGHTLY fmt --all -- --check - # TODO: Run over wasm targets (which are not part of the workspace) - build-no-std: - executor: docker-rust - steps: - - checkout - - enable-sccache - - run: - # HACK: We try to build for Cortex-M3, not because we want to create - # IoT devices, but because there is no `std` available for this target. - # This forces the build to fail if an accidental dependency on `std` - # exists. Wasm has a working std, so is not a suitable target here. - # See: https://forge.rust-lang.org/platform-support.html - name: Check no std build - command: | - cargo +$NIGHTLY nostd_all - - save-sccache-cache - clippy: - executor: docker-rust - steps: - - checkout - - enable-sccache - - run: - name: Lint with clippy - command: | - cargo lint - - save-sccache-cache - analysis: - executor: docker-rust + # Add steps to the job + # See: https://circleci.com/docs/jobs-steps/#steps-overview & https://circleci.com/docs/configuration-reference/#steps steps: + # Checkout the code as the first step. - checkout - run: - name: Check lints.rs copies - command: | - ./lints_check.sh - - run: - name: Check for vulnerabilities in workspace - command: | - # TODO: Add --deny-warnings - # Pending - cargo generate-lockfile - cargo audit - - run: - name: Check for use of unsafe in workspace - command: | - cargo hack geiger - - run: - name: Check for outdated dependencies - command: | - cargo outdated - accept: - executor: minimal - steps: - - run: - name: Stop - command: circleci-agent step halt - update-issues: - executor: minimal - steps: - - run: - name: Install python tools - command: | - # TODO: Have a prepared executor - sudo apt-get install python3 python3-pip - pip3 install PyGithub - pip3 install numpy - - checkout - - run: - name: Update GitHub issues - command: | - # This will dry run except on master - if [ "$CIRCLE_BRANCH" == "master" ] - then - DRY_RUN=false python3 .circleci/issue_tracker.py - else - DRY_RUN=true python3 .circleci/issue_tracker.py - fi - solidity-checking: - executor: docker-npm - steps: - - checkout - - run: - # TODO: Move to docker image - name: Install the npm packages - working_directory: ./crypto/stark-verifier-ethereum - command: npm i - - run: - name: Compile the contracts - working_directory: ./crypto/stark-verifier-ethereum - command: npm run-script build - - run: - name: Run the tests - working_directory: ./crypto/stark-verifier-ethereum - command: npm run-script test - - run: - name: Run the linter - working_directory: ./crypto/stark-verifier-ethereum - command: npm run-script lint + name: "Say hello" + command: "echo Hello, World!" +# Orchestrate jobs using workflows +# See: https://circleci.com/docs/workflows/ & https://circleci.com/docs/configuration-reference/#workflows workflows: - version: 2 - weekly: - triggers: - - schedule: - # Weekly on Sunday 0am. - cron: "0 0 * * 0" - filters: - branches: - only: - - master - jobs: - # Rebuild docker build environment to get up-to-date dependencies - # and build cache. - - rebuild-docker - commit: + say-hello-workflow: # This is the name of the workflow, feel free to change it to better match your workflow. + # Inside the workflow, you define the jobs you want to run. jobs: - - build-docker - - tests: - requires: - - build-docker - - benchmarks: - requires: - - build-docker - - doc: - requires: - - build-docker - - book: - requires: - - build-docker - - rustformat: - requires: - - build-docker - - build-no-std: - requires: - - build-docker - - clippy: - requires: - - build-docker - - analysis: - requires: - - build-docker - - benchmark-ec2: - requires: - - benchmarks - - benchmark-store: - requires: - - benchmark-ec2 - - update-issues: - filters: - branches: - only: master - - solidity-checking - # `accept` is a phony job which requires every check for a commit. - # This allows us to set a single 'required status check' in GitHub. - - accept: - requires: - - rustformat - - clippy - # TODO: Fix analysis - # - analysis - - tests - - benchmarks - - doc - - book - - build-no-std - - update-issues - - solidity-checking + - say-hello \ No newline at end of file