From 308af35eb3948528702259c39a2537ba1e515ea1 Mon Sep 17 00:00:00 2001 From: Pablo Suarez Date: Thu, 15 Jun 2023 17:21:36 +0400 Subject: [PATCH] feat: DEVOPS-700 scilla pipelines from jenkins to gha feat: DEVOPS-1247 move Scilla pipeline to GCP (#1247) --- .github/workflows/ci-image-dev.yml | 47 ++++++++++++++++++ .github/workflows/ci-image-release.yml | 68 ++++++++++++++++++++++++++ .github/workflows/ci-image-test.yml | 63 ++++++++++++++++++++++++ Jenkinsfile => Jenkinsfile.disabled | 0 Makefile | 2 +- docker/Dockerfile.test | 17 +++++++ src/base/cpp/config/discover.ml | 4 +- 7 files changed, 199 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci-image-dev.yml create mode 100644 .github/workflows/ci-image-release.yml create mode 100644 .github/workflows/ci-image-test.yml rename Jenkinsfile => Jenkinsfile.disabled (100%) create mode 100644 docker/Dockerfile.test diff --git a/.github/workflows/ci-image-dev.yml b/.github/workflows/ci-image-dev.yml new file mode 100644 index 000000000..e812ee7ea --- /dev/null +++ b/.github/workflows/ci-image-dev.yml @@ -0,0 +1,47 @@ +name: CI - Development + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + push: + branches-ignore: + - 'release/**' + +jobs: + push-to-ecr: + permissions: + id-token: write + contents: write + name: build + runs-on: docker + env: + GCP_REGISTRY_DOMAIN: asia-docker.pkg.dev + GCP_REGISTRY: asia-docker.pkg.dev/${{ secrets.GCP_STG_REGISTRY_PROJECT_ID }}/zilliqa-private + steps: + - name: Clean environment + # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). + shell: bash + run: | + docker system df + docker system prune -a -f --filter "until=168h" + docker system df + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Docker build and push + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile + push: true + tag: ${{ env.GCP_REGISTRY }}/scilla + tag-length: 8 + registry: ${{ env.GCP_REGISTRY_DOMAIN }} + workload-identity-provider: "${{ secrets.GCP_PRD_GITHUB_WIF }}" + service-account: "${{ secrets.GCP_STG_GITHUB_SA_DOCKER_REGISTRY }}" + cache-key: ${{ github.event.repository.name }} diff --git a/.github/workflows/ci-image-release.yml b/.github/workflows/ci-image-release.yml new file mode 100644 index 000000000..8ae6c9c7f --- /dev/null +++ b/.github/workflows/ci-image-release.yml @@ -0,0 +1,68 @@ +name: CI - Release + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + push: + tags: + - v* + +jobs: + release-image: + permissions: + id-token: write + contents: write + runs-on: docker + env: + GCP_REGISTRY_DOMAIN: asia-docker.pkg.dev + GCP_REGISTRY: asia-docker.pkg.dev/${{ secrets.GCP_PRD_REGISTRY_PROJECT_ID }}/zilliqa-private + steps: + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Check if the commit or tag was passed manually + id: set-tag + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "tag=$(git rev-parse HEAD | cut -c1-8)" >> $GITHUB_OUTPUT + else + echo "tag=${{ github.ref_name }}" >> $GITHUB_OUTPUT + fi + shell: bash + - name: Check if the tag head is the same as master head + id: check-latest + if: github.event_name != 'workflow_dispatch' + run: | + if test $(git rev-parse origin/master) = $(git rev-parse HEAD); then + echo "latest=true" >> $GITHUB_OUTPUT + else + echo "latest=false" >> $GITHUB_OUTPUT + fi + shell: bash + - name: Docker build and push (GCP) + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile + push: true + tag: ${{ env.GCP_REGISTRY }}/scilla:${{ steps.set-tag.outputs.tag }} + tag-latest: ${{ steps.check-latest.outputs.latest }} + registry: ${{ env.GCP_REGISTRY_DOMAIN }} + workload-identity-provider: "${{ secrets.GCP_PRD_GITHUB_WIF }}" + service-account: "${{ secrets.GCP_PRD_GITHUB_SA_DOCKER_REGISTRY }}" + cache-key: ${{ github.event.repository.name }} + - name: Docker build and push (Dockerhub) + uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1 + with: + file: docker/Dockerfile + push: true + tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }} + tag-latest: ${{ steps.check-latest.outputs.latest }} + cache-key: ${{ github.event.repository.name }} + registry-username: ${{ secrets.DOCKERHUB_USERNAME }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/ci-image-test.yml b/.github/workflows/ci-image-test.yml new file mode 100644 index 000000000..b7ad91119 --- /dev/null +++ b/.github/workflows/ci-image-test.yml @@ -0,0 +1,63 @@ +name: CI - Integration tests + +on: + workflow_dispatch: + inputs: + commitOrTag: + description: 'Commit or tag' + required: false + default: '' + pull_request: + branches: + - 'master' + +jobs: + run-tests: + permissions: + id-token: write + contents: write + name: tests + runs-on: docker + steps: + - name: Clean environment + # Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup). + shell: bash + run: | + docker system df + docker system prune -a -f --filter "until=336h" + docker system df + - name: 'Checkout scm ${{ inputs.commitOrTag }}' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ inputs.commitOrTag }} + - name: Configure AWS Credentials + uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }} + oidc-role: ${{ secrets.OIDC_ROLE }} + aws-region: ${{ secrets.AWS_REGION_ZILLIQA }} + - name: Login to the registry + uses: docker/login-action@v2 + with: + registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com + - name: Build Docker images + run: | + DOCKER_BUILDKIT=1 docker build -t scilla:tests --build-arg ACCOUNT_ID=${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }} -f docker/Dockerfile.test . + shell: bash + - name: Run make test + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test' + shell: bash + - name: Run make test_server + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server' + shell: bash + - name: Run make coveralls + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}' + shell: bash + - name: Run make lint + run: | + docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint' + shell: bash diff --git a/Jenkinsfile b/Jenkinsfile.disabled similarity index 100% rename from Jenkinsfile rename to Jenkinsfile.disabled diff --git a/Makefile b/Makefile index ce5496ef2..fb867a46a 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ clean: dune clean # Remove remaining files/folders ignored by git as defined in .gitignore (-X) # but keeping a local opam switch and other dependencies built. - git clean -dfXq --exclude=\!deps/** --exclude=\!_opam/** --exclude=\!_esy/** --exclude=\!vcpkg_installed + git clean -dfXq --exclude=\!deps/** --exclude=\!_opam/** --exclude=\!_esy/** --exclude=\!vcpkg_installed --exclude=\!vcpkg_installed/** # Clean up libff installation cleanall: clean diff --git a/docker/Dockerfile.test b/docker/Dockerfile.test new file mode 100644 index 000000000..f1e533cb7 --- /dev/null +++ b/docker/Dockerfile.test @@ -0,0 +1,17 @@ +ARG ACCOUNT_ID + +FROM ${ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9 + +ENV VCPKG_ROOT="/vcpkg" +ENV SCILLA_REPO_ROOT="/scilla/0" + +WORKDIR /scilla/0/ +COPY . /scilla/0/ + +RUN apt update \ + && apt install -y sudo + +RUN eval $(opam env) \ + && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes + +RUN ./scripts/install_shellcheck_ubuntu.sh diff --git a/src/base/cpp/config/discover.ml b/src/base/cpp/config/discover.ml index 5bae54e6d..c50227f51 100644 --- a/src/base/cpp/config/discover.ml +++ b/src/base/cpp/config/discover.ml @@ -30,7 +30,9 @@ let () = We rely on a simple python program (in vcpkg_cmd) to do so portably. *) let vcpkg_cmd = - "python3 -c 'import os, sys; print(os.path.realpath(os.environ[\"SCILLA_REPO_ROOT\"]" ^^ "+\"/%s\"))'" + "python3 -c 'import os, sys; \ + print(os.path.realpath(os.environ[\"SCILLA_REPO_ROOT\"]" + ^^ "+\"/%s\"))'" in let vcpkg_triplet_path = input_line