Updating pywhisker install #454
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly build | |
| on: | |
| push: | |
| branches: | |
| - dev | |
| paths-ignore: | |
| # - ".github/**" | |
| - "**.md" | |
| env: | |
| # fake local registry for base image | |
| FAKE_LOCAL_REGISTRY: "nwodtuhs/exegol-local" | |
| # intermediate registry in which architecture-specific images and base images must be pushed | |
| INTERMEDIATE_IMAGES_BUILDS_REGISTRY: "nwodtuhs/exegol-builds" | |
| # base image is used as initial layer when building the final image | |
| BASE_IMAGE_REGISTRY: "nwodtuhs/exegol-misc-preprod" | |
| BASE_IMAGE_TAG: "base" | |
| BASE_IMAGE_DOCKERFILE: "./sources/dockerfiles/base.dockerfile" | |
| # final image parameters | |
| IMAGE_TARGET_REGISTRY: "nwodtuhs/exegol" | |
| IMAGE_TAG: "nightly" | |
| DOCKERFILE: "./sources/dockerfiles/Dockerfile" | |
| # ThePorgs/Exegol-docs branch for tools lists | |
| DOCS_TARGET_BRANCH: "main" | |
| # creating a concurrency group for nightly builds | |
| # so that when pushing multiple things in dev, build is always running for the latest push | |
| # this is to save resources, by killing "in progress" jobs when another build starts for the last push | |
| concurrency: | |
| group: nightly_builds | |
| cancel-in-progress: true | |
| jobs: | |
| # https://github.com/orgs/community/discussions/26671, "can’t pass ENV variables to the reusable workflow" | |
| init: | |
| name: Initialize | |
| runs-on: self-hosted | |
| outputs: | |
| FAKE_LOCAL_REGISTRY: ${{ steps.varset.outputs.FAKE_LOCAL_REGISTRY }} | |
| INTERMEDIATE_IMAGES_BUILDS_REGISTRY: ${{ steps.varset.outputs.INTERMEDIATE_IMAGES_BUILDS_REGISTRY }} | |
| IMAGE_TARGET_REGISTRY: ${{ steps.varset.outputs.IMAGE_TARGET_REGISTRY }} | |
| IMAGE_TAG: ${{ steps.varset.outputs.IMAGE_TAG }} | |
| IMAGE_VERSION: ${{ steps.varset.outputs.IMAGE_VERSION }} | |
| DOCKERFILE: ${{ steps.varset.outputs.DOCKERFILE }} | |
| BASE_IMAGE_REGISTRY: ${{ steps.varset.outputs.BASE_IMAGE_REGISTRY }} | |
| BASE_IMAGE_TAG: ${{ steps.varset.outputs.BASE_IMAGE_TAG }} | |
| BASE_IMAGE_DOCKERFILE: ${{ steps.varset.outputs.BASE_IMAGE_DOCKERFILE }} | |
| DOCS_TARGET_BRANCH: ${{ steps.varset.outputs.DOCS_TARGET_BRANCH }} | |
| image_exists: ${{ steps.check_remote_image.outputs.image_exists }} | |
| steps: | |
| - name: Setting variables | |
| id: varset | |
| run: | | |
| COMMIT_ID=${{ github.sha }} | |
| echo "FAKE_LOCAL_REGISTRY=${{ env.FAKE_LOCAL_REGISTRY }}" >> $GITHUB_OUTPUT | |
| echo "INTERMEDIATE_IMAGES_BUILDS_REGISTRY=${{ env.INTERMEDIATE_IMAGES_BUILDS_REGISTRY }}" >> $GITHUB_OUTPUT | |
| echo "IMAGE_TARGET_REGISTRY=${{ env.IMAGE_TARGET_REGISTRY }}" >> $GITHUB_OUTPUT | |
| echo "IMAGE_TAG=${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT | |
| echo "IMAGE_VERSION=${COMMIT_ID:0:8}" >> $GITHUB_OUTPUT | |
| echo "DOCKERFILE=${{ env.DOCKERFILE }}" >> $GITHUB_OUTPUT | |
| echo "BASE_IMAGE_REGISTRY=${{ env.BASE_IMAGE_REGISTRY }}" >> $GITHUB_OUTPUT | |
| echo "BASE_IMAGE_TAG=${{ env.BASE_IMAGE_TAG }}" >> $GITHUB_OUTPUT | |
| echo "BASE_IMAGE_DOCKERFILE=${{ env.BASE_IMAGE_DOCKERFILE }}" >> $GITHUB_OUTPUT | |
| echo "DOCS_TARGET_BRANCH=${{ env.DOCS_TARGET_BRANCH }}" >> $GITHUB_OUTPUT | |
| cat $GITHUB_OUTPUT | |
| - name: Checking if remote image exists | |
| id: check_remote_image | |
| run: | | |
| echo "docker manifest inspect ${{ env.IMAGE_TARGET_REGISTRY }}:${{ env.IMAGE_TAG }}-${{ steps.varset.outputs.IMAGE_VERSION }}" | |
| if docker manifest inspect ${{ env.IMAGE_TARGET_REGISTRY }}:${{ env.IMAGE_TAG }}-${{ steps.varset.outputs.IMAGE_VERSION }}; then | |
| echo "Image exists" | |
| echo "image_exists=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "Image does not exist" | |
| echo "image_exists=false" >> $GITHUB_OUTPUT | |
| fi | |
| cat $GITHUB_OUTPUT | |
| code_check: | |
| name: Code compliance check | |
| uses: ./.github/workflows/sub_code_check.yml | |
| build_base: | |
| name: Base image build | |
| needs: [ init, code_check ] | |
| if: needs.code_check.result == 'success' && needs.init.outputs.image_exists == 'false' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: [ arm64, amd64 ] | |
| uses: ./.github/workflows/sub_build_belt.yml | |
| with: | |
| # ex: nwodtuhs/exegol-local | |
| IMAGE_REGISTRY: ${{ needs.init.outputs.FAKE_LOCAL_REGISTRY }} | |
| # ex: base | |
| IMAGE_TAG: ${{ needs.init.outputs.BASE_IMAGE_TAG }} | |
| # ex: base-f31c8d23-arm64 | |
| IMAGE_NAME: ${{ needs.init.outputs.BASE_IMAGE_TAG }}-${{ needs.init.outputs.IMAGE_VERSION }}-${{ matrix.arch }} | |
| # ex: f31c8d23 | |
| IMAGE_VERSION: ${{ needs.init.outputs.IMAGE_VERSION }} | |
| # ex: base.dockerfile | |
| DOCKERFILE: ${{ needs.init.outputs.BASE_IMAGE_DOCKERFILE }} | |
| # ex: arm64 | |
| ARCH: ${{ matrix.arch }} | |
| EXPORT_TOOLS: false | |
| # not pushing base image, we only need to push the final nightly image | |
| PUSH_IMAGE: false | |
| build: | |
| name: Final image build | |
| needs: [ init, code_check, build_base ] | |
| # Can't have always() in the if statement | |
| # as per https://github.com/orgs/community/discussions/26303 | |
| # and https://github.com/actions/runner/issues/1846 | |
| # running build if base was built successfully | |
| if: needs.build_base.result == 'success' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: [ arm64, amd64 ] | |
| uses: ./.github/workflows/sub_build_belt.yml | |
| with: | |
| # ex: nwodtuhs/exegol-builds | |
| IMAGE_REGISTRY: ${{ needs.init.outputs.INTERMEDIATE_IMAGES_BUILDS_REGISTRY }} | |
| # ex: nightly | |
| IMAGE_TAG: ${{ needs.init.outputs.IMAGE_TAG }} | |
| # ex: nightly-f31c8d23-arm64 | |
| IMAGE_NAME: ${{ needs.init.outputs.IMAGE_TAG }}-${{ needs.init.outputs.IMAGE_VERSION }}-${{ matrix.arch }} | |
| # ex: f31c8d23 | |
| IMAGE_VERSION: ${{ needs.init.outputs.IMAGE_VERSION }} | |
| # ex: Dockerfile | |
| DOCKERFILE: ${{ needs.init.outputs.DOCKERFILE }} | |
| # ex: arm64 | |
| ARCH: ${{ matrix.arch }} | |
| # ex: nwodtuhs/exegol-local | |
| BASE_IMAGE_REGISTRY: ${{ needs.init.outputs.FAKE_LOCAL_REGISTRY }} | |
| # ex: base-f31c8d23-arm64 | |
| BASE_IMAGE_NAME: ${{ needs.init.outputs.BASE_IMAGE_TAG }}-${{ needs.init.outputs.IMAGE_VERSION }}-${{ matrix.arch }} | |
| EXPORT_TOOLS: true | |
| DOCS_TARGET_BRANCH: ${{ needs.init.outputs.DOCS_TARGET_BRANCH }} | |
| PUSH_IMAGE: true | |
| secrets: | |
| DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} | |
| DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} | |
| EXEGOL_DOCS_SSH_DEPLOY_KEY: ${{ secrets.EXEGOL_DOCS_SSH_DEPLOY_KEY }} | |
| publish: | |
| name: Publish image | |
| needs: [ init, build ] | |
| # only publishing if the tests were a success (implicit by the success of build). | |
| if: needs.build.result == 'success' | |
| timeout-minutes: 60 | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: [ "${{ needs.init.outputs.IMAGE_TAG }}" ] | |
| steps: | |
| - name: Login to Dockerhub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Create manifest | |
| id: create_manifest | |
| if: success() | |
| # ex: docker manifest create nwodtuhs/exegol:nightly nwodtuhs/exegol-builds:nightly-f31c8d23-arm64 nwodtuhs/exegol-builds:nightly-f31c8d23-amd64 | |
| run: docker manifest create ${{ env.IMAGE_TARGET_REGISTRY }}:${{ matrix.image }} ${{ env.INTERMEDIATE_IMAGES_BUILDS_REGISTRY }}:${{ env.IMAGE_TAG }}-${{ needs.init.outputs.IMAGE_VERSION }}-arm64 ${{ env.INTERMEDIATE_IMAGES_BUILDS_REGISTRY }}:${{ env.IMAGE_TAG }}-${{ needs.init.outputs.IMAGE_VERSION }}-amd64 | |
| - name: Push manifest to remote | |
| if: success() | |
| # ex: docker manifest push nwodtuhs/exegol:nightly | |
| run: docker manifest push ${{ env.IMAGE_TARGET_REGISTRY }}:${{ matrix.image }} | |
| - name: Remove manifest locally | |
| if: always() && steps.create_manifest.outcome == 'success' | |
| # ex: docker manifest rm nwodtuhs/exegol:nightly | |
| run: docker manifest rm ${{ env.IMAGE_TARGET_REGISTRY }}:${{ matrix.image }} | |
| clean_runners: | |
| name: Clean runner | |
| needs: [ init, publish ] | |
| if: always() | |
| # even if this job fails, it won't affect the success/fail status of the whole workflow | |
| continue-on-error: true | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: [ arm64, amd64 ] | |
| runs-on: | |
| - self-hosted | |
| - builder | |
| - ${{ matrix.arch }} | |
| steps: | |
| - name: List docker images | |
| run: docker image ls | |
| - name: Remove local base image | |
| # always removing image, no need to keep it on the runner | |
| if: always() | |
| # ex: docker rmi nwodtuhs/exegol-local:base-f31c8d23-arm64 | |
| run: docker rmi ${{ env.FAKE_LOCAL_REGISTRY }}:${{ env.BASE_IMAGE_TAG }}-${{ needs.init.outputs.IMAGE_VERSION }}-${{ matrix.arch }} | |
| - name: Remove local final image | |
| # always removing image, no need to keep it on the runner | |
| if: always() | |
| # ex: docker rmi nwodtuhs/exegol-builds:nightly-f31c8d23-arm64 | |
| run: docker rmi ${{ env.INTERMEDIATE_IMAGES_BUILDS_REGISTRY }}:${{ env.IMAGE_TAG }}-${{ needs.init.outputs.IMAGE_VERSION }}-${{ matrix.arch }} | |
| clean_registries: | |
| name: Clean registries | |
| needs: [ init, publish ] | |
| # only cleaning if publish was a success. And publish requires that tests were a success. | |
| # If tests were a success, there's no need for debugging the images, they can be removed from the exegol-builds registry | |
| if: always() && needs.publish.result == 'success' | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: [ arm64, amd64 ] | |
| steps: | |
| - name: Remove remote arch-specific images in ${{ env.INTERMEDIATE_IMAGES_BUILDS_REGISTRY }} | |
| # ex: curl [...] --request DELETE [...]exegol-builds/tags/nightly-f31c8d23-arm64 | |
| run: | | |
| HUB_TOKEN=$(curl --silent --header "Content-Type: application/json" --request POST --data "{\"username\": \"${{ secrets.DOCKER_USERNAME }}\", \"password\": \"${{ secrets.DOCKER_PASSWORD }}\"}" https://hub.docker.com/v2/users/login/ | jq -r .token) | |
| curl --fail-with-body --include --request DELETE -H "Accept: application/json" --header "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/repositories/${{ env.INTERMEDIATE_IMAGES_BUILDS_REGISTRY }}/tags/${{ env.IMAGE_TAG }}-${{ needs.init.outputs.IMAGE_VERSION }}-${{ matrix.arch }}/ |