Integrate DRAGEN-SV into pipeline #927
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Docker Images | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'src/**' | |
| - 'dockerfiles/**' | |
| - 'scripts/docker/build_docker.py' | |
| - '.github/workflows/sv_pipeline_docker.yml' | |
| pull_request: | |
| branches: | |
| - main | |
| paths: | |
| - 'src/**' | |
| - 'dockerfiles/**' | |
| - 'scripts/docker/build_docker.py' | |
| - '.github/workflows/sv_pipeline_docker.yml' | |
| jobs: | |
| build_args_job: | |
| runs-on: ubuntu-20.04 | |
| name: Determine Build Args | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| outputs: | |
| base_sha: ${{ steps.commit_sha.outputs.BASE_SHA }} | |
| head_sha: ${{ steps.commit_sha.outputs.HEAD_SHA }} | |
| image_tag: ${{ steps.image_tag.outputs.IMAGE_TAG }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| # By default, this checks out only the current commit; | |
| # however, since a diff between the current commit and | |
| # the base commit is required to determined which docker | |
| # images to rebuild, we use the following to check out | |
| # the complete git history. | |
| fetch-depth: 0 | |
| - name: Determine Commit SHAs | |
| id: commit_sha | |
| # This action determines the SHA of two commits: | |
| # - BASE (BASE_SHA): The commit SHA of the base branch (e.g., | |
| # broadinstitute/gatk-sv:main) which the feature branch targets. | |
| # - HEAD (HEAD_SHA): The commit SHA of the latest commit on the | |
| # feature branch. | |
| # | |
| # In the following example, BASE_SHA=B and HEAD_SHA=Z | |
| # | |
| # X---Y---Z feature | |
| # / \ | |
| # A---B---C---D---E main | |
| # | |
| # 'E' is the merge commit (e.g., 'Merge pull request #0'). | |
| # | |
| # This action can be invoked as a result of (a) pushing commits X, | |
| # Y, or Z, or (b) pushing merge commit E (i.e., merging the PR). | |
| # Depending on (a) and (b) the commit SHAs are determined differently. | |
| # In case of (a), the commit SHAs are extracted from the | |
| # 'event.pull_request' key in the github's context JSON. In case of | |
| # (b), the commit SHAs are extracted from the list of commits recorded | |
| # under the 'event' key. | |
| # | |
| # Note: Github's context JSON is printed in the action's debug page. | |
| # | |
| run: | | |
| echo "::debug::EVENT_NAME: ${{ github.event_name }}" | |
| if [[ ${{ github.event_name }} == "pull_request" ]]; then | |
| BASE_SHA=${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA=${{ github.event.pull_request.head.sha }} | |
| else | |
| BASE_SHA=${{ github.event.before }} | |
| HEAD_SHA=$(echo "$GITHUB_CONTEXT" | jq '.event.commits[].id' | tail -2 | head -1 | sed 's/\"//g') | |
| fi | |
| echo "::debug::BASE_SHA: $BASE_SHA" | |
| echo "::debug::HEAD_SHA: $HEAD_SHA" | |
| # Avail the determined commit SHAs to other steps. | |
| echo "BASE_SHA=$BASE_SHA" >> $GITHUB_OUTPUT | |
| echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_OUTPUT | |
| - name: Compose Image Tag | |
| id: image_tag | |
| # This step composes a tag to be used for all the images created by | |
| # the build_docker.py script. The tag follows the following template: | |
| # | |
| # DATE-RELEAST_TAG-HEAD_SHA_8 | |
| # | |
| # where 'DATE' is YYYY-MM-DD extracted from the time stamp of the last | |
| # commit on the feature branch (HEAD), `RELEASE_TAG` is extracted from | |
| # the latest [pre-]release on Github, and the 'HEAD_SHA_8' is the first | |
| # eight letters of the SHA of the last commit on the feature branch (HEAD). | |
| run: | | |
| # Extract the time stamp of COMMIT_SHA in YYYY-MM-DD format. | |
| # See git-show documentation available at: | |
| # http://schacon.github.io/git/git-show | |
| DATE=$(git show -s --format=%ad --date=format:'%Y-%m-%d' $COMMIT_SHA) | |
| # Get latest [pre-]release tag. | |
| RELEASE_TAG=$(jq -r '.[0] | .tag_name' <<< $(curl --silent https://api.github.com/repos/broadinstitute/gatk-sv/releases)) | |
| COMMIT_SHA=${{ steps.commit_sha.outputs.HEAD_SHA }} | |
| IMAGE_TAG=$DATE-$RELEASE_TAG-${COMMIT_SHA::8} | |
| echo "::debug::Image tag: $IMAGE_TAG" | |
| echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_OUTPUT | |
| build_job: | |
| runs-on: ubuntu-20.04 | |
| name: Test Images Build | |
| if: github.event_name == 'pull_request' | |
| needs: build_args_job | |
| strategy: | |
| matrix: | |
| python-version: ['3.8'] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| # See the comment on build_args_job. | |
| fetch-depth: 0 | |
| - name: Setup Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Run build_docker.py | |
| run: | | |
| cd ./scripts/docker/ | |
| python build_docker.py \ | |
| --base-git-commit ${{ needs.build_args_job.outputs.base_sha }} \ | |
| --current-git-commit ${{ needs.build_args_job.outputs.head_sha }} \ | |
| --image-tag ${{ needs.build_args_job.outputs.image_tag }} \ | |
| --prune-after-each-image | |
| publish_job: | |
| # This job first configures gcloud with the authentication of a | |
| # service account. It then uses gcloud to configure the docker | |
| # credentials. Finally, it rebuilds the docker images (targets are | |
| # determined by the `build_args_job`) and pushes them to GCR. | |
| name: Publish | |
| runs-on: ubuntu-20.04 | |
| environment: Deploy | |
| if: github.event_name == 'push' | |
| needs: build_args_job | |
| strategy: | |
| matrix: | |
| python-version: ['3.8'] | |
| env: | |
| DOCKERS_AZURE: "./inputs/values/dockers_azure.json" | |
| DOCKERS_GCP: "./inputs/values/dockers.json" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| # See the comment on build_args_job. | |
| fetch-depth: 0 | |
| # Authenticates git using the bot's access token. | |
| token: ${{ secrets.BOT_PAT }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Azure login | |
| uses: azure/docker-login@v1 | |
| with: | |
| login-server: ${{ secrets.AZ_CR }} | |
| username: ${{ secrets.AZ_USERNAME }} | |
| password: ${{ secrets.AZ_PASSWORD }} | |
| - name: Setup gcloud CLI | |
| uses: google-github-actions/[email protected] | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| service_account_key: ${{ secrets.GCP_GCR_SA_KEY }} | |
| # xref: https://github.com/google-github-actions/setup-gcloud#inputs | |
| # If you need to set `export_default_credentials: true` | |
| # make sure to pass the `--disable-git-protect` flag to | |
| # build_docker.py; because the setting export_default_credentials | |
| # to true will cause creating an untracked file in the root of | |
| # GATK-SV cloned directory which will cause build_docker.py to | |
| # raise an exception about the uncommitted file. | |
| - name: Configure Docker using gcloud | |
| # This step uses gcloud to configure docker credentials to access GCR | |
| # using a service account. | |
| # See: https://cloud.google.com/container-registry/docs/advanced-authentication | |
| # | |
| # The build_docker.py uses the `--squash` flag when building the | |
| # images to be pushed to GCR. This flag is only available when | |
| # experimental features are enabled, hence the features are enabled | |
| # in this flag. | |
| run: | | |
| gcloud auth configure-docker | |
| tmp=$(mktemp) | |
| sudo jq '.+{experimental:true}' /etc/docker/daemon.json > "$tmp" | |
| sudo mv "$tmp" /etc/docker/daemon.json | |
| sudo systemctl restart docker.service | |
| - name: Build and Publish Docker Images to ACR & GCR | |
| id: build_and_publish | |
| run: | | |
| python ./scripts/docker/build_docker.py \ | |
| --base-git-commit ${{ needs.build_args_job.outputs.base_sha }} \ | |
| --current-git-commit ${{ needs.build_args_job.outputs.head_sha }} \ | |
| --docker-repo ${{ secrets.AZ_CR }} us.gcr.io/${{ secrets.GCP_PROJECT_ID }}/gatk-sv \ | |
| --image-tag ${{ needs.build_args_job.outputs.image_tag }} \ | |
| --input-json $DOCKERS_AZURE $DOCKERS_GCP \ | |
| --output-json $DOCKERS_AZURE $DOCKERS_GCP \ | |
| --disable-git-protect \ | |
| --prune-after-each-image | |
| CHANGED=$(git diff --quiet $DOCKERS_GCP || echo True) | |
| echo "CHANGED=$CHANGED" >> $GITHUB_OUTPUT | |
| - name: Commit Changes to dockers_*.json | |
| if: steps.build_and_publish.outputs.CHANGED | |
| run: | | |
| COMMIT_SHA=${{ needs.build_args_job.outputs.head_sha }} | |
| git config --global user.name 'gatk-sv-bot' | |
| git config --global user.email '[email protected]' | |
| git commit $DOCKERS_AZURE $DOCKERS_GCP -m "Update docker images list, triggered by "${COMMIT_SHA::8} | |
| git pull --rebase origin main | |
| # In the following, force-push is required when the above rebase updates the branch; | |
| # otherwise, the push will be rejected with the following error: | |
| # > Updates were rejected because the tip of your current branch is behind ts remote counterpart. | |
| # See this thread for details: https://stackoverflow.com/q/39399804 | |
| git push -f |