forked from broadinstitute/gatk-sv
-
Notifications
You must be signed in to change notification settings - Fork 0
236 lines (215 loc) · 9.25 KB
/
sv_pipeline_docker.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
name: Docker Images
on:
push:
branches:
- main
- workflow_bug2
paths:
- 'src/**'
- 'dockerfiles/**'
- 'scripts/docker/build_docker.py'
- '.github/workflows/sv_pipeline_docker.yml'
pull_request:
branches:
- main
- workflow_bug2
paths:
- 'src/**'
- 'dockerfiles/**'
- 'scripts/docker/build_docker.py'
- '.github/workflows/sv_pipeline_docker.yml'
jobs:
build_args_job:
runs-on: ubuntu-20.04
name: Determine Build Args
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
base_sha: ${{ steps.commit_sha.outputs.BASE_SHA }}
head_sha: ${{ steps.commit_sha.outputs.HEAD_SHA }}
image_tag: ${{ steps.image_tag.outputs.IMAGE_TAG }}
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
# By default, this checks out only the current commit;
# however, since a diff between the current commit and
# the base commit is required to determined which docker
# images to rebuild, we use the following to check out
# the complete git history.
fetch-depth: 0
- name: Determine Commit SHAs
id: commit_sha
# This action determines the SHA of two commits:
# - BASE (BASE_SHA): The commit SHA of the base branch (e.g.,
# broadinstitute/gatk-sv:main) which the feature branch targets.
# - HEAD (HEAD_SHA): The commit SHA of the latest commit on the
# feature branch.
#
# In the following example, BASE_SHA=B and HEAD_SHA=Z
#
# X---Y---Z feature
# / \
# A---B---C---D---E main
#
# 'E' is the merge commit (e.g., 'Merge pull request #0').
#
# This action can be invoked as a result of (a) pushing commits X,
# Y, or Z, or (b) pushing merge commit E (i.e., merging the PR).
# Depending on (a) and (b) the commit SHAs are determined differently.
# In case of (a), the commit SHAs are extracted from the
# 'event.pull_request' key in the github's context JSON. In case of
# (b), the commit SHAs are extracted from the list of commits recorded
# under the 'event' key.
#
# Note: Github's context JSON is printed in the action's debug page.
#
run: |
echo "::debug::EVENT_NAME: ${{ github.event_name }}"
if [[ ${{ github.event_name }} == "pull_request" ]]; then
BASE_SHA=${{ github.event.pull_request.base.sha }}
HEAD_SHA=${{ github.event.pull_request.head.sha }}
else
BASE_SHA=${{ github.event.before }}
HEAD_SHA=$(echo "$GITHUB_CONTEXT" | jq '.event.commits[].id' | tail -2 | head -1 | sed 's/\"//g')
fi
echo "::debug::BASE_SHA: $BASE_SHA"
echo "::debug::HEAD_SHA: $HEAD_SHA"
# Avail the determined commit SHAs to other steps.
echo "BASE_SHA=$BASE_SHA" >> $GITHUB_OUTPUT
echo "HEAD_SHA=$HEAD_SHA" >> $GITHUB_OUTPUT
- name: Compose Image Tag
id: image_tag
# This step composes a tag to be used for all the images created by
# the build_docker.py script. The tag follows the following template:
#
# DATE-RELEAST_TAG-HEAD_SHA_8
#
# where 'DATE' is YYYY-MM-DD extracted from the time stamp of the last
# commit on the feature branch (HEAD), `RELEASE_TAG` is extracted from
# the latest [pre-]release on Github, and the 'HEAD_SHA_8' is the first
# eight letters of the SHA of the last commit on the feature branch (HEAD).
run: |
# Extract the time stamp of COMMIT_SHA in YYYY-MM-DD format.
# See git-show documentation available at:
# http://schacon.github.io/git/git-show
DATE=$(git show -s --format=%ad --date=format:'%Y-%m-%d' $COMMIT_SHA)
# Get latest [pre-]release tag.
RELEASE_TAG=$(jq -r '.[0] | .tag_name' <<< $(curl --silent https://api.github.com/repos/broadinstitute/gatk-sv/releases))
COMMIT_SHA=${{ steps.commit_sha.outputs.HEAD_SHA }}
IMAGE_TAG=$DATE-$RELEASE_TAG-${COMMIT_SHA::8}
echo "::debug::Image tag: $IMAGE_TAG"
echo "IMAGE_TAG=$IMAGE_TAG" >> $GITHUB_OUTPUT
build_job:
runs-on: ubuntu-20.04
name: Test Images Build
if: github.event_name == 'pull_request'
needs: build_args_job
strategy:
matrix:
python-version: ['3.8']
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
# See the comment on build_args_job.
fetch-depth: 0
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Run build_docker.py
run: |
cd ./scripts/docker/
python build_docker.py \
--base-git-commit ${{ needs.build_args_job.outputs.base_sha }} \
--current-git-commit ${{ needs.build_args_job.outputs.head_sha }} \
--image-tag ${{ needs.build_args_job.outputs.image_tag }} \
--prune-after-each-image
publish_job:
# This job first configures gcloud with the authentication of a
# service account. It then uses gcloud to configure the docker
# credentials. Finally, it rebuilds the docker images (targets are
# determined by the `build_args_job`) and pushes them to GCR.
name: Publish
runs-on: ubuntu-20.04
environment: Deploy
if: github.event_name == 'push'
needs: build_args_job
strategy:
matrix:
python-version: ['3.8']
env:
DOCKERS_AZURE: "./inputs/values/dockers_azure.json"
DOCKERS_GCP: "./inputs/values/dockers.json"
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
# See the comment on build_args_job.
fetch-depth: 0
# Authenticates git using the bot's access token.
token: ${{ secrets.BOT_PAT }}
- name: Setup Python
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: Azure login
uses: azure/docker-login@v1
with:
login-server: ${{ secrets.AZ_CR }}
username: ${{ secrets.AZ_USERNAME }}
password: ${{ secrets.AZ_PASSWORD }}
- name: Setup gcloud CLI
uses: google-github-actions/[email protected]
with:
project_id: ${{ secrets.GCP_PROJECT_ID }}
service_account_key: ${{ secrets.GCP_GCR_SA_KEY }}
# xref: https://github.com/google-github-actions/setup-gcloud#inputs
# If you need to set `export_default_credentials: true`
# make sure to pass the `--disable-git-protect` flag to
# build_docker.py; because the setting export_default_credentials
# to true will cause creating an untracked file in the root of
# GATK-SV cloned directory which will cause build_docker.py to
# raise an exception about the uncommitted file.
- name: Configure Docker using gcloud
# This step uses gcloud to configure docker credentials to access GCR
# using a service account.
# See: https://cloud.google.com/container-registry/docs/advanced-authentication
#
# The build_docker.py uses the `--squash` flag when building the
# images to be pushed to GCR. This flag is only available when
# experimental features are enabled, hence the features are enabled
# in this flag.
run: |
gcloud auth configure-docker
tmp=$(mktemp)
sudo jq '.+{experimental:true}' /etc/docker/daemon.json > "$tmp"
sudo mv "$tmp" /etc/docker/daemon.json
sudo systemctl restart docker.service
- name: Build and Publish Docker Images to ACR & GCR
id: build_and_publish
run: |
python ./scripts/docker/build_docker.py \
--targets manta \
--docker-repo us.gcr.io/${{ secrets.GCP_PROJECT_ID }}/gatk-sv ${{ secrets.AZ_CR }} \
--image-tag ${{ needs.build_args_job.outputs.image_tag }} \
--input-json $DOCKERS_AZURE $DOCKERS_GCP \
--output-json $DOCKERS_AZURE $DOCKERS_GCP \
--disable-git-protect \
--prune-after-each-image
CHANGED=$(git diff --quiet $DOCKERS_GCP || echo True)
echo "CHANGED=$CHANGED" >> $GITHUB_OUTPUT
- name: Commit Changes to dockers_*.json
if: steps.build_and_publish.outputs.CHANGED
run: |
COMMIT_SHA=${{ needs.build_args_job.outputs.head_sha }}
git config --global user.name 'gatk-sv-bot'
git config --global user.email '[email protected]'
git commit $DOCKERS_AZURE $DOCKERS_GCP -m "Update docker images list, triggered by "${COMMIT_SHA::8}
git pull --rebase origin main
# In the following, force-push is required when the above rebase updates the branch;
# otherwise, the push will be rejected with the following error:
# > Updates were rejected because the tip of your current branch is behind ts remote counterpart.
# See this thread for details: https://stackoverflow.com/q/39399804
git push -f