-
Notifications
You must be signed in to change notification settings - Fork 0
/
.gitlab-ci.yml
229 lines (215 loc) · 8.6 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
# SPDX-FileCopyrightText: 2023 The WESkit Contributors
#
# SPDX-License-Identifier: MIT
# Official language image. Look for the different tagged releases at:
# https://hub.docker.com/r/library/python/tags/
services:
- docker:dind
cache:
key: cache-for-all-branches
paths:
- /opt/cache/pip
- /opt/conda/pkgs
- /root/.conda/pkgs
variables:
DOCKER_HOST: "tcp://docker:2375"
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
PIP_CACHE_DIR: "/opt/cache/pip"
CONTAINER_UID: "35671"
CONTAINER_USER: weskit
CONTAINER_GID: "35671"
CONTAINER_GROUP: weskit
REGISTRY: registry.gitlab.com
stages:
- build # Build the branch's api/base container
- test # Test the code using the branch's api/base container
- deploy # Build the deployment container, using the branch's api/base container
# See
# https://docs.gitlab.com/ee/user/project/deploy_tokens/index.html
# https://docs.gitlab.com/ee/user/packages/container_registry/#authenticate-with-the-container-registry
# https://gitlab.com/one-touch-pipeline/weskit/api/-/settings/repository/deploy_token
#
# Use this hidden job to `extend` deployment to registry and container tests jobs.
# Note that CI_DEPLOY_USER and CI_DEPLOY_PASSWORD are automatically set, because of the deploy key
# with the name "gitlab-deploy-token" (Settings -> Repository -> Deploy Tokens).
.gitlab-registry:
image: docker:stable
before_script:
- docker info
- docker login -u "$CI_DEPLOY_USER" -p "$CI_DEPLOY_PASSWORD" "$REGISTRY"
# Build-image $REGISTRY/$CONTAINER_IMAGE_NAME:$VERSION_TAG from $DOCKER_FILE.
# The corresponding variables need to be set (REGISTRY is set globally, though).
.build_image:
extends: .gitlab-registry
script:
- |
cat - <<THE_END
CI_PIPELINE_SOURCE='$CI_PIPELINE_SOURCE'
CI_COMMIT_SHORT_SHA='$CI_COMMIT_SHORT_SHA'
CI_COMMIT_TAG='$CI_COMMIT_TAG'
CI_COMMIT_BRANCH='$CI_COMMIT_BRANCH'
CI_MERGE_REQUEST_SOURCE_BRANCH_NAME='$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'
CI_MERGE_REQUEST_TARGET_BRANCH_NAME='$CI_MERGE_REQUEST_TARGET_BRANCH_NAME'
Publishing '$REGISTRY/$CONTAINER_IMAGE_NAME:$VERSION_TAG'
From '$DOCKER_FILE'
THE_END
- |
docker build \
--build-arg USER_ID="$CONTAINER_UID" \
--build-arg USER="$CONTAINER_USER" \
--build-arg GROUP_ID="$CONTAINER_GID" \
--build-arg GROUP="$CONTAINER_GROUP" \
-t "$REGISTRY/$CONTAINER_IMAGE_NAME:$VERSION_TAG" \
-f "$DOCKER_FILE" \
.
- docker push "$REGISTRY/$CONTAINER_IMAGE_NAME:$VERSION_TAG"
tag_branch_base_image:
stage: build
extends: .gitlab-registry
rules:
# The purpose of this rule is to have not only a branch-specific image if there are changes
# to the environment or dockerfile, but always. Furthermore, this image can then be used again
# as target-branch image of a branch deriving from this branch. As these are only tags,
# and tags are automatically collected after some time, this will neither add much overhead to
# the CI nor need extra storage.
#
# For now this is implemented with the "master" branch as only target branch. A soon as
# https://gitlab.com/gitlab-org/gitlab/-/issues/369916 is implemented, change we may
# `compare_to: "refs/heads/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"` and implement this and
# the build rule by comparing to the target branch.
#
# If there are no changes of the environment or docker files, just reuse the existing image
# of the target branch and tag it with the source branch name.
- if: '$CI_COMMIT_BRANCH && ($CI_COMMIT_BRANCH != "master")'
changes:
compare_to: "refs/heads/master"
paths:
- environment.yaml
- Dockerfile-Base
when:
never
- if: '$CI_COMMIT_BRANCH && ($CI_COMMIT_BRANCH != "master")'
variables:
REFERENCE_TAG: master
VERSION_TAG: "$CI_COMMIT_BRANCH"
variables:
DOCKER_FILE: Dockerfile-Base
CONTAINER_IMAGE_NAME: one-touch-pipeline/weskit/api/base
script:
- |
cat - <<THE_END
CI_PIPELINE_SOURCE='$CI_PIPELINE_SOURCE'
CI_COMMIT_SHORT_SHA='$CI_COMMIT_SHORT_SHA'
CI_COMMIT_TAG='$CI_COMMIT_TAG'
CI_COMMIT_BRANCH='$CI_COMMIT_BRANCH'
CI_MERGE_REQUEST_SOURCE_BRANCH_NAME='$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'
CI_MERGE_REQUEST_TARGET_BRANCH_NAME='$CI_MERGE_REQUEST_TARGET_BRANCH_NAME'
VERSION_TAG='$VERSION_TAG'
REFERENCE_TAG='$REFERENCE_TAG'
Re-tagging '$REGISTRY/$CONTAINER_IMAGE_NAME:$REFERENCE_TAG' as '$REGISTRY/$CONTAINER_IMAGE_NAME:$VERSION_TAG'
THE_END
- docker pull "$REGISTRY/$CONTAINER_IMAGE_NAME:$REFERENCE_TAG"
- docker tag "$REGISTRY/$CONTAINER_IMAGE_NAME:$REFERENCE_TAG" "$REGISTRY/$CONTAINER_IMAGE_NAME:$VERSION_TAG"
- docker push "$REGISTRY/$CONTAINER_IMAGE_NAME:$VERSION_TAG"
build_branch_base_image:
stage: build
extends: .build_image
rules:
# Build the image for merge-requests.
- if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'
changes:
compare_to: "refs/heads/master^1"
paths:
- environment.yaml
- Dockerfile-Base
variables:
VERSION_TAG: "$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME"
# Build a branch-specific image for all branches with changes relative to the master branch.
- if: '$CI_COMMIT_BRANCH != "master"'
changes:
compare_to: "refs/heads/master"
paths:
- environment.yaml
- Dockerfile-Base
variables:
VERSION_TAG: "$CI_COMMIT_BRANCH"
# Also build the image on the master branch, if there were changes.
- if: '$CI_COMMIT_BRANCH == "master"'
changes:
compare_to: "refs/heads/master^1"
paths:
- environment.yaml
- Dockerfile-Base
variables:
VERSION_TAG: "$CI_COMMIT_BRANCH"
variables:
DOCKER_FILE: Dockerfile-Base
CONTAINER_IMAGE_NAME: one-touch-pipeline/weskit/api/base
test:
image: "$REGISTRY/one-touch-pipeline/weskit/api/base:$IMAGE_VERSION"
stage: test
needs:
- job: build_branch_base_image
optional: true
- job: tag_branch_base_image
optional: true
rules:
# Do not run on commits of a branch, if there is an open merge request for the branch.
# In that case rather the first rule will match, which runs on the merge-commit.
# This is to prevent double runs on both, branch-commits and merge-commits.
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
when: never
# https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
# If this is a merge-request pipeline, run the test. Always use the source branch image tag.
# That source image tag always exists, independent of whether there are changes in the
# environment or docker file or not. See "build_branch_base_image" and "tag_branch_base_image".
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
variables:
IMAGE_VERSION: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME'
# If this is a commit with a tag, run the test. Relates to job "build_tagged_image"
# Use the tag as image name
- if: '$CI_COMMIT_TAG'
variables:
IMAGE_VERSION: '$CI_COMMIT_TAG'
# Otherwise, on commits on branches without a merge request, run the tests.
# Again just use the branch-tagged image, probably created by "tag_branch_base_image".
- if: '$CI_COMMIT_BRANCH'
variables:
IMAGE_VERSION: '$CI_COMMIT_BRANCH'
script:
- reuse lint
- mypy weskit/ tests/
- flake8
- bandit -r weskit/ uwsgi_server/
- cp tests/remote-template.yaml tests/remote.yaml
- python -m pytest -vvv --cov=weskit/ --cov-report term-missing -m "not slow and not ssh"
build_latest_image:
stage: deploy
extends: .build_image
needs:
# Only build master images, if the QA jobs succeeded.
- job: test
optional: true # Gitlab-CI seems not smart enough to recognize that this job's rules
# match only on a subset of the rules on "test" job.
# Build the master image only after the master base-image was build, if one had to be built.
- job: build_master_base_image
optional: true
rules:
- if: $CI_COMMIT_BRANCH == "master"
changes:
compare_to: "refs/heads/master^1"
paths:
# The image depends on the base-image ...
- environment.yaml
- Dockerfile-Base
- Dockerfile
# ... and includes the code for running WESkit.
- uwsgi_server/**/*
- weskit/**/*
- tools/**/*
- config/**/*
variables:
DOCKER_FILE: Dockerfile
CONTAINER_IMAGE_NAME: one-touch-pipeline/weskit/api
VERSION_TAG: latest