-
Notifications
You must be signed in to change notification settings - Fork 36
249 lines (246 loc) · 10.1 KB
/
integration-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
name: Integration Workflow
on:
pull_request:
push:
branches: [main]
tags: ["*"]
schedule:
# runs every Wednesday at 7 AM UTC
- cron: "0 7 * * 3"
workflow_dispatch:
inputs:
pytest_addopts:
description: Extra options for pytest; use -vv for full details; see
https://docs.pytest.org/en/latest/example/simple.html#how-to-change-command-line-options-defaults
required: false
default: ""
env:
LANG: "en_US.utf-8"
LC_ALL: "en_US.utf-8"
PIP_CACHE_DIR: ${{ github.workspace }}/.cache/pip
POETRY_CACHE_DIR: ${{ github.workspace }}/.cache/pypoetry
POETRY_VIRTUALENVS_IN_PROJECT: "true"
PRE_COMMIT_HOME: ${{ github.workspace }}/.cache/pre-commit
PYTEST_ADDOPTS: ${{ github.event.inputs.pytest_addopts }}
PYTHONIOENCODING: "UTF-8"
TARGET_PYTHON_VERSION: "3.9"
jobs:
quality-test:
# This job is used to run pre-commit checks to ensure that all files are
# are formatted correctly.
name: Pre-commit checks
# Runs pre-commit checks on all files
# This job doesn't fail fast to ensure that feedback on function is still provided
strategy:
fail-fast: false
runs-on: ubuntu-22.04
if: github.event_name != 'schedule'
steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
# Full history required for branch-based pre-commit checks
fetch-depth: 0
- name: Setup python, and check pre-commit cache
uses: ./.github/actions/setup-env
with:
python-version: ${{ env.TARGET_PYTHON_VERSION }}
cache-pre-commit: true
cache-venv: false
setup-poetry: false
install-deps: false
- name: Run pre-commit checks on all files
uses: pre-commit/[email protected]
with:
extra_args: --all-files
integration-test:
name: Pytest (Python ${{ matrix.python-version }} on ${{ matrix.os }})
# Runs pytest on all tested versions of python and OSes
strategy:
fail-fast: false
matrix:
os:
- macos-13
- ubuntu-22.04
python-version: ["3.8", "3.9", "3.10", "3.11"]
runs-on: ${{ matrix.os }}
env:
OS: ${{ matrix.os }}
# This is needed to avoid a warning from SQLAlchemy
# https://sqlalche.me/e/b8d9
# We can remove this once we upgrade to SQLAlchemy >= 2.0
SQLALCHEMY_SILENCE_UBER_WARNING: "1"
if: github.event_name != 'schedule'
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Setup python, and load cache
uses: ./.github/actions/setup-env
with:
python-version: ${{ matrix.python-version }}
cache-pre-commit: false
cache-venv: true
setup-poetry: true
install-deps: true
- name: Run pytest and generate coverage report
# For the target version and ubuntu, run pytest and generate coverage report
if: (matrix.os == 'ubuntu-22.04') && (matrix.python-version == env.TARGET_PYTHON_VERSION)
run: poetry run pytest --cov=./ --cov-report=xml ${{ github.event.inputs.pytest_addopts }}
- name: Upload coverage to Codecov
# For the target version and ubuntu, upload coverage to Codecov
if: (matrix.os == 'ubuntu-22.04') && (matrix.python-version == env.TARGET_PYTHON_VERSION )
uses: codecov/codecov-action@v4
env:
OS: ${{ matrix.os }}
PYTHON: ${{ matrix.python-version }}
with:
file: ./coverage.xml
files: ./coverage1.xml,./coverage2.xml
directory: ./coverage/reports/
env_vars: OS,PYTHON
fail_ci_if_error: true
flags: unittests
name: pycytominer
# Adds codecov token. See the link below for more information:
# https://docs.codecov.com/docs/adding-the-codecov-token
token: ${{ secrets.CODECOV_TOKEN }}
- name: Run pytest
# For every other version and/or OS, run pytest without coverage
if: (matrix.os != 'ubuntu-22.04') || (matrix.python-version != env.TARGET_PYTHON_VERSION )
run: poetry run pytest ${{ github.event.inputs.pytest_addopts }}
build:
name: Build versioned package
# This job is used to build the package and upload the artifacts to GitHub Actions workflow results.
# See https://github.com/actions/upload-artifact?tab=readme-ov-file#where-does-the-upload-go
runs-on: ubuntu-22.04
outputs:
version: ${{ steps.get_version.outputs.version }}
steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup python, and load cache
uses: ./.github/actions/setup-env
with:
python-version: ${{ env.TARGET_PYTHON_VERSION }}
cache-pre-commit: false
cache-venv: true
setup-poetry: true
install-deps: true
- name: Build
run: poetry build
- name: Get pycytominer version
id: get_version
run: |
echo "version=$(poetry version | cut -d' ' -f2 )" >> "$GITHUB_OUTPUT"
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: wheel
path: dist/*.whl
if-no-files-found: error
retention-days: 90
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: sdist
path: dist/*.tar.gz
if-no-files-found: error
retention-days: 90
docker-image-test-and-push:
runs-on: ubuntu-22.04
# set a dependency sequence to occur after build job completion
needs: build
env:
version: ${{ needs.build.outputs.version }}
sdist_filename: pycytominer-${{ needs.build.outputs.version}}.tar.gz
sdist_extracted_name: pycytominer-${{ needs.build.outputs.version}}
steps:
# checks out selected files for docker image build testing
- name: Checkout selected files
uses: actions/checkout@v4
with:
sparse-checkout: |
build/docker/Dockerfile
tests
sparse-checkout-cone-mode: false
- name: Download sdist artifact
uses: actions/download-artifact@v4
with:
name: sdist
# gather the sdist tar.gz name (which varies)
# unzips the sdist
- name: Extract sdist
run: |
tar -xzvf "${{ env.sdist_filename}}"
# note: roughly follows Docker documentation on GitHub Actions usage
# found on the following https://github.com/docker/build-push-action
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
# attempts to build a test image for observing test behavior
# and modeling a production image build. Version label
# assumes pwd of, for example: pycytominer-1.0.1.post37.dev0+55690e4,
# where we attempt to use 1.0.1.post37.dev0+55690e4 as a version label.
- name: Build docker image for testing
run: |
cd "${{ env.sdist_extracted_name }}" && \
cp -r ../tests . && \
docker build -f ../build/docker/Dockerfile \
-t pycytominer:testing \
--label version="${{ env.version }}" \
--target testing \
.
# runs pytest for pycytominer within a docker container based on the image above
- name: Run tests through docker image
run: |
docker run pycytominer:testing pytest
# Gather and prepare the dynamic version and other vars for use with work below.
# For example, 1.1.0.post6.dev0+bc093ef becomes 1.1.0.post6.dev0_bc093ef
- name: Gather dynamic version and set env var
if: (github.event_name == 'push'|| github.event_name == 'schedule') && github.ref == 'refs/heads/main'
id: env-vars
run: >-
echo "CLEAN_VERSION=$(echo ${{ env.sdist_extracted_name }} | tr '+' '_' )" >> "$GITHUB_OUTPUT" &&
echo "DATE_TAG=$(date +'%y%m%d')" >> "$GITHUB_OUTPUT"
- name: Login to Docker Hub
# only run this task if we have a pull request merged or the schedule triggers from main
if: (github.event_name == 'push'|| github.event_name == 'schedule') && github.ref == 'refs/heads/main'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
# Docker image push based on weekly schedule
- name: Schedule-based docker image push
# only run this task if we have a schedule trigger from main on cytomining/pycytominer
if: github.event_name == 'schedule' && github.ref == 'refs/heads/main' && github.repository == 'cytomining/pycytominer'
uses: docker/build-push-action@v6
with:
context: "${{ env.sdist_extracted_name }}"
push: true
file: ./build/docker/Dockerfile
target: production
tags: cytomining/pycytominer:${{ steps.env-vars.outputs.CLEAN_VERSION }}_${{ steps.env-vars.outputs.DATE_TAG }}
# Docker image push based on GitHub pushes to main
- name: Push-based docker image push
# only run this task if we have a pull request which is merged
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'cytomining/pycytominer'
uses: docker/build-push-action@v6
with:
context: "${{ env.sdist_extracted_name }}"
push: true
file: ./build/docker/Dockerfile
target: production
tags: cytomining/pycytominer:latest,cytomining/pycytominer:${{ steps.env-vars.outputs.CLEAN_VERSION }}
# update docker hub description using readme on merge
- name: Update Docker Hub repository description from readme
# only run this task if we have a pull request which is merged
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'cytomining/pycytominer'
uses: peter-evans/dockerhub-description@v4
with:
username: ${{ secrets.DOCKER_USER }}
password: ${{ secrets.DOCKER_TOKEN }}
repository: cytomining/pycytominer
readme-filepath: "${{ env.sdist_extracted_name }}/README.md"