-
Notifications
You must be signed in to change notification settings - Fork 35
241 lines (206 loc) · 8.51 KB
/
pip-compile.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
# This Software (Dioptra) is being made available as a public service by the
# National Institute of Standards and Technology (NIST), an Agency of the United
# States Department of Commerce. This software was developed in part by employees of
# NIST and in part by NIST contractors. Copyright in portions of this software that
# were developed by NIST contractors has been licensed or assigned to NIST. Pursuant
# to Title 17 United States Code Section 105, works of NIST employees are not
# subject to copyright protection in the United States. However, NIST may hold
# international copyright in software created by its employees and domestic
# copyright (or licensing rights) in portions of software that were assigned or
# licensed to NIST. To the extent that NIST holds copyright in this software, it is
# being made available under the Creative Commons Attribution 4.0 International
# license (CC BY 4.0). The disclaimers of the CC BY 4.0 license apply to all parts
# of the software developed or licensed by NIST.
#
# ACCESS THE FULL CC BY 4.0 LICENSE HERE:
# https://creativecommons.org/licenses/by/4.0/legalcode
name: pip-compile runs
on:
schedule:
- cron: "10 1 * * *" # at 1:10am every day
jobs:
pip-compile:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: ["ubuntu-22.04", "macos-13", "macos-latest", "windows-latest"]
python-version: ["3.11"]
architecture: ["x86_64", "aarch64"]
requirements:
- "requirements-dev"
- "requirements-dev-pytorch"
- "requirements-dev-tensorflow"
exclude:
- os: "macos-latest"
architecture: "x86_64"
- os: "ubuntu-22.04"
architecture: "aarch64"
- os: "macos-13"
architecture: "aarch64"
- os: "windows-latest"
architecture: "aarch64"
steps:
- uses: actions/checkout@v4
- name: setup python ${{ matrix.python-version }}
uses: actions/[email protected]
with:
python-version: ${{ matrix.python-version }}
- name: upgrade pip
run: |
# install pip=>20.1 to use "pip cache dir"
python3 -m pip install --upgrade pip
- name: get pip cache dir
if: ${{ matrix.os != 'windows-latest' }}
id: pip-cache
run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: get pip cache dir (Windows)
if: ${{ matrix.os == 'windows-latest' }}
id: pip-cache-win
run: echo "dir=$(pip cache dir)" >> $env:GITHUB_OUTPUT
- name: cache dependencies
if: ${{ matrix.os != 'windows-latest' }}
uses: actions/[email protected]
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml', '**/tox.ini', 'requirements-dev*.in') }}
restore-keys: |
${{ runner.os }}-pip-
- name: cache dependencies (Windows)
if: ${{ matrix.os == 'windows-latest' }}
uses: actions/[email protected]
with:
path: ${{ steps.pip-cache-win.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml', '**/tox.ini', 'requirements-dev*.in') }}
restore-keys: |
${{ runner.os }}-pip-
- name: install dependencies
run: python3 -m pip install --upgrade tox
- name: run tox (Linux, Python 3.11)
if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.11' }}
run: |
python3 -m tox run -e py311-linux-${{ matrix.architecture }}-${{ matrix.requirements }}
- name: run tox (MacOS, Python 3.11)
if: ${{ (matrix.os == 'macos-13' || matrix.os == 'macos-latest') && matrix.python-version == '3.11' }}
run: |
python3 -m tox run -e py311-macos-${{ matrix.architecture }}-${{ matrix.requirements }}
- name: run tox (Windows, Python 3.11)
if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.11' }}
run: |
python3 -m tox run -e py311-win-${{ matrix.architecture }}-${{ matrix.requirements }}
- name: archive the requirements file
uses: actions/upload-artifact@v4
with:
name: py${{ matrix.python-version }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.requirements }}
path: venvs
pip-compile-containers:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: ["ubuntu-22.04"]
python-version: ["3.11"]
architecture: ["x86_64"]
requirements:
- "mlflow-tracking-requirements"
- "restapi-requirements"
- "pytorch-cpu-requirements"
- "pytorch-gpu-requirements"
- "tensorflow2-cpu-requirements"
- "tensorflow2-gpu-requirements"
steps:
- uses: actions/checkout@v4
- name: setup python ${{ matrix.python-version }}
uses: actions/[email protected]
with:
python-version: ${{ matrix.python-version }}
- name: upgrade pip
run: |
# install pip=>20.1 to use "pip cache dir"
python3 -m pip install --upgrade pip
- name: get pip cache dir
id: pip-cache
run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
- name: cache dependencies
uses: actions/[email protected]
with:
path: ${{ steps.pip-cache.outputs.dir }}
key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml', '**/tox.ini', '**/requirements-dev*.in', '**/docker/pip-tools/*-requirements.in') }}
restore-keys: |
${{ runner.os }}-pip-
- name: install dependencies
run: python3 -m pip install --upgrade tox
- name: run tox
run: |
python3 -m tox run -e py311-linux-${{ matrix.architecture }}-${{ matrix.requirements }}
- name: archive the requirements file
uses: actions/upload-artifact@v4
with:
name: py${{ matrix.python-version }}-${{ matrix.os }}-${{ matrix.architecture }}-${{ matrix.requirements }}
path: venvs
pip-compile-aarch64:
runs-on: ubuntu-22.04
strategy:
matrix:
tox-testenv:
- "py311-linux-aarch64-requirements-dev"
- "py311-linux-aarch64-requirements-dev-pytorch"
- "py311-linux-aarch64-requirements-dev-tensorflow"
- "py311-linux-aarch64-mlflow-tracking-requirements"
- "py311-linux-aarch64-restapi-requirements"
- "py311-linux-aarch64-pytorch-cpu-requirements"
- "py311-linux-aarch64-tensorflow2-cpu-requirements"
steps:
- uses: actions/checkout@v4
- name: set up docker qemu
uses: docker/setup-qemu-action@v3
- name: set up docker buildx
uses: docker/setup-buildx-action@v3
- name: setup dockerfile and output folder
run: |
echo 'FROM python:3.11-slim-bullseye' > Dockerfile.aarch64
echo 'COPY . /app' >> Dockerfile.aarch64
echo 'WORKDIR /app' >> Dockerfile.aarch64
echo 'RUN python -m pip install --upgrade pip tox && \' >> Dockerfile.aarch64
echo ' python -m tox run -e ${{ matrix.tox-testenv }}' >> Dockerfile.aarch64
sudo mkdir -p /image
sudo chown $(id -u):$(id -g) /image
sudo chmod 0777 /image
- name: build and save to output folder
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/arm64
push: false
file: Dockerfile.aarch64
tags: ${{ matrix.tox-testenv }}:dev
outputs: type=local,dest=/image
no-cache: true
pull: true
- name: archive the requirements file
uses: actions/upload-artifact@v4
with:
name: requirements-${{ matrix.tox-testenv }}
path: /image/app/venvs
consolidate-requirements-files:
runs-on: ubuntu-22.04
needs:
- pip-compile
- pip-compile-containers
- pip-compile-aarch64
steps:
- name: install dos2unix tool
run: sudo apt install -y dos2unix
- name: download all the compiled requirements files
uses: actions/download-artifact@v4
with:
path: artifacts
- name: standardize and consolidate files into venvs directory
run: |
find artifacts -type f -name "*requirements*.txt" -exec chmod 0644 {} \;
find artifacts -type f -name "*requirements*.txt" -exec dos2unix {} \;
mkdir venvs
mv artifacts/**/*.txt venvs
- name: archive the consolidated requirements files
uses: actions/upload-artifact@v4
with:
name: requirements-files
path: venvs