-
Notifications
You must be signed in to change notification settings - Fork 137
258 lines (242 loc) · 11.9 KB
/
ci-sage.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
name: Run Sage CI for Linux/Cygwin/macOS
## This GitHub Actions workflow provides:
##
## - portability testing, by building and testing this project on many platforms
## (Linux variants and Cygwin), each with two configurations (installed packages),
##
## - continuous integration, by building and testing other software
## that depends on this project.
##
## It runs on every pull request and push of a tag to the GitHub repository.
##
## The testing can be monitored in the "Actions" tab of the GitHub repository.
##
## After all jobs have finished (or are canceled) and a short delay,
## tar files of all logs are made available as "build artifacts".
##
## This GitHub Actions workflow uses the portability testing framework
## of SageMath (https://www.sagemath.org/). For more information, see
## https://doc.sagemath.org/html/en/developer/portability_testing.html
## The workflow consists of two jobs:
##
## - First, it builds a source distribution of the project
## and generates a script "update-pkgs.sh". It uploads them
## as a build artifact named upstream.
##
## - Second, it checks out a copy of the SageMath source tree.
## It downloads the upstream artifact and replaces the project's
## package in the SageMath distribution by the newly packaged one
## from the upstream artifact, by running the script "update-pkgs.sh".
## Then it builds a small portion of the Sage distribution.
##
## Many copies of the second step are run in parallel for each of the tested
## systems/configurations.
#on: [push, pull_request]
on:
pull_request:
types: [opened, synchronize]
push:
tags:
- '*'
env:
# Ubuntu packages to install so that the project's "make dist" can succeed
DIST_PREREQ:
# Name of this project in the Sage distribution
SPKG: arb
# Sage distribution packages to build
TARGETS_PRE: build/make/Makefile
TARGETS: SAGE_CHECK=yes arb symengine
TARGETS_OPTIONAL: SAGE_CHECK=warn e_antic
# Standard setting: Test the current beta release of Sage:
SAGE_REPO: sagemath/sage
SAGE_REF: develop
# Temporarily test on the branch from sage ticket https://trac.sagemath.org/ticket/34102
# (FLINT/Arb upgrade ticket); this is a no-op when merged in Sage develop.
SAGE_TRAC_GIT: https://github.com/sagemath/sagetrac-mirror.git
SAGE_TICKET: 34102
#REMOVE_PATCHES: "*"
jobs:
dist:
runs-on: ubuntu-latest
steps:
- name: Check out ${{ env.SPKG }}
uses: actions/checkout@v2
with:
path: build/pkgs/${{ env.SPKG }}/src
- name: Install prerequisites
run: |
sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install $DIST_PREREQ
if: env.DIST_PREREQ != ''
- name: Run make dist, prepare upstream artifact
run: |
(cd build/pkgs/${{ env.SPKG }}/src && (cd .. && tar czf - src) > arb-git.tar.gz) \
&& mkdir -p upstream && cp build/pkgs/${{ env.SPKG }}/src/*.tar.gz upstream/${{ env.SPKG }}-git.tar.gz \
&& echo "sage-package create ${{ env.SPKG }} --version git --tarball ${{ env.SPKG }}-git.tar.gz --type=optional" > upstream/update-pkgs.sh \
&& if [ -n "${{ env.REMOVE_PATCHES }}" ]; then echo "(cd ../build/pkgs/${{ env.SPKG }}/patches && rm -f ${{ env.REMOVE_PATCHES }}; :)" >> upstream/update-pkgs.sh; fi \
&& ls -l upstream/
- uses: actions/upload-artifact@v2
with:
path: upstream
name: upstream
cygwin:
uses: sagemath/sagetrac-mirror/.github/workflows/cygwin.yml@u/mkoeppe/ci_cygwin__refactor_using_reusable_workflows
with:
# FIXME: duplicated from env.TARGETS
targets: SAGE_CHECK=yes arb symengine
prefix: /opt/sage-arb
sage_repo: sagemath/sage
sage_ref: develop
upstream_artifact: upstream
sage_trac_git: https://github.com/sagemath/sagetrac-mirror.git
sage_trac_ticket: 34102
needs: [dist]
docker:
runs-on: ubuntu-latest
needs: [dist]
strategy:
fail-fast: false
max-parallel: 32
matrix:
tox_system_factor: [ubuntu-trusty-toolchain-gcc_9, ubuntu-xenial-toolchain-gcc_9, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy, ubuntu-kinetic, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, linuxmint-21, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, fedora-37, centos-7-devtoolset-gcc_11, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-devtoolset-gcc_11, raspbian-buster-armhf]
tox_packages_factor: [minimal, standard]
env:
TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
DOCKER_TARGETS: configured with-targets with-targets-optional
steps:
- name: Check out SageMath
uses: actions/checkout@v2
with:
repository: ${{ env.SAGE_REPO }}
ref: ${{ env.SAGE_REF }}
fetch-depth: 2000
if: env.SAGE_REPO != ''
- name: Check out git-trac-command
uses: actions/checkout@v2
with:
repository: sagemath/git-trac-command
path: git-trac-command
if: env.SAGE_TRAC_GIT != ''
- name: Check out SageMath from trac.sagemath.org
shell: bash {0}
run: |
git config --global user.email "[email protected]"
git config --global user.name "ci-sage workflow"
if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac fetch $SAGE_TICKET; then git merge FETCH_HEAD || echo "(ignored)"; exit 0; fi; sleep 40; done; exit 1
if: env.SAGE_TRAC_GIT != ''
- uses: actions/download-artifact@v2
with:
path: upstream
name: upstream
- name: Install test prerequisites
run: |
sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install tox python3-setuptools
- name: Update Sage packages from upstream artifact
run: |
(export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && sed -i.bak '/upstream/d' .dockerignore && echo "/:toolchain:/i ADD upstream upstream" | sed -i.bak -f - build/bin/write-dockerfile.sh && git diff)
- name: Configure and build Sage distribution within a Docker container
run: |
set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
- name: Copy logs from the Docker image or build container
run: |
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags) /bin/bash || true); fi
if [ -n "$CONTAINERS" ]; then for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done; fi
if: always()
- uses: actions/upload-artifact@v2
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
if: always()
- name: Print out logs for immediate inspection
# and markup the output with GitHub Actions logging commands
run: |
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
- name: Push Docker images
run: |
if [ -f .tox/$TOX_ENV/Dockertags ]; then
TOKEN="${{ secrets.DOCKER_PKG_GITHUB_TOKEN }}"
if [ -z "$TOKEN" ]; then
TOKEN="${{ secrets.GITHUB_TOKEN }}"
fi
echo "$TOKEN" | docker login docker.pkg.github.com -u ${{ github.actor }} --password-stdin
for a in $(cat .tox/$TOX_ENV/Dockertags); do
FULL_TAG=docker.pkg.github.com/$(echo ${{ github.repository }}|tr 'A-Z' 'a-z')/$a
docker tag $a $FULL_TAG
echo Pushing $FULL_TAG
docker push $FULL_TAG
done || echo "(Ignoring errors)"
fi
if: always()
macos:
runs-on: macos-latest
strategy:
fail-fast: false
max-parallel: 4
matrix:
os: [ macos-10.15, macos-11.0 ]
tox_system_factor: [homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, conda-forge-macos]
tox_packages_factor: [minimal, standard]
xcode_version_factor: [11.7, default, 12.3]
needs: [dist]
env:
TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}-${{ matrix.os }}-xcode_${{ matrix.xcode_version_factor }}
DOCKER_TARGETS: configured with-targets with-targets-optional
steps:
- name: Select Xcode version
run: |
if [ ${{ matrix.xcode_version_factor }} != default ]; then sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode_version_factor }}.app; fi
- name: Check out SageMath
uses: actions/checkout@v2
with:
repository: ${{ env.SAGE_REPO }}
ref: ${{ env.SAGE_REF }}
fetch-depth: 2000
if: env.SAGE_REPO != ''
- name: Check out git-trac-command
uses: actions/checkout@v2
with:
repository: sagemath/git-trac-command
path: git-trac-command
if: env.SAGE_TRAC_GIT != ''
- name: Check out SageMath from trac.sagemath.org
shell: bash {0}
run: |
git config --global user.email "[email protected]"
git config --global user.name "ci-sage workflow"
if [ ! -d .git ]; then git init; fi; git remote add trac ${{ env.SAGE_TRAC_GIT }} && x=1 && while [ $x -le 5 ]; do x=$(( $x + 1 )); sleep $(( $RANDOM % 60 + 1 )); if git-trac-command/git-trac fetch $SAGE_TICKET; then git merge FETCH_HEAD || echo "(ignored)"; exit 0; fi; sleep 40; done; exit 1
if: env.SAGE_TRAC_GIT != ''
- uses: actions/download-artifact@v2
with:
path: upstream
name: upstream
- name: Update Sage packages from upstream artifact
run: |
(export PATH=$(pwd)/build/bin:$PATH; (cd upstream && bash -x update-pkgs.sh) && git diff)
- name: Install test prerequisites
run: |
brew install tox
- name: Build and test with tox
# We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
# For doctesting, we use a lower parallelization to avoid timeouts.
run: |
MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
- name: Prepare logs artifact
run: |
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
if: always()
- uses: actions/upload-artifact@v1
with:
path: artifacts
name: ${{ env.LOGS_ARTIFACT_NAME }}
if: always()
- name: Print out logs for immediate inspection
# and markup the output with GitHub Actions logging commands
run: |
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
if: always()