-
Notifications
You must be signed in to change notification settings - Fork 3
317 lines (280 loc) · 14.6 KB
/
run_and_report_pytests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
name: Test PyKotor
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
OS_RUNNERS_JSON: '["windows-2019", "ubuntu-20.04", "macos-12"]'
PYTHON_VERSIONS_JSON: '["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]'
ARCHITECTURES_JSON: '["x86", "x64"]'
INTERPRETERS_JSON: '["python"]'
on:
# push:
pull_request:
types: [opened, reopened]
workflow_dispatch:
schedule:
- cron: '0 5 * * *'
permissions:
contents: write
jobs:
setup:
runs-on: ubuntu-latest
outputs:
os: ${{ steps.set_env.outputs.os }}
python-version: ${{ steps.set_env.outputs.python-version }}
architecture: ${{ steps.set_env.outputs.architecture }}
python-pypy: ${{ steps.set_env.outputs.python-pypy }}
#qt_version: ${#{ steps.set_env.outputs.qt_version }}
steps:
- name: Set environment variables
id: set_env
run: |
# Use a single line of JSON to avoid issues
$singleLineJson = '${{ env.OS_RUNNERS_JSON }}' -replace "`r", ""
$singleLineJson = $singleLineJson -replace "`n", ""
Write-Host $singleLineJson
echo "os<<EOF" >> $env:GITHUB_OUTPUT
echo $singleLineJson >> $env:GITHUB_OUTPUT
echo "EOF" >> $env:GITHUB_OUTPUT
$singleLineJson = '${{ env.PYTHON_VERSIONS_JSON }}' -replace "`r", ""
$singleLineJson = $singleLineJson -replace "`n", ""
Write-Host $singleLineJson
echo "python-version<<EOF" >> $env:GITHUB_OUTPUT
echo $singleLineJson >> $env:GITHUB_OUTPUT
echo "EOF" >> $env:GITHUB_OUTPUT
$singleLineJson = '${{ env.ARCHITECTURES_JSON }}' -replace "`r", ""
$singleLineJson = $singleLineJson -replace "`n", ""
Write-Host $singleLineJson
echo "architecture<<EOF" >> $env:GITHUB_OUTPUT
echo $singleLineJson >> $env:GITHUB_OUTPUT
echo "EOF" >> $env:GITHUB_OUTPUT
$singleLineJson = '${{ env.INTERPRETERS_JSON }}' -replace "`r", ""
$singleLineJson = $singleLineJson -replace "`n", ""
Write-Host $singleLineJson
echo "python-pypy<<EOF" >> $env:GITHUB_OUTPUT
echo $singleLineJson >> $env:GITHUB_OUTPUT
echo "EOF" >> $env:GITHUB_OUTPUT
shell: pwsh
runtests:
needs: setup
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false # Disable automatic cancellation of other jobs
matrix:
os: ${{ fromJson(needs.setup.outputs.os) }}
python-version: ${{ fromJson(needs.setup.outputs.python-version) }}
architecture: ${{ fromJson(needs.setup.outputs.architecture) }}
python_pypy: ${{ fromJson(needs.setup.outputs.python-pypy) }}
#qt_version: ['pyqt5', 'pyqt6', 'pyside2', 'pyside6']
exclude:
# unix x86 is definitely not supported.
- os: ubuntu-20.04
architecture: x86
- os: macos-12
architecture: x86
# Latest pypy is 3.10
- python_pypy: 'pypy'
python-version: '3.11'
- python_pypy: 'pypy'
python-version: '3.12'
steps:
- name: Determine Python version string
id: set-python-version
run: |
if ( "${{ matrix.python_pypy }}" -eq "pypy" ) {
Add-Content -Path $env:GITHUB_ENV -Value "PYTHON_VERSION=pypy-${{ matrix.python-version }}"
} else {
Add-Content -Path $env:GITHUB_ENV -Value "PYTHON_VERSION=${{ matrix.python-version }}"
}
shell: pwsh
- uses: actions/checkout@v4
- name: Set up ${{ matrix.python_pypy }} ${{ matrix.python-version }}
if: ${{ matrix.python_pypy == 'pypy' }}
uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
architecture: ${{ matrix.architecture }}
- name: Install development packages
if: ${{ success() || failure() }}
env:
MATRIX_ARCH: ${{ matrix.architecture }}
run: |
. ./install_python_venv.ps1 -noprompt -venv_name .venv_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }} -force_python_version ${{ matrix.python-version }}
& $pythonExePath -m pip install --upgrade pip
if ("${{ matrix.python-version }}" -eq "3.7") {
pip install -r requirements-dev-py37.txt --prefer-binary
}
else {
pip install -r requirements-dev.txt --prefer-binary
}
shell: pwsh
- name: Run all unittests/pytests
if: ${{ success() || failure() }}
run: |
. ./install_python_venv.ps1 -noprompt -venv_name .venv_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
& $pythonExePath -m pytest tests -v -ra -o log_cli=true --capture=no --junitxml=pytest_report.xml --html=pytest_report.html --self-contained-html --tb=no --continue-on-collection-errors
shell: pwsh
continue-on-error: true
- name: Upload Pytest Reports
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v4
with:
name: pytest_report_${{ matrix.os }}_${{ matrix.python_pypy }}_${{ matrix.python-version }}_${{ matrix.architecture }}
path: |
pytest_report.html
pytest_report.xml
retention-days: 90
add-test-result-status-badges:
needs: runtests
if: ${{ success() || failure() }}
runs-on: ubuntu-latest
concurrency:
group: add-test-status-badges-${{ github.ref }}
cancel-in-progress: true
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: all_pytest_reports
pattern: pytest_report_*
- name: Extract and update README with custom test status badges
shell: pwsh
run: |
# Git configuration and commit
git config --global user.name "GitHub Action"
git config --global user.email "[email protected]"
# Determine the branch that triggered the workflow
$branchName = "${{ github.ref_name }}"
$repository_owner = "${{ github.repository_owner }}"
$repository = "${{ github.repository }}"
$commitSHA = "${{ github.sha }}"
$testsResultsPath = "tests/results"
Remove-Item -Path $testsResultsPath -Recurse -Force -ErrorAction SilentlyContinue
$testsResultsCommitPath = Join-Path -Path $testsResultsPath -ChildPath "$commitSHA"
New-Item -ItemType Directory -Force -Path $testsResultsCommitPath -ErrorAction SilentlyContinue
$OS_NAMES = '${{ env.OS_RUNNERS_JSON }}' | ConvertFrom-Json
$PYTHON_VERSIONS = '${{ env.PYTHON_VERSIONS_JSON }}' | ConvertFrom-Json
$ARCHITECTURES = '${{ env.ARCHITECTURES_JSON }}' | ConvertFrom-Json
$INTERPRETERS = '${{ env.INTERPRETERS_JSON }}' | ConvertFrom-Json
$artifact_reports_dir = "./all_pytest_reports"
New-Item -ItemType Directory -Force -Path $artifact_reports_dir -ErrorAction SilentlyContinue
Get-ChildItem $artifact_reports_dir | ForEach-Object {
Write-Output "Moving $($_.FullName) to $testsResultsCommitPath..."
Move-Item -LiteralPath $_.FullName -Destination $testsResultsCommitPath
}
Remove-Item -Path $artifact_reports_dir -Recurse -Force -ErrorAction SilentlyContinue
$ReadmePath = "./README.md"
$ReadmeContent = Get-Content $ReadmePath -Raw
# Remove the readme temporarily to prevent merge conflicts
Remove-Item -Path $ReadmePath -Force -ErrorAction Stop
git fetch origin $branchName
git add $ReadmePath --force
git add $testsResultsPath --force
git add $testsResultsCommitPath --force
# Checking if there are newer commits on the remote branch than the commit that triggered the workflow
if (git log "${{ github.sha }}"..origin/$branchName --oneline) {
Write-Error "Newer commits are present on the remote branch, cannot update readme"
exit 1
}
git commit -m "Add test results, prepare README"
git push --force-with-lease origin HEAD:${{ github.ref_name }}
$commitSHA = git rev-parse HEAD
$testResults = @{}
Write-Output "Iterate through $testsResultsCommitPath"
Get-ChildItem $testsResultsCommitPath -Recurse -Filter pytest_report.xml | ForEach-Object {
$os, $python_or_pypy, $pythonVersion, $architecture = $_.BaseName -replace '^publish_', '' -split '_'
[xml]$TestResultsXml = Get-Content $_.FullName
$totalTests = [int]$TestResultsXml.testsuites.testsuite.tests
$failedTests = [int]$TestResultsXml.testsuites.testsuite.failures
$errors = [int]$TestResultsXml.testsuites.testsuite.errors
$passedTests = $totalTests - $failedTests - $errors
$resultFilePathHtml = $_.FullName -replace '\.xml$', '.html'
$relHtmlFilePath = Resolve-Path -Path $resultFilePathHtml -Relative
if ($relHtmlFilePath.StartsWith(".\") -or $relHtmlFilePath.StartsWith("./")) {
$cleanRelHtmlFilePath = $relHtmlFilePath.Substring(2)
} else {
$cleanRelHtmlFilePath = $relHtmlFilePath
}
$DetailsURL = "https://htmlpreview.github.io/?https://github.com/$repository/blob/$commitSHA/$cleanRelHtmlFilePath"
$key = $_.Directory.Name.Replace('pytest_report_', '').Replace('_', '-')
Write-Host "KEY FOUND: '$key'"
$testResults[$key] = @{
Passed = $passedTests
Failed = $failedTests + $errors
Total = $totalTests
DetailsURL = $DetailsURL
}
}
$WindowsBadgeContent = ""
$LinuxBadgeContent = ""
$MacOSBadgeContent = ""
$windowsBadgesStartPlaceholder = "<!-- WINDOWS-BADGES-START -->"
$windowsBadgesEndPlaceholder = "<!-- WINDOWS-BADGES-END -->"
$linuxBadgesStartPlaceholder = "<!-- LINUX-BADGES-START -->"
$linuxBadgesEndPlaceholder = "<!-- LINUX-BADGES-END -->"
$macosBadgesStartPlaceholder = "<!-- MACOS-BADGES-START -->"
$macosBadgesEndPlaceholder = "<!-- MACOS-BADGES-END -->"
function Replace-BadgeContent {
param (
[string]$readmeContent,
[string]$badgeContent,
[string]$startPlaceholder,
[string]$endPlaceholder
)
$pattern = [regex]::Escape($startPlaceholder) + "(.|\n)*?" + [regex]::Escape($endPlaceholder)
$replacement = $startPlaceholder + "`n" + $badgeContent + "`n" + $endPlaceholder
return $readmeContent -replace $pattern, $replacement
}
foreach ($OS in $OS_NAMES) {
foreach ($INTERPRETER in $INTERPRETERS) {
foreach ($PYTHON_VERSION in $PYTHON_VERSIONS) {
foreach ($ARCH in $ARCHITECTURES) {
if ($OS -ne "windows-2019" -and $OS -ne "windows-2022" -and $OS -ne "windows-latest" -and $ARCH -eq "x86") {
continue # no x86 support for unix.
}
$key = "$OS-$INTERPRETER-$PYTHON_VERSION-$ARCH"
$shortKey = "$INTERPRETER-$PYTHON_VERSION-$ARCH"
if ($testResults.ContainsKey($key)) {
$passedTests = $testResults[$key]['Passed']
$failedTests = $testResults[$key]['Failed']
$DetailsURL = $testResults[$key]['DetailsURL']
# Encode the label to replace spaces with underscores and URI-encode other special characters
$encodedKey = [System.Web.HttpUtility]::UrlEncode($shortKey.Replace(' ', '_').Replace('-', '--'))
$BadgeMarkdown = '[](' + $DetailsURL + ')'
} else {
Write-Host "No test results for '$key', must have failed, generating 'Build Failed' badge..."
$encodedKey = [System.Web.HttpUtility]::UrlEncode($shortKey.Replace(' ', '_').Replace('-', '--'))
$BadgeURLBuildFailed = "https://img.shields.io/badge/${encodedKey}_Build_Failed-lightgrey"
$DetailsURL = "https://github.com/$repository/actions/runs/${{ github.run_id }}"
$BadgeMarkdown = "[]($DetailsURL)"
}
switch ($OS) {
"windows-2019" { $WindowsBadgeContent += $BadgeMarkdown + "`n" }
"windows-2022" { $WindowsBadgeContent += $BadgeMarkdown + "`n" }
"windows-latest" { $WindowsBadgeContent += $BadgeMarkdown + "`n" }
"ubuntu-20.04" { $LinuxBadgeContent += $BadgeMarkdown + "`n" }
"ubuntu-22.04" { $LinuxBadgeContent += $BadgeMarkdown + "`n" }
"ubuntu-latest" { $LinuxBadgeContent += $BadgeMarkdown + "`n" }
"macos-10" { $MacOSBadgeContent += $BadgeMarkdown + "`n" }
"macos-12" { $MacOSBadgeContent += $BadgeMarkdown + "`n" }
"macos-latest" { $MacOSBadgeContent += $BadgeMarkdown + "`n" }
}
}
}
}
}
$ReadmeContent = Replace-BadgeContent -readmeContent $ReadmeContent -badgeContent $WindowsBadgeContent.TrimEnd() -startPlaceholder $windowsBadgesStartPlaceholder -endPlaceholder $windowsBadgesEndPlaceholder
$ReadmeContent = Replace-BadgeContent -readmeContent $ReadmeContent -badgeContent $LinuxBadgeContent.TrimEnd() -startPlaceholder $linuxBadgesStartPlaceholder -endPlaceholder $linuxBadgesEndPlaceholder
$ReadmeContent = Replace-BadgeContent -readmeContent $ReadmeContent -badgeContent $MacOSBadgeContent.TrimEnd() -startPlaceholder $macosBadgesStartPlaceholder -endPlaceholder $macosBadgesEndPlaceholder
Set-Content -Path $ReadmePath -Value $ReadmeContent
git fetch origin $branchName
git add $ReadmePath
# Checking if there are newer commits on the remote branch than the commit that triggered the workflow
if (git log "${{ github.sha }}"..origin/$branchName --oneline) {
Write-Error "Newer commits are present on the remote branch, cannot update readme"
exit 1
}
git commit -m "Update README.md status badges."
git push --force-with-lease origin HEAD:${{ github.ref_name }}