Skip to content

Python rust backend #2860

Python rust backend

Python rust backend #2860

name: Build and test
on:
push:
branches: [ "master" ]
tags: "v*.*.*"
pull_request:
branches: [ "master" ]
workflow_dispatch:
env:
checkpoints_enabled: false
CONVERT_PATH_FN: "echo"
RUST_BACKTRACE: "1"
QT_QPA_PLATFORM: "offscreen"
NATIVE_SHELL_CMD: "bash -c"
BASH_FLAGS: ""
PIP_BREAK_SYSTEM_PACKAGES: 1
permissions:
contents: write
checks: write
jobs:
##############################################################
## Release creation for tag pushes
##############################################################
maybe_create_release:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
if: startsWith(github.ref, 'refs/tags/')
- name: Create Release
if: startsWith(github.ref, 'refs/tags/')
uses: softprops/action-gh-release@v2
id: create_release
with:
draft: true
generate_release_notes: true
append_body: true
body: |
This is a development release. Consider alpha / unstable until v1.0.0.
## Installation
Choose between Python wheels, portable folder archives and packages/installers per platform.
For details, refer to the [full installation instructions](https://github.com/SanderVocke/shoopdaloop/blob/master/INSTALL.md).
- name: Store asset upload URL
if: startsWith(github.ref, 'refs/tags/')
shell: bash
run: echo "${{ steps.create_release.outputs.upload_url }}" > release_upload_assets_url.txt
- name: Upload asset upload URL
if: startsWith(github.ref, 'refs/tags/')
uses: actions/upload-artifact@v4
with:
name: release_upload_assets_url
path: release_upload_assets_url.txt
- uses: ./.github/actions/checkpoint
if: startsWith(github.ref, 'refs/tags/')
##############################################################
## Linux
##############################################################
build_linux:
strategy:
fail-fast: false
matrix:
kind:
- name: linux_release
release_build: true
package_kind: portable_folder
second_package_kind: appimage
third_package_kind: test_binaries
upload_release_assets: true
platform: linux-x64
- name: linux_debug
release_build: false
package_kind: portable_folder
second_package_kind: test_binaries
upload_release_assets: true
platform: linux-x64
# - name: linux_asan
# # Note: ASAN build does not have a test because it is not trivial to deploy
# # an asan-built program on another system. The build is here to ensure the
# # build process still works. To use ASAN, compile like this natively.
# release_build: false
# package_kind: appimage
# cmake_opts: '"ENABLE_ASAN" = "On"'
# platform: linux-x64
# - name: linux_coverage
# release_build: false
# cmake_opts: '"ENABLE_COVERAGE" = "On"'
# package_kind: appimage
# coverage: true
# platform: linux-x64
container: [ "docker.io/sandervocke/shoopdaloop_build_base_debian_bullseye_x86_64:latest" ]
container_options: [ "--user root --workdir /" ]
arch: [ x86_64 ]
manylinux_arch: [ x86_64 ]
runs-on: ubuntu-latest
container:
image: ${{ matrix.container }}
options: ${{ matrix.container_options }}
env:
PATH_SEPARATOR: ":"
COMPRESS_FOLDER_FN: bash -c "OUT=\$(pwd)/\$(basename \$1.tar.gz); PARENT=\$1/..; FOLDER=\$(basename \$1); cd \$PARENT; tar -czf \$OUT \$FOLDER > /dev/null; echo \$OUT" _
# checkpoints_enabled: true
needs: maybe_create_release
#if: false
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- uses: ./.github/actions/checkpoint_setup
with:
shell: bash --noprofile --norc ${{ env.BASH_FLAGS }} -eo pipefail
- name: Build
if: always()
uses: ./.github/actions/build_toplevel
with:
name: ${{ matrix.kind.name }}
platform: ${{ matrix.kind.platform }}
release_build: ${{ matrix.kind.release_build }}
cmake_opts: ${{ matrix.kind.cmake_opts }}
prepare_kind: 'linux'
python: python3.9
package_kind: ${{ matrix.kind.package_kind }}
second_package_kind: ${{ matrix.kind.second_package_kind }}
third_package_kind: ${{ matrix.kind.third_package_kind }}
coverage: ${{ matrix.kind.coverage }}
upload_release_assets: ${{ (matrix.kind.upload_release_assets && startsWith(github.ref, 'refs/tags/')) && 'true' || 'false' }}
- uses: ./.github/actions/checkpoint
if: always()
test_linux:
strategy:
fail-fast: false
matrix:
kind:
- name: release_debian_stable
python: python3
container: docker.io/sandervocke/shoopdaloop_run_base_debian_latest_x86_64:latest
package: shoopdaloop.*.release-linux-x64.portable.tar.gz
second_package: shoopdaloop.*.release-linux-x64.test_binaries.tar.gz
- name: debug_debian_stable
python: python3
container: docker.io/sandervocke/shoopdaloop_run_base_debian_latest_x86_64:latest
package: shoopdaloop.*.debug-linux-x64.portable.tar.gz
second_package: shoopdaloop.*.debug-linux-x64.test_binaries.tar.gz
# - name: coverage_debian_bullseye
# python: python3
# container: docker.io/sandervocke/shoopdaloop_build_base_debian_bullseye_x86_64:latest
# package: wheel_linux_coverage
# coverage: true
# run_cmd_prefix: >-
# BASEDIR=$(pwd)
# EXCLUDE='/*/third_party/* /*/catch2/*'
# REPORTDIR=coverage_reports
# BUILDDIR=$(realpath build/cp*/)
# ORI_BUILD_DIR=$(cat ori_build_dir.txt)
# DO_GENHTML=0
# SHOOP_LOG=QoverageCollectorFactory=debug
# QML_IMPORT_PATH=$(~/.local/bin/qoverage --importpath)
# scripts/run_and_generate_coverage.sh
runs-on: ubuntu-latest
needs: build_linux
#if: false
container:
image: ${{ matrix.kind.container }}
options: --user root --workdir / # Note: this gets disregarded if container is null
# env:
# checkpoints_enabled: true
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: ./.github/actions/checkpoint_setup
with:
shell: bash --noprofile --norc ${{ env.BASH_FLAGS }} -eo pipefail
- uses: ./.github/actions/checkpoint
if: always()
- name: Test
if: always()
uses: ./.github/actions/test_toplevel
with:
name: ${{ matrix.kind.name }}
package: ${{ matrix.kind.package }}
second_package: ${{ matrix.kind.second_package }}
asan: ${{ matrix.kind.asan }}
coverage: ${{ matrix.kind.coverage }}
python: ${{ matrix.kind.python }}
codecov_token: ${{ secrets.CODECOV_TOKEN }}
- uses: ./.github/actions/checkpoint
if: always()
##############################################################
## Linux (ARM emulated)
##############################################################
# build_linux_arm:
# strategy:
# fail-fast: false
# matrix:
# container: [ "docker.io/sandervocke/shoopdaloop_build_base_debian_bullseye_arm64:latest" ]
# arch: [ arm64 ]
# manylinux_arch: [ aarch64 ]
#
# kind:
# - name: linux_arm_release
# release_build: true
# package_kind: portable_folder
# second_package_kind: appimage
# upload_release_assets: true
# platform: linux-arm
# - name: linux_arm_debug
# build_type: 'Debug'
# platform: linux-arm
# runs-on: ubuntu-latest
# needs: maybe_create_release
# env:
# PATH_SEPARATOR: ":"
# COMPRESS_PORTABLE_FOLDER: 'bash -c "OUT=\$(pwd)/\$(basename \$1.tar.gz); PARENT=\$1/..; FOLDER=\$(basename \$1); cd \$PARENT; tar -czvf \$OUT \$FOLDER > /dev/null; echo \$OUT"'
# #if: false
# steps:
# - name: Checkout
# uses: actions/checkout@v4
# with:
# submodules: recursive
# - name: Start QEMU container
# if: matrix.container
# uses: sandervocke/setup-qemu-container@v1
# with:
# container: ${{ matrix.container }}
# arch: ${{ matrix.arch }}
# - name: Setup Shell Wrapper
# uses: sandervocke/setup-shell-wrapper@v1
# - name: Use container shell
# shell: bash
# run: echo "WRAP_SHELL=run-in-container.sh --shell bash" >> $GITHUB_ENV !!!!!!! fix
# - name: Build
# uses: ./.github/actions/build_toplevel
# with:
# name: ${{ matrix.kind.name }}
# platform: ${{ matrix.kind.platform }}
# cmake_build_type: ${{ matrix.kind.build_type }}
# cmake_opts: ${{ matrix.kind.cmake_opts }}
# prepare_kind: 'linux'
# python: python3.9
# package_kind: ${{ matrix.kind.package_kind }}
# second_package_kind: ${{ matrix.kind.package_kind }}
# coverage: ${{ matrix.kind.coverage }}
# upload_release_assets: ${{ (matrix.kind.upload_release_assets && startsWith(github.ref, 'refs/tags/')) && 'true' || 'false' }}
# test_linux_arm:
# strategy:
# fail-fast: false
# matrix:
# container_arch: [ arm64 ]
# kind:
# - name: release_wheel_debian_bullseye_arm
# python: python3
# container: docker.io/sandervocke/shoopdaloop_run_base_debian_latest_arm64:latest
# package: wheel_linux_arm_release
# - name: release_deb_ubuntu_latest_arm
# python: python3
# container: docker.io/sandervocke/shoopdaloop_run_base_ubuntu_latest_arm64:latest
# package: package_linux_arm_release_deb
# install_with_sudo: 'true'
# runs-on: ubuntu-latest
# needs: build_linux_arm
# #if: false
# steps:
# - name: Checkout
# uses: actions/checkout@v4
# - name: Start QEMU container
# uses: sandervocke/setup-qemu-container@v1
# with:
# container: ${{ matrix.kind.container }}
# arch: ${{ matrix.container_arch }}
# - name: Setup Shell Wrapper
# uses: sandervocke/setup-shell-wrapper@v1
# - name: Use container shell
# shell: bash
# run: echo "WRAP_SHELL=run-in-container.sh --shell bash" >> $GITHUB_ENV !!!!!!! fix this
# - name: Test
# uses: ./.github/actions/test_toplevel
# with:
# name: ${{ matrix.kind.name }}
# package: ${{ matrix.kind.package }}
# asan: ${{ matrix.kind.asan }}
# coverage: ${{ matrix.kind.coverage }}
# python: ${{ matrix.kind.python }}
# codecov_token: ${{ secrets.CODECOV_TOKEN }}
# install_with_sudo: ${{ matrix.kind.install_with_sudo }}
# do_qml_tests: false # Too slow on ARM emulation
# - uses: ./.github/actions/checkpoint
##############################################################
## MacOS
##############################################################
build_macos:
strategy:
fail-fast: false
matrix:
kind:
- name: macos_release
release_build: true
package_kind: portable_folder
second_package_kind: dmg
third_package_kind: test_binaries
os: macos-12
upload_release_assets: true
platform: macos-x64
- name: macos_debug
release_build: false
package_kind: portable_folder
second_package_kind: test_binaries
os: macos-12
upload_release_assets: true
platform: macos-x64
- name: macos_release_arm
release_build: true
package_kind: portable_folder
second_package_kind: dmg
third_package_kind: test_binaries
os: macos-14
upload_release_assets: true
platform: macos-arm
- name: macos_debug_arm
release_build: false
package_kind: portable_folder
second_package_kind: test_binaries
os: macos-14
upload_release_assets: true
platform: macos-arm
runs-on: ${{ matrix.kind.os }}
needs: maybe_create_release
#if: false
env:
PATH_SEPARATOR: ":"
COMPRESS_FOLDER_FN: bash -c "OUT=\$(pwd)/\$(basename \$1.tar.gz); PARENT=\$1/..; FOLDER=\$(basename \$1); cd \$PARENT; tar -czf \$OUT \$FOLDER > /dev/null; echo \$OUT" _
# checkpoints_enabled: true
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- uses: ./.github/actions/checkpoint_setup
with:
shell: bash --noprofile --norc ${{ env.BASH_FLAGS }} -eo pipefail
- name: Build
uses: ./.github/actions/build_toplevel
with:
name: ${{ matrix.kind.name }}
platform: ${{ matrix.kind.platform }}
cmake_build_type: ${{ matrix.kind.build_type }}
cmake_opts: ${{ matrix.kind.cmake_opts }}
prepare_kind: 'macos'
python: python3.9
appbundle: ${{ matrix.kind.appbundle }}
package_kind: ${{ matrix.kind.package_kind }}
second_package_kind: ${{ matrix.kind.second_package_kind }}
third_package_kind: ${{ matrix.kind.third_package_kind }}
release_build: ${{ matrix.kind.release_build }}
coverage: ${{ matrix.kind.coverage }}
upload_release_assets: ${{ (matrix.kind.upload_release_assets && startsWith(github.ref, 'refs/tags/')) && 'true' || 'false' }}
- uses: ./.github/actions/checkpoint
if: always()
test_macos:
strategy:
fail-fast: false
matrix:
kind:
- name: release_macos_arm
python: python3
container: null
os: macos-14
package: shoopdaloop.*.release-macos-arm.portable.tar.gz
second_package: shoopdaloop.*.release-macos-arm.test_binaries.tar.gz
- name: debug_macos_arm
python: python3
container: null
os: macos-14
package: shoopdaloop.*.debug-macos-arm.portable.tar.gz
second_package: shoopdaloop.*.debug-macos-arm.test_binaries.tar.gz
- name: release_macos
python: python3
container: null
os: macos-12
package: shoopdaloop.*.release-macos-x64.portable.tar.gz
second_package: shoopdaloop.*.release-macos-x64.test_binaries.tar.gz
- name: debug_macos
python: python3
container: null
os: macos-12
package: shoopdaloop.*.debug-macos-x64.portable.tar.gz
second_package: shoopdaloop.*.debug-macos-x64.test_binaries.tar.gz
runs-on: ${{ matrix.kind.os }}
needs: build_macos
#if: false
container:
image: ${{ matrix.kind.container }}
options: --user root --workdir / # Note: this gets disregarded if container is null
env:
PATH_SEPARATOR: ":"
# checkpoints_enabled: true
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: ./.github/actions/checkpoint_setup
with:
shell: bash --noprofile --norc ${{ env.BASH_FLAGS }} -eo pipefail
- name: Test
uses: ./.github/actions/test_toplevel
with:
name: ${{ matrix.kind.name }}
package: ${{ matrix.kind.package }}
second_package: ${{ matrix.kind.second_package }}
asan: ${{ matrix.kind.asan }}
coverage: ${{ matrix.kind.coverage }}
python: ${{ matrix.kind.python }}
codecov_token: ${{ secrets.CODECOV_TOKEN }}
- uses: ./.github/actions/checkpoint
# # ##############################################################
# # ## Windows
# # ##############################################################
build_windows:
strategy:
fail-fast: false
matrix:
kind:
- name: windows_release
os: windows-2022
python: python.exe
pip: python.exe -m pip
container_image: null
prepare: 'windows'
release_build: true
package_kind: portable_folder
second_package_kind: innosetup
third_package_kind: test_binaries
upload_release_assets: true
platform: windows-msvc-x64
- name: windows_debug
os: windows-2022
python: python.exe
pip: python.exe -m pip
container_image: null
prepare: 'windows'
release_build: false
package_kind: portable_folder
second_package_kind: test_binaries
platform: windows-msvc-x64
runs-on: ${{ matrix.kind.os}}
needs: maybe_create_release
env:
NATIVE_SHELL_CMD: "powershell -Command"
PATH_SEPARATOR: ";"
CONVERT_PATH_FN: bash -c "cygpath -w \$1 | sed 's/\\\\/\\\\\\\\/g'" _
COMPRESS_FOLDER_FN: bash -c "OUT=\$(pwd)/\$(basename \$1.tar.gz); PARENT=\$1/..; FOLDER=\$(basename \$1); cd \$PARENT; tar -czf \$OUT \$FOLDER > /dev/null; echo \$OUT" _
# checkpoints_enabled: true
container:
image: ${{ matrix.kind.container_image }}
options: --user root --workdir /
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: recursive
- uses: ./.github/actions/checkpoint_setup
with:
shell: bash --noprofile --norc ${{ env.BASH_FLAGS }} -eo pipefail
- name: Build
uses: ./.github/actions/build_toplevel
with:
name: ${{ matrix.kind.name }}
platform: ${{ matrix.kind.platform }}
cmake_build_type: ${{ matrix.kind.build_type }}
cmake_opts: ${{ matrix.kind.cmake_opts }}
prepare_kind: ${{ matrix.kind.prepare }}
python: ${{ matrix.kind.python }}
package_kind: ${{ matrix.kind.package_kind }}
second_package_kind: ${{ matrix.kind.second_package_kind }}
third_package_kind: ${{ matrix.kind.third_package_kind }}
coverage: ${{ matrix.kind.coverage }}
release_build: ${{ matrix.kind.release_build }}
upload_release_assets: ${{ (matrix.kind.upload_release_assets && startsWith(github.ref, 'refs/tags/')) && 'true' || 'false' }}
- uses: ./.github/actions/checkpoint
test_windows:
strategy:
fail-fast: false
matrix:
kind:
- name: release_windows
python: python.exe
container: null
os: windows-2022
package: shoopdaloop.*.release-windows-msvc-x64.portable.tar.gz
second_package: shoopdaloop.*.release-windows-msvc-x64.test_binaries.tar.gz
pathconvert: "| cygpath -u -f -"
# - name: release_windows_innosetup
# python: python.exe
# container: null
# os: windows-2022
# package: package_windows_release
# pathconvert: "| cygpath -u -f -"
- name: debug_windows
python: python.exe
container: null
os: windows-2022
package: shoopdaloop.*.debug-windows-msvc-x64.portable.tar.gz
second_package: shoopdaloop.*.debug-windows-msvc-x64.test_binaries.tar.gz
pathconvert: "| cygpath -u -f -"
runs-on: ${{ matrix.kind.os }}
needs: build_windows
# env:
# checkpoints_enabled: true
#if: false
container:
image: ${{ matrix.kind.container }}
options: --user root --workdir / # Note: this gets disregarded if container is null
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: ./.github/actions/checkpoint_setup
with:
shell: bash --noprofile --norc ${{ env.BASH_FLAGS }} -eo pipefail
- name: Test
uses: ./.github/actions/test_toplevel
with:
name: ${{ matrix.kind.name }}
package: ${{ matrix.kind.package }}
second_package: ${{ matrix.kind.second_package }}
asan: ${{ matrix.kind.asan }}
coverage: ${{ matrix.kind.coverage }}
python: ${{ matrix.kind.python }}
codecov_token: ${{ secrets.CODECOV_TOKEN }}
- uses: ./.github/actions/checkpoint
# ##############################################################
# ## Crash collection
# ##############################################################
collect_crashes:
runs-on: ubuntu-latest
needs:
- build_linux
- test_linux
# - build_linux_arm
# - test_linux_arm
- build_macos
# - test_macos
- build_windows
# - test_windows
if: always()
container:
image: archlinux:latest
options: --user root --workdir /
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: ./.github/actions/checkpoint_setup
with:
shell: bash --noprofile --norc -eo pipefail
- name: Download artifacts
uses: actions/download-artifact@v4
with:
pattern: crashdumps_*_${{ github.run_number }}
path: crashdumps
- name: Count crash dumps
id: count
shell: wrap-shell {0}
run: |
mkdir -p crashdumps
echo "crashdumps=$(ls crashdumps/crashdumps_* | wc -l)" | tee -a $GITHUB_OUTPUT
pacman --noconfirm -Syy
pacman --noconfirm -S --needed tree git
- name: Download debug information
uses: actions/download-artifact@v4
if: steps.count.outputs.crashdumps != 0
with:
path: debuginfo
pattern: debuginfo_*
merge-multiple: true
- name: Cache minidump-stackwalk
id: cache-minidump
uses: actions/cache@v4
with:
path: minidump-stackwalk
key: minidump-stackwalk-1
- name: Get minidump package sources
if: steps.count.outputs.crashdumps != 0 && steps.cache-minidump.outputs.cache-hit != 'true'
shell: wrap-shell {0}
run: |
pacman --noconfirm -Syy
pacman --noconfirm -S --needed git base-devel && git clone https://aur.archlinux.org/minidump-stackwalk.git
- name: Makepkg Build
id: makepkg
if: steps.count.outputs.crashdumps != 0 && steps.cache-minidump.outputs.cache-hit != 'true'
uses: edlanglois/pkgbuild-action@v1
with:
pkgdir: minidump-stackwalk
- name: Install minidump package
if: steps.count.outputs.crashdumps != 0
shell: wrap-shell {0}
run: |
pacman --noconfirm -U minidump-stackwalk/minidump-stackwalk*.pkg*
- name: Process dumps
if: steps.count.outputs.crashdumps != 0
shell: wrap-shell {0}
run: |
tree crashdumps
if [ -e debuginfo ]; then
cp -r debuginfo crashdumps/debuginfo
echo "found_debuginfo=true" | tee -a $GITHUB_ENV
fi
mkdir -p debuginfo/breakpad_symbols
mkdir -p crashdumps/debuginfo
for dump in $(find crashdumps -name "*.dmp"); do
echo ""
echo "================================================="
echo "Processing $dump..."
echo "================================================="
minidump-stackwalk $dump debuginfo/breakpad_symbols || true | tee $dump.stackwalk
done
- name: Upload crash dumps to Bugsplat
if: (success() || failure()) && steps.count.outputs.crashdumps != 0
shell: wrap-shell {0}
run: |
for dump in $(find crashdumps -name "*.dmp"); do
echo ""
echo "================================================="
echo "Uploading $dump..."
echo "================================================="
bash ./scripts/upload_crash_info.sh $dump git-${{ github.sha }} "[email protected]" "Found on GitHub Actions run: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" ${dump}.moreinfo
done
- uses: actions/setup-node@v4
if: (success() || failure()) && steps.count.outputs.crashdumps != 0
with:
node-version: 19
- name: Upload BugSplat debug symbols
if: (success() || failure()) && steps.count.outputs.crashdumps != 0 && env.found_debuginfo == 'true'
uses: SanderVocke/symbol-upload@test_action
with:
clientId: "${{ secrets.BUGSPLAT_CLIENT_ID }}"
clientSecret: "${{ secrets.BUGSPLAT_CLIENT_SECRET }}"
database: shoopdaloop
application: shoopdaloop
version: git-${{ github.sha }}
files: "**/*.sym"
directory: crashdumps/debuginfo
- name: Upload crash dumps as artifact
if: (success() || failure()) && steps.count.outputs.crashdumps != 0
uses: actions/upload-artifact@v4
with:
name: crashdumps_summary
path: crashdumps