Skip to content

Desktop - End-to-end tests #445

Desktop - End-to-end tests

Desktop - End-to-end tests #445

Workflow file for this run

# Workflow for triggering `test-manager` on select platforms.
#
# This is a rather complex workflow. The complexity mainly stems from these sources:
# * figuring out which platforms to test on which runners (prepare-matrices)
# * figuring out if the app and e2e-tests should be built on the runner (build-{linux,windows,macos})
# or if we should download the artifacts from https://releases.mullvad.net/desktop/
# * compiling the output from the different runners and executed platforms.
---
name: Desktop - End-to-end tests
on:
schedule:
- cron: '0 0 * * *'
workflow_dispatch:
inputs:
oses:
description: "Space-delimited list of targets to run tests on, e.g. `debian11 debian12`. \
Available images are:\n
`debian11 debian12 ubuntu2004 ubuntu2204 ubuntu2304 ubuntu2310 fedora39 \
fedora38 fedora37 fedora36 windows10 windows11 macos12 macos13 macos14`."
default: ''
required: false
type: string
jobs:
prepare-matrices:
runs-on: ubuntu-latest
steps:
- name: Generate matrix for Linux builds
shell: bash
run: |
# A list of VMs to run the tests on. These refer to the names defined
# in $XDG_CONFIG_DIR/mullvad-test/config.json on the runner.
all='["debian11","debian12","ubuntu2004","ubuntu2204","ubuntu2304","ubuntu2310","fedora39","fedora38","fedora37","fedora36"]'
oses="${{ github.event.inputs.oses }}"
echo "OSES: $oses"
if [[ -z "$oses" || "$oses" == "null" ]]; then
selected="$all"
else
oses=$(printf '%s\n' $oses | jq . -R | jq . -s)
selected=$(jq -cn --argjson oses "$oses" --argjson all "$all" '$all - ($all - $oses)')
fi
echo "Selected targets: $selected"
echo "linux_matrix=$selected" >> $GITHUB_ENV
- name: Generate matrix for Windows builds
shell: bash
run: |
all='["windows10","windows11"]'
oses="${{ github.event.inputs.oses }}"
if [[ -z "$oses" || "$oses" == "null" ]]; then
selected="$all"
else
oses=$(printf '%s\n' $oses | jq . -R | jq . -s)
selected=$(jq -cn --argjson oses "$oses" --argjson all "$all" '$all - ($all - $oses)')
fi
echo "Selected targets: $selected"
echo "windows_matrix=$selected" >> $GITHUB_ENV
- name: Generate matrix for macOS builds
shell: bash
run: |
all='["macos12","macos13","macos14"]'
oses="${{ github.event.inputs.oses }}"
if [[ -z "$oses" || "$oses" == "null" ]]; then
selected="$all"
else
oses=$(printf '%s\n' $oses | jq . -R | jq . -s)
selected=$(jq -cn --argjson oses "$oses" --argjson all "$all" '$all - ($all - $oses)')
fi
echo "Selected targets: $selected"
echo "macos_matrix=$selected" >> $GITHUB_ENV
outputs:
linux_matrix: ${{ env.linux_matrix }}
windows_matrix: ${{ env.windows_matrix }}
macos_matrix: ${{ env.macos_matrix }}
prepare-linux:
needs: prepare-matrices
if: |
needs.prepare-matrices.outputs.linux_matrix != '[]' &&
!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main'
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Use custom container image if specified
if: ${{ github.event.inputs.override_container_image != '' }}
run: echo "inner_container_image=${{ github.event.inputs.override_container_image }}"
>> $GITHUB_ENV
- name: Use default container image and resolve digest
if: ${{ github.event.inputs.override_container_image == '' }}
run: |
echo "inner_container_image=$(cat ./building/linux-container-image.txt)" >> $GITHUB_ENV
outputs:
container_image: ${{ env.inner_container_image }}
build-linux:
needs: prepare-linux
runs-on: ubuntu-latest
container:
image: ${{ needs.prepare-linux.outputs.container_image }}
continue-on-error: true
steps:
# Fix for HOME path overridden by GH runners when building in containers, see:
# https://github.com/actions/runner/issues/863
- name: Fix HOME path
run: echo "HOME=/root" >> $GITHUB_ENV
- name: Checkout repository
uses: actions/checkout@v3
- name: Checkout binaries submodule
run: |
git config --global --add safe.directory '*'
git submodule update --init --depth=1 dist-assets/binaries
- name: Build app
env:
USE_MOLD: false
run: ./build.sh
- name: Build test executable
run: ./gui/scripts/build-test-executable.sh
- uses: actions/upload-artifact@v3
if: '!cancelled()'
with:
name: linux-build
path: |
./dist/*.rpm
./dist/*.deb
./dist/app-e2e-*
e2e-test-linux:
name: Linux end-to-end tests
needs: [prepare-matrices, build-linux]
if: '!cancelled()'
runs-on: [self-hosted, desktop-test, Linux] # app-test-linux
timeout-minutes: 240
strategy:
fail-fast: false
matrix:
os: ${{ fromJSON(needs.prepare-matrices.outputs.linux_matrix) }}
steps:
- uses: actions/download-artifact@v3
if: ${{ needs.build-linux.result == 'success' }}
with:
name: linux-build
path: ~/.cache/mullvad-test/packages
- name: Checkout repository
uses: actions/checkout@v4
- name: Run end-to-end tests
shell: bash -ieo pipefail {0}
run: |
git fetch --tags --force
./test/ci-runtests.sh ${{ matrix.os }}
- uses: actions/upload-artifact@v3
if: '!cancelled()'
with:
name: ${{ matrix.os }}_report
path: ./test/.ci-logs/${{ matrix.os }}_report
build-windows:
needs: prepare-matrices
if: |
needs.prepare-matrices.outputs.windows_matrix != '[]' &&
!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main'
runs-on: windows-latest
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Checkout submodules
run: git submodule update --init --depth=1
- name: Install Protoc
uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/setup-node@v3
with:
token: ${{ secrets.GITHUB_TOKEN }}
node-version: 18
- name: Install Rust
uses: actions-rs/[email protected]
with:
toolchain: stable
target: i686-pc-windows-msvc
default: true
- name: Install msbuild
uses: microsoft/[email protected]
with:
vs-version: 16
- name: Build app
shell: bash
# TODO: For some reason, 'npm run pack:win' wants to publish the artifacts
continue-on-error: true
run: ./build.sh
- name: Build test executable
shell: bash
run: ./gui/scripts/build-test-executable.sh
- uses: actions/upload-artifact@v3
if: '!cancelled()'
with:
name: windows-build
path: .\dist\*.exe
e2e-test-windows:
needs: [prepare-matrices, build-windows]
if: '!cancelled()'
name: Windows end-to-end tests
runs-on: [self-hosted, desktop-test, Linux] # app-test-linux
timeout-minutes: 240
strategy:
fail-fast: false
matrix:
os: ${{ fromJSON(needs.prepare-matrices.outputs.windows_matrix) }}
steps:
- uses: actions/download-artifact@v3
if: ${{ needs.build-windows.result == 'success' }}
with:
name: windows-build
path: ~/.cache/mullvad-test/packages
- name: Checkout repository
uses: actions/checkout@v4
- name: Run end-to-end tests
shell: bash -ieo pipefail {0}
run: |
git fetch --tags --force
./test/ci-runtests.sh ${{ matrix.os }}
- uses: actions/upload-artifact@v3
if: '!cancelled()'
with:
name: ${{ matrix.os }}_report
path: ./test/.ci-logs/${{ matrix.os }}_report
build-macos:
needs: prepare-matrices
if: |
needs.prepare-matrices.outputs.macos_matrix != '[]' &&
!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main'
runs-on: [self-hosted, desktop-test, macOS] # app-test-macos-arm
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Checkout submodules
run: git submodule update --init --depth=1
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: 1.21.3
- name: Install Protoc
uses: arduino/setup-protoc@v2
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/setup-node@v3
with:
token: ${{ secrets.GITHUB_TOKEN }}
node-version: 18
cache: 'npm'
cache-dependency-path: gui/package-lock.json
- name: Build app
run: ./build.sh
- name: Build test executable
run: ./gui/scripts/build-test-executable.sh
- uses: actions/upload-artifact@v3
if: '!cancelled()'
with:
name: macos-build
path: |
./dist/*.pkg
./dist/app-e2e-*
e2e-test-macos:
needs: [prepare-matrices, build-macos]
if: '!cancelled()'
name: macOS end-to-end tests
runs-on: [self-hosted, desktop-test, macOS] # app-test-macos-arm
timeout-minutes: 240
strategy:
fail-fast: false
matrix:
os: ${{ fromJSON(needs.prepare-matrices.outputs.macos_matrix) }}
steps:
- uses: actions/download-artifact@v3
if: ${{ needs.build-macos.result == 'success' }}
with:
name: macos-build
path: ~/Library/Caches/mullvad-test/packages
- name: Checkout repository
uses: actions/checkout@v4
- name: Run end-to-end tests
shell: bash -ieo pipefail {0}
run: |
git fetch --tags --force
./test/ci-runtests.sh ${{ matrix.os }}
- uses: actions/upload-artifact@v3
if: '!cancelled()'
with:
name: ${{ matrix.os }}_report
path: ./test/.ci-logs/${{ matrix.os }}_report
compile-test-matrix:
name: Result matrix
needs: [e2e-test-linux, e2e-test-windows, e2e-test-macos]
if: '!cancelled()'
runs-on: [self-hosted, desktop-test, Linux]
timeout-minutes: 240
strategy:
fail-fast: false
steps:
- name: Checkout repository
uses: actions/checkout@v4
- uses: actions/download-artifact@v3
with:
path: ./test/.ci-logs/artifacts
- name: Generate test result matrix
shell: bash -ieo pipefail {0}
run: |
cd test
# "Unpack" the downloaded report artifacts: https://github.com/actions/download-artifact/issues/141
cp ./.ci-logs/artifacts/*_report/*_report ./.ci-logs/
cargo run --bin test-manager format-test-reports ./.ci-logs/*_report \
| tee summary.html >> $GITHUB_STEP_SUMMARY
- uses: actions/upload-artifact@v3
with:
name: summary.html
path: test/summary.html