Desktop - End-to-end tests #433
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Workflow for triggering `test-manager` on select platforms. | |
| # | |
| # This is a rather complex workflow. The complexity mainly stems from these sources: | |
| # * figuring out which platforms to test on which runners (prepare-matrices) | |
| # * figuring out if the app and e2e-tests should be built on the runner (build-{linux,windows,macos}) | |
| # or if we should download the artifacts from https://releases.mullvad.net/desktop/ | |
| # * compiling the output from the different runners and executed platforms. | |
| --- | |
| name: Desktop - End-to-end tests | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| oses: | |
| description: "Space-delimited list of targets to run tests on, e.g. `debian11 debian12`. \ | |
| Available images are:\n | |
| `debian11 debian12 ubuntu2004 ubuntu2204 ubuntu2304 ubuntu2310 fedora39 \ | |
| fedora38 fedora37 fedora36 windows10 windows11 macos12 macos13 macos14`." | |
| default: '' | |
| required: false | |
| type: string | |
| jobs: | |
| prepare-matrices: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Generate matrix for Linux builds | |
| shell: bash | |
| run: | | |
| # A list of VMs to run the tests on. These refer to the names defined | |
| # in $XDG_CONFIG_DIR/mullvad-test/config.json on the runner. | |
| all='["debian11","debian12","ubuntu2004","ubuntu2204","ubuntu2304","ubuntu2310","fedora39","fedora38","fedora37","fedora36"]' | |
| oses="${{ github.event.inputs.oses }}" | |
| echo "OSES: $oses" | |
| if [[ -z "$oses" || "$oses" == "null" ]]; then | |
| selected="$all" | |
| else | |
| oses=$(printf '%s\n' $oses | jq . -R | jq . -s) | |
| selected=$(jq -cn --argjson oses "$oses" --argjson all "$all" '$all - ($all - $oses)') | |
| fi | |
| echo "Selected targets: $selected" | |
| echo "linux_matrix=$selected" >> $GITHUB_ENV | |
| - name: Generate matrix for Windows builds | |
| shell: bash | |
| run: | | |
| all='["windows10","windows11"]' | |
| oses="${{ github.event.inputs.oses }}" | |
| if [[ -z "$oses" || "$oses" == "null" ]]; then | |
| selected="$all" | |
| else | |
| oses=$(printf '%s\n' $oses | jq . -R | jq . -s) | |
| selected=$(jq -cn --argjson oses "$oses" --argjson all "$all" '$all - ($all - $oses)') | |
| fi | |
| echo "Selected targets: $selected" | |
| echo "windows_matrix=$selected" >> $GITHUB_ENV | |
| - name: Generate matrix for macOS builds | |
| shell: bash | |
| run: | | |
| all='["macos12","macos13","macos14"]' | |
| oses="${{ github.event.inputs.oses }}" | |
| if [[ -z "$oses" || "$oses" == "null" ]]; then | |
| selected="$all" | |
| else | |
| oses=$(printf '%s\n' $oses | jq . -R | jq . -s) | |
| selected=$(jq -cn --argjson oses "$oses" --argjson all "$all" '$all - ($all - $oses)') | |
| fi | |
| echo "Selected targets: $selected" | |
| echo "macos_matrix=$selected" >> $GITHUB_ENV | |
| outputs: | |
| linux_matrix: ${{ env.linux_matrix }} | |
| windows_matrix: ${{ env.windows_matrix }} | |
| macos_matrix: ${{ env.macos_matrix }} | |
| prepare-linux: | |
| needs: prepare-matrices | |
| if: | | |
| needs.prepare-matrices.outputs.linux_matrix != '[]' && | |
| !startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Use custom container image if specified | |
| if: ${{ github.event.inputs.override_container_image != '' }} | |
| run: echo "inner_container_image=${{ github.event.inputs.override_container_image }}" | |
| >> $GITHUB_ENV | |
| - name: Use default container image and resolve digest | |
| if: ${{ github.event.inputs.override_container_image == '' }} | |
| run: | | |
| echo "inner_container_image=$(cat ./building/linux-container-image.txt)" >> $GITHUB_ENV | |
| outputs: | |
| container_image: ${{ env.inner_container_image }} | |
| build-linux: | |
| needs: prepare-linux | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ${{ needs.prepare-linux.outputs.container_image }} | |
| continue-on-error: true | |
| steps: | |
| # Fix for HOME path overridden by GH runners when building in containers, see: | |
| # https://github.com/actions/runner/issues/863 | |
| - name: Fix HOME path | |
| run: echo "HOME=/root" >> $GITHUB_ENV | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Checkout binaries submodule | |
| run: | | |
| git config --global --add safe.directory '*' | |
| git submodule update --init --depth=1 dist-assets/binaries | |
| - name: Build app | |
| env: | |
| USE_MOLD: false | |
| run: ./build.sh | |
| - name: Build test executable | |
| run: ./gui/scripts/build-test-executable.sh | |
| - uses: actions/upload-artifact@v3 | |
| if: '!cancelled()' | |
| with: | |
| name: linux-build | |
| path: | | |
| ./dist/*.rpm | |
| ./dist/*.deb | |
| ./dist/app-e2e-* | |
| e2e-test-linux: | |
| name: Linux end-to-end tests | |
| needs: [prepare-matrices, build-linux] | |
| if: '!cancelled()' | |
| runs-on: [self-hosted, desktop-test, Linux] # app-test-linux | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ${{ fromJSON(needs.prepare-matrices.outputs.linux_matrix) }} | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| if: ${{ needs.build-linux.result == 'success' }} | |
| with: | |
| name: linux-build | |
| path: ~/.cache/mullvad-test/packages | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Run end-to-end tests | |
| shell: bash -ieo pipefail {0} | |
| run: | | |
| git fetch --tags --force | |
| ./test/ci-runtests.sh ${{ matrix.os }} | |
| - uses: actions/upload-artifact@v3 | |
| if: '!cancelled()' | |
| with: | |
| name: ${{ matrix.os }}_report | |
| path: ./test/.ci-logs/${{ matrix.os }}_report | |
| build-windows: | |
| needs: prepare-matrices | |
| if: | | |
| needs.prepare-matrices.outputs.windows_matrix != '[]' && | |
| !startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main' | |
| runs-on: windows-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Checkout submodules | |
| run: git submodule update --init --depth=1 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/setup-node@v3 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| node-version: 18 | |
| - name: Install Rust | |
| uses: actions-rs/[email protected] | |
| with: | |
| toolchain: stable | |
| target: i686-pc-windows-msvc | |
| default: true | |
| - name: Install msbuild | |
| uses: microsoft/[email protected] | |
| with: | |
| vs-version: 16 | |
| - name: Build app | |
| shell: bash | |
| # TODO: For some reason, 'npm run pack:win' wants to publish the artifacts | |
| continue-on-error: true | |
| run: ./build.sh | |
| - name: Build test executable | |
| shell: bash | |
| run: ./gui/scripts/build-test-executable.sh | |
| - uses: actions/upload-artifact@v3 | |
| if: '!cancelled()' | |
| with: | |
| name: windows-build | |
| path: .\dist\*.exe | |
| e2e-test-windows: | |
| needs: [prepare-matrices, build-windows] | |
| if: '!cancelled()' | |
| name: Windows end-to-end tests | |
| runs-on: [self-hosted, desktop-test, Linux] # app-test-linux | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ${{ fromJSON(needs.prepare-matrices.outputs.windows_matrix) }} | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| if: ${{ needs.build-windows.result == 'success' }} | |
| with: | |
| name: windows-build | |
| path: ~/.cache/mullvad-test/packages | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Run end-to-end tests | |
| shell: bash -ieo pipefail {0} | |
| run: | | |
| git fetch --tags --force | |
| ./test/ci-runtests.sh ${{ matrix.os }} | |
| - uses: actions/upload-artifact@v3 | |
| if: '!cancelled()' | |
| with: | |
| name: ${{ matrix.os }}_report | |
| path: ./test/.ci-logs/${{ matrix.os }}_report | |
| build-macos: | |
| needs: prepare-matrices | |
| if: | | |
| needs.prepare-matrices.outputs.macos_matrix != '[]' && | |
| !startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main' | |
| runs-on: [self-hosted, desktop-test, macOS] # app-test-macos-arm | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Checkout submodules | |
| run: git submodule update --init --depth=1 | |
| - name: Install Go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: 1.21.3 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v2 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions/setup-node@v3 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| node-version: 18 | |
| cache: 'npm' | |
| cache-dependency-path: gui/package-lock.json | |
| - name: Build app | |
| run: ./build.sh | |
| - name: Build test executable | |
| run: ./gui/scripts/build-test-executable.sh | |
| - uses: actions/upload-artifact@v3 | |
| if: '!cancelled()' | |
| with: | |
| name: macos-build | |
| path: | | |
| ./dist/*.pkg | |
| ./dist/app-e2e-* | |
| e2e-test-macos: | |
| needs: [prepare-matrices, build-macos] | |
| if: '!cancelled()' | |
| name: macOS end-to-end tests | |
| runs-on: [self-hosted, desktop-test, macOS] # app-test-macos-arm | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ${{ fromJSON(needs.prepare-matrices.outputs.macos_matrix) }} | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| if: ${{ needs.build-macos.result == 'success' }} | |
| with: | |
| name: macos-build | |
| path: ~/Library/Caches/mullvad-test/packages | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Run end-to-end tests | |
| shell: bash -ieo pipefail {0} | |
| run: | | |
| git fetch --tags --force | |
| ./test/ci-runtests.sh ${{ matrix.os }} | |
| - uses: actions/upload-artifact@v3 | |
| if: '!cancelled()' | |
| with: | |
| name: ${{ matrix.os }}_report | |
| path: ./test/.ci-logs/${{ matrix.os }}_report | |
| compile-test-matrix: | |
| name: Result matrix | |
| needs: [e2e-test-linux, e2e-test-windows, e2e-test-macos] | |
| if: '!cancelled()' | |
| runs-on: [self-hosted, desktop-test, Linux] | |
| timeout-minutes: 240 | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| path: ./test/.ci-logs/artifacts | |
| - name: Generate test result matrix | |
| shell: bash -ieo pipefail {0} | |
| run: | | |
| cd test | |
| # "Unpack" the downloaded report artifacts: https://github.com/actions/download-artifact/issues/141 | |
| cp ./.ci-logs/artifacts/*_report/*_report ./.ci-logs/ | |
| cargo run --bin test-manager format-test-reports ./.ci-logs/*_report \ | |
| | tee summary.html >> $GITHUB_STEP_SUMMARY | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: summary.html | |
| path: test/summary.html |