Introduce automated UI testing based on asciinema #35
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2023 Sebastian Pipping <[email protected]> | |
| # Licensed under Apache License Version 2.0 | |
| name: Build on Linux/macOS | |
| # Drop permissions to minimum, for security | |
| permissions: | |
| contents: read | |
| on: | |
| pull_request: | |
| push: | |
| schedule: | |
| - cron: '0 3 * * 5' # Every Friday at 3am | |
| workflow_dispatch: | |
| jobs: | |
| linux: | |
| name: Build (${{ matrix.cc }} on ${{ matrix.runs-on }}) | |
| runs-on: ${{ matrix.runs-on }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - cc: gcc-13 | |
| cxx: g++-13 | |
| clang_major_version: null | |
| clang_repo_suffix: null | |
| runs-on: ubuntu-22.04 | |
| - cc: clang-17 | |
| cxx: clang++-17 | |
| clang_major_version: 17 | |
| clang_repo_suffix: -17 | |
| runs-on: ubuntu-22.04 | |
| - cc: clang-18 | |
| cxx: clang++-18 | |
| clang_major_version: 18 | |
| clang_repo_suffix: | |
| runs-on: ubuntu-22.04 | |
| - cc: gcc-13 | |
| cxx: g++-13 | |
| clang_major_version: null | |
| clang_repo_suffix: null | |
| runs-on: macos-12 | |
| - cc: clang-15 | |
| cxx: clang++-15 | |
| clang_major_version: 15 | |
| clang_repo_suffix: null | |
| runs-on: macos-12 | |
| steps: | |
| - name: Add Clang/LLVM repositories | |
| if: "${{ runner.os == 'Linux' && contains(matrix.cxx, 'clang') }}" | |
| run: |- | |
| set -x | |
| source /etc/os-release | |
| wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - | |
| sudo add-apt-repository "deb http://apt.llvm.org/${UBUNTU_CODENAME}/ llvm-toolchain-${UBUNTU_CODENAME}${{ matrix.clang_repo_suffix }} main" | |
| - name: Install build dependencies | |
| if: "${{ runner.os == 'Linux' }}" | |
| run: |- | |
| sudo apt-get update | |
| sudo apt-get install --yes --no-install-recommends \ | |
| libncurses-dev \ | |
| pkg-config | |
| - name: Install build dependencies | |
| if: "${{ runner.os == 'macOS' }}" | |
| run: |- | |
| brew tap homebrew/cask-fonts | |
| brew install \ | |
| agg \ | |
| asciinema \ | |
| coreutils \ | |
| font-liberation \ | |
| imagemagick | |
| - name: Install build dependency Clang ${{ matrix.clang_major_version }} | |
| if: "${{ runner.os == 'Linux' && contains(matrix.cxx, 'clang') }}" | |
| run: |- | |
| sudo apt-get install --yes --no-install-recommends -V \ | |
| clang-${{ matrix.clang_major_version }} | |
| - name: Add versioned aliases for Clang ${{ matrix.clang_major_version }} | |
| if: "${{ runner.os == 'macOS' && contains(matrix.cxx, 'clang') }}" | |
| run: |- | |
| set -x | |
| sudo ln -s "$(brew --prefix llvm@${{ matrix.clang_major_version }})"/bin/clang /usr/local/bin/clang-${{ matrix.clang_major_version }} | |
| sudo ln -s "$(brew --prefix llvm@${{ matrix.clang_major_version }})"/bin/clang++ /usr/local/bin/clang++-${{ matrix.clang_major_version }} | |
| - name: Checkout Git branch | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - name: 'Build' | |
| env: | |
| CC: ${{ matrix.cc }} | |
| CXX: ${{ matrix.cxx }} | |
| run: |- | |
| CFLAGS='-std=gnu99 -pedantic -Werror' make all torture | |
| - name: 'Install' | |
| run: |- | |
| set -x -o pipefail | |
| make install DESTDIR="${PWD}"/ROOT/ | |
| find ROOT/ | sort | xargs ls -ld | |
| - name: 'Uninstall' | |
| run: |- | |
| set -x | |
| make uninstall DESTDIR="${PWD}"/ROOT/ | |
| [[ "$(find ROOT/ -not -type d | tee /dev/stderr)" == '' ]] # i.e. fail CI if leftover files | |
| - name: 'Run UI tests' | |
| if: "${{ runner.os == 'macOS' }}" | |
| run: |- | |
| ./recordings/record.sh | |
| - name: 'Upload UI test renderings for inspection' | |
| if: "${{ runner.os == 'macOS' }}" | |
| uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 | |
| with: | |
| name: ttyplot_ui_test_${{ github.sha }}_${{ matrix.runs-on }}_${{ matrix.cc }} | |
| path: recordings/actual* | |
| if-no-files-found: error | |
| - name: 'Evaluate UI test results' | |
| if: "${{ runner.os == 'macOS' }}" | |
| run: |- | |
| assert_images_equal_enough() { | |
| local a="${1}" | |
| local b="${2}" | |
| local diff_output="${3}" | |
| local dissimilarity="$(compare -metric DSSIM "${a}" "${b}" "${diff_output}" 2>&1)" | |
| if ! python3 <<<"import sys; sys.exit(int(${dissimilarity} > 0.0022))"; then | |
| echo "Image \"${a}\" is not close enough of a match to image \"${b}\", dissimilarity is ${dissimilarity}." >&2 | |
| return 1 | |
| fi | |
| true | |
| } | |
| cd recordings/ | |
| error=0 | |
| for expected in expected*.png; do | |
| actual=${expected/expected/actual} | |
| diff=${expected/expected/diff} | |
| assert_images_equal_enough ${actual} ${expected} ${diff} || error=1 | |
| done | |
| rm -f actual*.* diff*.* | |
| cat <<"EOF" | |
| ############################################################ | |
| ## If this step FAILS you can get the expected images | |
| ## back in sync with reality by running: | |
| ## | |
| ## $ ./recordings/get_back_in_sync.sh | |
| ############################################################ | |
| EOF | |
| exit ${error} | |
| - name: 'Clean' | |
| run: |- | |
| set -x -o pipefail | |
| make clean | |
| [[ "$(git ls-files -o | tee /dev/stderr | wc -l)" -eq 0 ]] |