[AP] Testing Infrastructure #9776
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: | |
| # We want to run the CI when anything is pushed to master. | |
| # Since master is a protected branch this only happens when a PR is merged. | |
| # This is a double check in case the PR was stale and had some issues. | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: # Prevents from running if only docs are updated | |
| - 'doc/**' | |
| - '**/*README*' | |
| - '**.md' | |
| - '**.rst' | |
| pull_request: | |
| paths-ignore: # Prevents from running if only docs are updated | |
| - 'doc/**' | |
| - '**/*README*' | |
| - '**.md' | |
| - '**.rst' | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0 * * *' # daily | |
| # We want to cancel previous runs for a given PR or branch / ref if another CI | |
| # run is requested. | |
| # See: https://docs.github.com/en/actions/using-jobs/using-concurrency | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| # default compiler for all non-compatibility tests | |
| MATRIX_EVAL: "CC=gcc-11 && CXX=g++-11" | |
| jobs: | |
| Build: | |
| name: 'B: Building VtR' | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - { build_type: 'release', verbose: '0' } | |
| - { build_type: 'debug', verbose: '0' } | |
| - { build_type: 'debug', verbose: '1' } | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - uses: hendrikmuhs/[email protected] | |
| - name: Test | |
| env: | |
| BUILD_TYPE: ${{ matrix.build_type }} | |
| NUM_PROC: ${{ steps.cpu-cores.outputs.count }} | |
| run: | | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| ./.github/scripts/build.sh VERBOSE=${{ matrix.verbose }} | |
| Format: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - { name: 'C/C++', script: 'check-format.sh' } | |
| - { name: 'Python', script: 'check-format-py.sh' } | |
| - { name: 'Python Lint', script: 'pylint_check.py' } | |
| name: 'F: ${{ matrix.name }}' | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - name: Test | |
| run: ./dev/${{ matrix.script }} | |
| UniTests: | |
| name: 'U: C++ Unit Tests' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - name: Test | |
| env: | |
| CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on" | |
| NUM_PROC: ${{ steps.cpu-cores.outputs.count }} | |
| run: ./.github/scripts/unittest.sh | |
| Warnings: | |
| name: 'W: Check Compilation Warnings' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - uses: hendrikmuhs/[email protected] | |
| - name: Test | |
| env: | |
| #In order to get compilation warnings produced per source file, we must do a non-IPO build | |
| #We also turn warnings into errors for this target by doing a strict compile | |
| CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_STRICT_COMPILE=on -DVTR_IPO_BUILD=off" | |
| NUM_PROC: ${{ steps.cpu-cores.outputs.count }} | |
| run: | | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| ./.github/scripts/build.sh | |
| Regression: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: [ | |
| { | |
| name: 'Basic', | |
| params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on', | |
| suite: 'vtr_reg_basic', | |
| extra_pkgs: "" | |
| }, | |
| { | |
| name: 'Basic with highest assertion level', | |
| params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=4 -DWITH_BLIFEXPLORER=on', | |
| suite: 'vtr_reg_basic', | |
| extra_pkgs: "" | |
| }, | |
| { | |
| name: 'Basic_odin', | |
| params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on', | |
| suite: 'vtr_reg_basic_odin', | |
| extra_pkgs: "" | |
| }, | |
| { | |
| name: 'Basic with NO_GRAPHICS', | |
| params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVPR_USE_EZGL=off', | |
| suite: 'vtr_reg_basic', | |
| extra_pkgs: "" | |
| }, | |
| { | |
| name: 'Basic with NO_SERVER', | |
| params: '-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVPR_USE_EZGL=on -DVPR_USE_SERVER=off', | |
| suite: 'vtr_reg_basic', | |
| extra_pkgs: "" | |
| }, | |
| { | |
| name: 'Basic with CAPNPROTO disabled', | |
| params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_CAPNPROTO=off', | |
| suite: 'vtr_reg_basic', | |
| extra_pkgs: "" | |
| }, | |
| { | |
| name: 'Basic with VTR_ENABLE_DEBUG_LOGGING', | |
| params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_DEBUG_LOGGING=on', | |
| suite: 'vtr_reg_basic', | |
| extra_pkgs: "" | |
| }, | |
| { | |
| name: 'Basic_odin with VTR_ENABLE_DEBUG_LOGGING', | |
| params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DVTR_ENABLE_DEBUG_LOGGING=on -DWITH_PARMYS=OFF -DWITH_ODIN=on', | |
| suite: 'vtr_reg_basic_odin', | |
| extra_pkgs: "" | |
| }, | |
| { | |
| name: 'Strong', | |
| params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on', | |
| suite: 'vtr_reg_strong', | |
| extra_pkgs: "libeigen3-dev" | |
| }, | |
| { | |
| name: 'Strong_odin', | |
| params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on', | |
| suite: 'vtr_reg_strong_odin', | |
| extra_pkgs: "" | |
| }, | |
| { | |
| name: 'Valgrind Memory', | |
| params: '-DCMAKE_COMPILE_WARNING_AS_ERROR=on -DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_ODIN=on', | |
| suite: 'vtr_reg_valgrind_small', | |
| extra_pkgs: "" | |
| } | |
| ] | |
| name: 'R: ${{ matrix.name }}' | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - name: Install external libraries | |
| run: sudo apt install -y ${{ matrix.extra_pkgs }} | |
| if: ${{ matrix.extra_pkgs }} | |
| - uses: hendrikmuhs/[email protected] | |
| - name: Test | |
| env: | |
| CMAKE_PARAMS: ${{ matrix.params }} | |
| NUM_PROC: ${{ steps.cpu-cores.outputs.count }} | |
| run: | | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| ./.github/scripts/build.sh | |
| ./run_reg_test.py ${{ matrix.suite }} -show_failures -j${{ steps.cpu-cores.outputs.count}} | |
| - name: Upload regression run files | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{matrix.name}}_run_files | |
| path: | | |
| vtr_flow/**/*.out | |
| # vtr_flow/**/*.blif # Removed since it was taking too much space and was hardly used. | |
| vtr_flow/**/*.p | |
| vtr_flow/**/*.net | |
| vtr_flow/**/*.r | |
| - name: Upload regression results | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{matrix.name}}_results | |
| path: | | |
| vtr_flow/**/*.log | |
| vtr_flow/**/parse_results*.txt | |
| Sanitized: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: [ | |
| { | |
| name: 'Basic', | |
| params: '-DVTR_ASSERT_LEVEL=3 -DVTR_ENABLE_SANITIZE=on -DVTR_IPO_BUILD=off -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on', | |
| suite: 'vtr_reg_basic_odin' | |
| } | |
| #- { name: 'Strong', suite: 'vtr_reg_strong' } # SKIP Too long to run on GitHub Actions (max 6h) | |
| ] | |
| name: 'S: ${{ matrix.name }}' | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - uses: hendrikmuhs/[email protected] | |
| - name: Test | |
| env: | |
| CMAKE_PARAMS: ${{ matrix.params }} | |
| BUILD_TYPE: debug | |
| LSAN_OPTIONS: 'exitcode=42' #Use a non-standard exit code to ensure LSAN errors are detected | |
| # In Ubuntu 20240310.1.0, the entropy of ASLR has increased (28 -> 32). LLVM 14 in this | |
| # image is not compatible with this increased ASLR entropy. Apparently, memory sanitizer | |
| # depends on LLVM and all CI tests where VTR_ENABLE_SANITIZE is enabled fail. For a temporary | |
| # fix, we manually reduce the entropy. This quick fix should be removed in the future | |
| # when github deploys a more stable Ubuntu image. | |
| NUM_PROC: ${{ steps.cpu-cores.outputs.count }} | |
| run: | | |
| sudo sysctl -w vm.mmap_rnd_bits=28 | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| ./.github/scripts/build.sh | |
| # We skip QoR since we are only checking for errors in sanitizer runs | |
| ./run_reg_test.py ${{ matrix.suite }} -show_failures -j${{ steps.cpu-cores.outputs.count }} -skip_qor | |
| Parmys: | |
| name: 'Parmys Basic Test' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - uses: hendrikmuhs/[email protected] | |
| - name: Test | |
| env: | |
| CMAKE_PARAMS: '-DVTR_IPO_BUILD=off' | |
| BUILD_TYPE: debug | |
| NUM_PROC: ${{ steps.cpu-cores.outputs.count }} | |
| run: | | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| ./.github/scripts/build.sh | |
| ./run_reg_test.py parmys_reg_basic -show_failures -j${{ steps.cpu-cores.outputs.count }} | |
| ODINII: | |
| name: 'ODIN-II Basic Test' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - uses: hendrikmuhs/[email protected] | |
| - name: Test | |
| env: | |
| CMAKE_PARAMS: '-DVTR_ASSERT_LEVEL=3 -DVTR_ENABLE_SANITIZE=on -DVTR_IPO_BUILD=off -DWITH_BLIFEXPLORER=on -DWITH_PARMYS=OFF -DWITH_ODIN=on' | |
| BUILD_TYPE: debug | |
| NUM_PROC: ${{ steps.cpu-cores.outputs.count }} | |
| run: | | |
| sudo sysctl -w vm.mmap_rnd_bits=28 | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| ./.github/scripts/build.sh | |
| ./run_reg_test.py odin_reg_basic -show_failures -j${{ steps.cpu-cores.outputs.count }} | |
| VQM2BLIF: | |
| name: 'VQM2BLIF Basic Tests' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - uses: hendrikmuhs/[email protected] | |
| - name: Test | |
| env: | |
| BUILD_TYPE: release | |
| NUM_PROC: ${{ steps.cpu-cores.outputs.count }} | |
| run: | | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| ./.github/scripts/build.sh | |
| ./utils/vqm2blif/test/scripts/test_vqm2blif.sh | |
| Compatibility: | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - { name: 'GCC 9 (Ubuntu Jammy - 22.04)', eval: 'CC=gcc-9 && CXX=g++-9', } | |
| - { name: 'GCC 10 (Ubuntu Jammy - 22.04)', eval: 'CC=gcc-10 && CXX=g++-10', } | |
| - { name: 'GCC 11 (Ubuntu Jammy - 22.04)', eval: 'CC=gcc-11 && CXX=g++-11', } | |
| - { name: 'GCC 12 (Ubuntu Jammy - 22.04)', eval: 'CC=gcc-12 && CXX=g++-12', } | |
| - { name: 'Clang 11 (Ubuntu Jammy - 22.04)', eval: 'CC=clang-11 && CXX=clang++-11', } | |
| - { name: 'Clang 12 (Ubuntu Jammy - 22.04)', eval: 'CC=clang-12 && CXX=clang++-12', } | |
| - { name: 'Clang 13 (Ubuntu Jammy - 22.04)', eval: 'CC=clang-13 && CXX=clang++-13', } | |
| - { name: 'Clang 14 (Ubuntu Jammy - 22.04)', eval: 'CC=clang-14 && CXX=clang++-14', } | |
| name: 'B: ${{ matrix.name }}' | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - uses: hendrikmuhs/[email protected] | |
| - name: Test | |
| env: | |
| CMAKE_PARAMS: "-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on -DWITH_ODIN=on" | |
| MATRIX_EVAL: ${{ matrix.eval }} | |
| BUILD_TYPE: release | |
| NUM_PROC: ${{ steps.cpu-cores.outputs.count }} | |
| run: | | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| ./.github/scripts/build.sh | |
| Coverity: | |
| name: 'Coverity Scan' | |
| needs: | |
| - Build | |
| - Format | |
| - UniTests | |
| - Warnings | |
| - Regression | |
| - Sanitized | |
| - Parmys | |
| - ODINII | |
| - VQM2BLIF | |
| - Compatibility | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.10.10 | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: 'true' | |
| - name: Get number of CPU cores | |
| uses: SimenB/github-actions-cpu-cores@v2 | |
| id: cpu-cores | |
| - name: Install dependencies | |
| run: ./.github/scripts/install_dependencies.sh | |
| - uses: hendrikmuhs/[email protected] | |
| - name: Test | |
| env: | |
| CMAKE_PARAMS: '-DVTR_ASSERT_LEVEL=3 -DWITH_BLIFEXPLORER=on' | |
| NUM_PROC: ${{ steps.cpu-cores.outputs.count }} | |
| _COVERITY_URL: 'https://scan.coverity.com/download/linux64' | |
| _COVERITY_MD5: 'd0d7d7df9d6609e578f85096a755fb8f' | |
| run: | | |
| export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" | |
| ./.github/scripts/build.sh |