ci: combine build and test jobs to one monolithic job matrix #4
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| id: | ||
| description: 'ID' | ||
| required: true | ||
| type: string | ||
| runner: | ||
| description: 'GitHub runner' | ||
| required: true | ||
| type: string | ||
| container: | ||
| description: 'Docker container' | ||
| required: false | ||
| type: string | ||
| default: '' | ||
| verset: | ||
| description: 'Dependency version set' | ||
| required: false | ||
| type: string | ||
| default: 'latest' | ||
| defaults: | ||
| run: | ||
| shell: bash | ||
| env: | ||
| hipo_fork: c-dilks/hipo | ||
| hipo_ref: 4c8325bcf289c5da84f20e6e2b84f9b9ad121447 # for libHipoDataFrame in pkg-config | ||
| # test options | ||
| num_events: 1000 | ||
| verbose_test: 'false' # only set this to 'true' if `meson test` fails and you need more output to investigate | ||
| jobs: | ||
| # download test data | ||
| ######################################################### | ||
| download_test_data: | ||
| runs-on: ubuntu-latest | ||
| env: | ||
| type: physics | ||
| steps: | ||
| - uses: actions/cache@v4 | ||
| id: cache | ||
| with: | ||
| key: test_data | ||
| path: test_data.hipo | ||
| lookup-only: true | ||
| - name: download | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: | | ||
| wget -nv --no-check-certificate http://clasweb.jlab.org/clas12offline/distribution/clas12-timeline/validation_files_${{ env.type }}.tar.zst | ||
| tar xvf validation_files_${{ env.type }}.tar.zst | ||
| mv -v $(find validation_files -type f -name "*.hipo" | head -n1) test_data.hipo | ||
| # dependencies | ||
| ######################################################### | ||
| build_root: | ||
| name: Build ROOT | ||
| runs-on: ${{ inputs.runner }} | ||
| container: | ||
| image: ${{ inputs.container }} | ||
| outputs: | ||
| key: ${{ steps.key.outputs.key }} | ||
| steps: | ||
| - name: checkout iguana | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: iguana_src | ||
| - name: key | ||
| id: key | ||
| run: | | ||
| root_version=$(iguana_src/meson/minimum-version.sh root src | sed 's;.*/;;' | sed 's;\.tar\.gz;;') | ||
| echo key=root---${{ inputs.id }}---${root_version}---$(date +%Y-week%U) >> $GITHUB_OUTPUT | ||
| - uses: actions/cache/restore@v4 | ||
| id: cache | ||
| with: | ||
| key: ${{ steps.key.outputs.key }} | ||
| path: root.tar.zst | ||
| lookup-only: true | ||
| - name: install dependencies | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} | ||
| - name: download ROOT source code | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: | | ||
| srcURL=$(iguana_src/meson/minimum-version.sh root src) | ||
| echo "[+] DOWNLOADING ROOT SOURCE CODE FROM: $srcURL" | ||
| wget -nv --no-check-certificate $srcURL | ||
| tar xf $(ls -t *.gz | head -n1) | ||
| ls -t | ||
| mv -v $(ls -td root-* | head -n1) root_src | ||
| - name: build ROOT | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: | | ||
| cmake -S root_src -B build -G Ninja --install-prefix $(pwd)/root -DCMAKE_CXX_STANDARD=17 | ||
| cmake --build build | ||
| cmake --install build | ||
| tar caf root{.tar.zst,} | ||
| - uses: actions/cache/save@v4 | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| id: cache_save | ||
| with: | ||
| key: ${{ steps.key.outputs.key }} | ||
| path: root.tar.zst | ||
| build_hipo: | ||
| name: Build HIPO | ||
| needs: | ||
| - build_root # for dataframe support | ||
| runs-on: ${{ inputs.runner }} | ||
| container: | ||
| image: ${{ inputs.container }} | ||
| strategy: | ||
| fail-fast: true | ||
| matrix: | ||
| root_dep: | ||
| - noROOT # exclude dataframe lib, which depends on ROOT | ||
| - withROOT | ||
| outputs: | ||
| key: ${{ steps.key.outputs.key }} | ||
| steps: | ||
| - name: get ROOT build | ||
| if: ${{ matrix.root_dep == 'withROOT' }} | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| key: ${{ needs.build_root.outputs.key }} | ||
| path: root.tar.zst | ||
| - name: untar ROOT build | ||
| if: ${{ matrix.root_dep == 'withROOT' }} | ||
| run: ls *.tar.zst | xargs -I{} tar xvf {} | ||
| - name: key | ||
| id: key | ||
| run: echo key=hipo---${{ inputs.id }}---${{ env.hipo_ref }}---$(date +%Y-week%U) >> $GITHUB_OUTPUT | ||
| - uses: actions/cache/restore@v4 | ||
| id: cache | ||
| with: | ||
| key: ${{ steps.key.outputs.key }}---${{ matrix.root_dep }} | ||
| path: hipo.tar.zst | ||
| lookup-only: true | ||
| - name: checkout iguana for dependency installation script | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: iguana_src | ||
| - name: checkout hipo | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| repository: ${{ env.hipo_fork }} | ||
| ref: ${{ env.hipo_ref }} | ||
| path: hipo_src | ||
| - name: build | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: | | ||
| iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} | ||
| [ "${{ matrix.root_dep }}" = "withROOT" ] && source root/bin/thisroot.sh | ||
| cmake -S hipo_src -B build -G Ninja --install-prefix $(pwd)/hipo -DCMAKE_POSITION_INDEPENDENT_CODE=ON # using PIE build, for sanitizer readibility | ||
| cmake --build build | ||
| cmake --install build | ||
| tar cavf hipo{.tar.zst,} | ||
| - uses: actions/cache/save@v4 | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| id: cache_save | ||
| with: | ||
| key: ${{ steps.key.outputs.key }}---${{ matrix.root_dep }} | ||
| path: hipo.tar.zst | ||
| # build and test Iguana | ||
| ######################################################### | ||
| iguana: | ||
| name: Iguana | ||
| needs: | ||
| - build_hipo | ||
| - build_root | ||
| runs-on: ${{ inputs.runner }} | ||
| container: | ||
| image: ${{ inputs.container }} | ||
| strategy: | ||
| fail-fast: true | ||
| matrix: | ||
| id: | ||
| - cpp | ||
| - address-sanitizer | ||
| - thread-sanitizer | ||
| - undefined-sanitizer | ||
| - leak-sanitizer | ||
| - noROOT | ||
| - python | ||
| include: | ||
| - { id: cpp, CC: gcc, CXX: g++, opts: '-Dbuildtype=release -Drequire_ROOT=true -Db_coverage=true' } | ||
| - { id: address-sanitizer, CC: clang, CXX: clang++, opts: '-Dbuildtype=debug -Drequire_ROOT=true -Db_sanitize=address -Db_lundef=false -Db_pie=true' } | ||
| - { id: thread-sanitizer, CC: clang, CXX: clang++, opts: '-Dbuildtype=debug -Drequire_ROOT=true -Db_sanitize=thread -Db_lundef=false -Db_pie=true' } | ||
| - { id: undefined-sanitizer, CC: clang, CXX: clang++, opts: '-Dbuildtype=debug -Drequire_ROOT=true -Db_sanitize=undefined -Db_lundef=false -Db_pie=true' } | ||
| - { id: leak-sanitizer, CC: clang, CXX: clang++, opts: '-Dbuildtype=debug -Drequire_ROOT=true -Db_sanitize=leak -Db_lundef=false -Db_pie=true' } | ||
| - { id: noROOT, CC: gcc, CXX: g++, opts: '-Dbuildtype=release -Drequire_ROOT=false' } | ||
| - { id: python, CC: gcc, CXX: g++, opts: '-Dbuildtype=release -Drequire_ROOT=true -Dbind_python=true' } | ||
| env: | ||
| CC: ${{ matrix.CC }} | ||
| CXX: ${{ matrix.CXX }} | ||
| steps: | ||
| ### setup | ||
| - uses: actions/checkout@v4 | ||
| with: # settings needed for version number detection | ||
| clean: false | ||
| fetch-tags: true | ||
| fetch-depth: 0 | ||
| path: iguana_src | ||
| ### get test data | ||
| - name: get test data | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| key: test_data | ||
| path: test_data.hipo | ||
| ### dependencies | ||
| - name: install dependency packages | ||
| run: iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} | ||
| - name: get `hipo-withROOT` build | ||
| if: ${{ matrix.id != 'noROOT' }} | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| key: ${{ steps.hipo_key.output.hipo_key }}---withROOT | ||
| path: hipo.tar.zst | ||
| - name: get `hipo-noROOT` build | ||
| if: ${{ matrix.id == 'noROOT' }} | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| key: ${{ steps.hipo_key.output.hipo_key }}---noROOT | ||
| path: hipo.tar.zst | ||
| - name: get `ROOT` build | ||
| if: ${{ matrix.id != 'noROOT' }} | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| key: ${{ needs.build_root.outputs.key }} | ||
| path: root.tar.zst | ||
| - name: untar dependency builds | ||
| run: ls *.tar.zst | xargs -I{} tar xvf {} | ||
| - name: tree local dependencies | ||
| run: tree hipo | ||
| - name: summarize dependencies | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| run: | | ||
| echo '### Dependencies' >> $GITHUB_STEP_SUMMARY | ||
| echo '| Dependency | Version |' >> $GITHUB_STEP_SUMMARY | ||
| echo '| --- | --- |' >> $GITHUB_STEP_SUMMARY | ||
| echo "| \`hipo\` | ${{ env.hipo_ref }} |" >> $GITHUB_STEP_SUMMARY | ||
| cat pkg_summary.md >> $GITHUB_STEP_SUMMARY | ||
| ### build | ||
| - name: set ROOT environment | ||
| if: ${{ matrix.id != 'noROOT' }} | ||
| run: iguana_src/.github/source-ROOT.sh root | ||
| - name: meson setup | ||
| run: | | ||
| meson setup iguana_build iguana_src \ | ||
| --prefix=$(pwd)/iguana \ | ||
| --cmake-prefix-path=$(pwd)/root \ | ||
| --pkg-config-path=$(pwd)/hipo/lib/pkgconfig \ | ||
| -Dtest_data_file=$(pwd)/test_data.hipo \ | ||
| -Dtest_num_events=${{ env.num_events }} \ | ||
| -Dtest_output_dir=$(pwd)/validation_results \ | ||
| ${{ matrix.opts }} | ||
| - name: dump build options | ||
| run: meson configure iguana_build | cat | ||
| - run: meson compile | ||
| working-directory: iguana_build | ||
| - run: meson install | ||
| working-directory: iguana_build | ||
| ### dump info about this build | ||
| - name: dump build log | ||
| if: always() | ||
| run: cat iguana_build/meson-logs/meson-log.txt | ||
| - name: dump rpaths | ||
| run: iguana_src/.github/rpath-dump.sh iguana ${{ inputs.runner }} | ||
| - name: cat pkg-config pc files | ||
| run: | | ||
| pcfiles=$(find iguana -type f -name "*.pc") | ||
| for pcfile in $pcfiles; do | ||
| echo "[+++] cat $pcfile" | ||
| cat $pcfile | ||
| done | ||
| - run: tree iguana installation prefix | ||
| ### run tests | ||
| - name: meson test | ||
| working-directory: iguana_build | ||
| run: | | ||
| if [ "${{ env.verbose_test }}" = "true" ]; then | ||
| [ ${{ inputs.runner }} = "macos-latest" ] && stdbuf_cmd=gstdbuf || stdbuf_cmd=stdbuf | ||
| $stdbuf_cmd -o0 meson test --print-errorlogs --verbose --no-stdsplit | ||
| else | ||
| meson test --print-errorlogs | ||
| fi | ||
| ### coverage | ||
| - name: coverage | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| run: | | ||
| ninja -C iguana_build coverage | ||
| mv iguana_build/meson-logs/coveragereport coverage-report | ||
| echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| echo '' >> $GITHUB_STEP_SUMMARY | ||
| echo '- for details, see the `coverage-report` artifact' >> $GITHUB_STEP_SUMMARY | ||
| echo '- to compare to the report from the `main` branch, see <https://jeffersonlab.github.io/iguana/coverage-report>' >> $GITHUB_STEP_SUMMARY | ||
| ### set iguana environment, since the next steps will check the iguana installation | ||
| - name: set iguana environment | ||
| run: source iguana/bin/this_iguana.sh --verbose --githubCI | ||
| ### test installed examples | ||
| ###### cpp | ||
| - run: iguana-example-00-basic test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| - run: iguana-example-01-bank-rows test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| - run: iguana-example-02-dataframes test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| - run: iguana-example-03-config-files iguana/etc/iguana/examples | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| ###### python | ||
| - run: iguana-example-00-basic.py test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'python' }} | ||
| - run: iguana-example-01-bank-rows.py test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'python' }} | ||
| ### test consumers | ||
| - name: consumer test: make | ||
| run: iguana_src/.github/test-consumer-build.sh make | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| - name: consumer test: meson | ||
| run: iguana_src/.github/test-consumer-build.sh meson | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| - name: consumer test: cmake | ||
| run: CMAKE_PREFIX_PATH=$(pwd)/hipo iguana_src/.github/test-consumer-build.sh cmake | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| ### upload artifacts | ||
| - uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: logs_iguana_build_${{ matrix.id }} | ||
| retention-days: 5 | ||
| path: iguana_build/meson-logs | ||
| - uses: actions/upload-artifact@v4 | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| with: | ||
| name: coverage-report | ||
| retention-days: 5 | ||
| path: coverage-report | ||
| - uses: actions/upload-artifact@v4 | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| with: | ||
| name: _validation_results | ||
| retention-days: 5 | ||
| path: validation_results | ||
| # documentation | ||
| ######################################################### | ||
| doc_generate: | ||
| if: ${{ inputs.id == 'linux-latest' }} | ||
| name: Generate documentation | ||
| runs-on: ${{ inputs.runner }} | ||
| container: | ||
| image: ${{ inputs.container }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: install dependencies | ||
| run: | | ||
| pacman -Syu --noconfirm | ||
| pacman -S --noconfirm doxygen graphviz | ||
| - name: doxygen | ||
| run: doxygen doc/Doxyfile | ||
| - uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: doxygen | ||
| retention-days: 5 | ||
| path: doc/api/ | ||
| # deployment | ||
| ######################################################### | ||
| collect_webpages: | ||
| if: ${{ (github.head_ref == 'main' || github.ref_name == 'main') && inputs.id == 'linux-latest' }} | ||
| name: Collect webpages | ||
| needs: | ||
| - doc_generate | ||
| - iguana | ||
| runs-on: ${{ inputs.runner }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| path: iguana_src | ||
| - name: download doxygen documentation | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: doxygen | ||
| path: doxygen | ||
| - name: download coverage report | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: coverage-report | ||
| path: coverage-report | ||
| - run: tree | ||
| - name: collect | ||
| run: | | ||
| mkdir pages | ||
| cp iguana_src/.github/pages-index.html pages/index.html | ||
| mv doxygen pages/ | ||
| mv coverage-report pages/ | ||
| - run: tree | ||
| - uses: actions/upload-pages-artifact@v3 | ||
| with: | ||
| retention-days: 5 | ||
| path: pages/ | ||
| deploy_webpages: | ||
| if: ${{ (github.head_ref == 'main' || github.ref_name == 'main') && inputs.id == 'linux-latest' }} | ||
| name: Deploy webpages | ||
| needs: collect_webpages | ||
| permissions: | ||
| pages: write | ||
| id-token: write | ||
| environment: | ||
| name: github-pages | ||
| url: ${{ steps.deployment.outputs.page_url }} | ||
| runs-on: ${{ inputs.runner }} | ||
| steps: | ||
| - name: deployment | ||
| id: deployment | ||
| uses: actions/deploy-pages@v4 | ||
| # finalize | ||
| ######################################################### | ||
| final: | ||
| name: Final | ||
| needs: | ||
| - iguana | ||
| runs-on: ${{ inputs.runner }} | ||
| steps: | ||
| - run: exit 0 | ||
