ci: test with RCDB #6

Workflow file for this run

	name: CI

	on:
	workflow_call:
	inputs:
	id:
	description: 'ID'
	required: true
	type: string
	runner:
	description: 'GitHub runner'
	required: true
	type: string
	container:
	description: 'Docker container'
	required: false
	type: string
	default: ''
	verset:
	description: 'Dependency version set'
	required: false
	type: string
	default: 'latest'
	matrix:
	description: 'Iguana job matrix'
	required: true
	type: string

	concurrency:
	group: ${{ github.workflow }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true

	defaults:
	run:
	shell: bash

	env:
	hipo_fork: gavalian/hipo
	hipo_ref: 20094876df040ec234dd376fa527cf58442b775a
	# test options
	num_events: 1000
	num_threads: 0 # use 0 for "all" available cores
	verbose_test: 'false' # only set this to 'true' if `meson test` fails and you need more output to investigate

	jobs:

	# download test data
	#########################################################

	download_test_data:
	name: Download test data
	runs-on: ubuntu-latest # no need to run this job on other runners
	outputs:
	key: ${{ steps.key.outputs.key }}
	steps:
	- name: key # unique for ${{ inputs.id }}, to avoid cache collisions
	id: key
	run: echo key=test_data---${{ inputs.id }}---$(date +%Y-week%U) >> $GITHUB_OUTPUT
	- uses: actions/cache@v4
	id: cache
	with:
	key: ${{ steps.key.outputs.key }}
	path: test_data.hipo
	lookup-only: true
	- name: install xrootd-client
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	run: \|
	sudo apt -y update
	sudo apt -y upgrade
	sudo apt -y install xrootd-client
	- name: download
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	run: xrdcp xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/validation/recon/dst/validation_files.tar.zst ./
	- name: rename
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	run: \|
	tar xvf validation_files.tar.zst
	mv -v $(find validation_files -type f -name "*.hipo" \| head -n1) test_data.hipo

	# dependencies
	#########################################################

	build_root:
	name: Build ROOT
	runs-on: ${{ inputs.runner }}
	container:
	image: ${{ inputs.container }}
	outputs:
	key: ${{ steps.key.outputs.key }}
	steps:
	### define key
	- name: checkout iguana
	uses: actions/checkout@v4
	with:
	path: iguana_src
	- name: key
	id: key
	# FIXME: this builds the source code URL; using the GitHub REST API
	# (via `curl --request`) is preferred to get the source code URL, but
	# needs a PAT to mitigate API rate limits; for now just assume the URL
	# - see https://github.com/JeffersonLab/iguana/pull/276
	run: \|
	[ "${{ inputs.verset }}" = "minver" ] && \
	root_version=$(iguana_src/meson/minimum-version.sh root tag) \|\| \
	root_version=$(iguana_src/meson/minimum-version.sh root tag_latest)
	echo root_url=https://api.github.com/repos/root-project/root/tarball/refs/tags/${root_version} >> $GITHUB_OUTPUT
	echo key=root---${{ inputs.id }}---${root_version}---$(date +%Y-week%U) >> $GITHUB_OUTPUT
	### cache restore
	- uses: actions/cache/restore@v4
	id: cache
	with:
	key: ${{ steps.key.outputs.key }}
	path: root.tar.zst
	lookup-only: true
	### download and build
	- name: install dependencies
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	run: iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }}
	- name: download ROOT source code
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	run: \|
	wget -nv --no-check-certificate --output-document root.tar.gz ${{ steps.key.outputs.root_url }}
	tar xf root.tar.gz
	ls -t
	mv -v $(ls -td root-* \| head -n1) root_src
	- name: build ROOT
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	run: \|
	cmake -S root_src -B build -G Ninja --install-prefix $(pwd)/root -DCMAKE_CXX_STANDARD=17
	cmake --build build
	cmake --install build
	tar caf root{.tar.zst,}
	### cache save
	- uses: actions/cache/save@v4
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	id: cache_save
	with:
	key: ${{ steps.key.outputs.key }}
	path: root.tar.zst

	build_hipo:
	name: Build HIPO
	needs:
	- build_root # for dataframe support
	runs-on: ${{ inputs.runner }}
	container:
	image: ${{ inputs.container }}
	strategy:
	fail-fast: true
	matrix:
	root_dep:
	- noROOT # exclude dataframe lib, which depends on ROOT
	- withROOT
	outputs:
	key: ${{ steps.key.outputs.key }}
	steps:
	### get ROOT
	- name: get ROOT build
	if: ${{ matrix.root_dep == 'withROOT' }}
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build_root.outputs.key }}
	path: root.tar.zst
	- name: untar ROOT build
	if: ${{ matrix.root_dep == 'withROOT' }}
	run: ls *.tar.zst \| xargs -I{} tar xvf {}
	### define key
	- name: key
	id: key
	run: echo key=hipo---${{ inputs.id }}---${{ env.hipo_ref }}---$(date +%Y-week%U) >> $GITHUB_OUTPUT
	### cache restore
	- uses: actions/cache/restore@v4
	id: cache
	with:
	key: ${{ steps.key.outputs.key }}---${{ matrix.root_dep }}
	path: hipo.tar.zst
	lookup-only: true
	### build
	- name: checkout iguana for dependency installation script
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	uses: actions/checkout@v4
	with:
	path: iguana_src
	- name: checkout hipo
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	uses: actions/checkout@v4
	with:
	repository: ${{ env.hipo_fork }}
	ref: ${{ env.hipo_ref }}
	path: hipo_src
	- name: build
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	run: \|
	iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }}
	[ "${{ matrix.root_dep }}" = "withROOT" ] && source root/bin/thisroot.sh
	meson setup build hipo_src --prefix=$(pwd)/hipo -Dbuildtype=debug -Db_pie=true # using PIE build, for sanitizer readibility
	meson install -C build
	tar cavf hipo{.tar.zst,}
	### cache save
	- uses: actions/cache/save@v4
	if: ${{ steps.cache.outputs.cache-hit != 'true' }}
	id: cache_save
	with:
	key: ${{ steps.key.outputs.key }}---${{ matrix.root_dep }}
	path: hipo.tar.zst

	build_ruby_minver:
	name: Build Minver Ruby
	runs-on: ${{ inputs.runner }}
	container:
	image: ${{ inputs.container }}
	outputs:
	key: ${{ steps.key.outputs.key }}
	steps:
	- name: checkout iguana
	if: ${{ inputs.verset == 'minver' }}
	uses: actions/checkout@v4
	with:
	path: iguana_src
	- name: key
	if: ${{ inputs.verset == 'minver' }}
	id: key
	run: \|
	ver=$(iguana_src/meson/minimum-version.sh ruby)
	echo ver=$ver >> $GITHUB_OUTPUT
	echo key=ruby---${{ inputs.id }}---${ver}---$(date +%Y-week%U) >> $GITHUB_OUTPUT
	- uses: actions/cache/restore@v4
	if: ${{ inputs.verset == 'minver' }}
	id: cache
	with:
	key: ${{ steps.key.outputs.key }}
	path: ruby.tar.zst
	lookup-only: true
	- name: build ruby
	if: ${{ steps.cache.outputs.cache-hit != 'true' && inputs.verset == 'minver' }}
	run: \|
	iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }}
	export RBENV_ROOT=$(pwd)/.rbenv
	git clone https://github.com/rbenv/rbenv.git $RBENV_ROOT
	eval "$($RBENV_ROOT/bin/rbenv init - bash)"
	git clone https://github.com/rbenv/ruby-build.git $(rbenv root)/plugins/ruby-build
	rbenv install ${{ steps.key.outputs.ver }}
	rbenv global ${{ steps.key.outputs.ver }}
	tar cavf ruby.tar.zst .rbenv
	- uses: actions/cache/save@v4
	if: ${{ steps.cache.outputs.cache-hit != 'true' && inputs.verset == 'minver' }}
	id: cache_save
	with:
	key: ${{ steps.key.outputs.key }}
	path: ruby.tar.zst

	# build and test Iguana
	#########################################################

	iguana:
	name: Iguana
	needs:
	- download_test_data
	- build_hipo
	- build_root
	- build_ruby_minver
	runs-on: ${{ inputs.runner }}
	container:
	image: ${{ inputs.container }}
	strategy:
	fail-fast: false
	matrix: ${{ fromJson(inputs.matrix) }}
	env:
	CC: ${{ matrix.CC }}
	CXX: ${{ matrix.CXX }}
	steps:
	### setup
	- uses: actions/checkout@v4
	with: # settings needed for version number detection
	clean: false
	fetch-tags: true
	fetch-depth: 0
	path: iguana_src
	### get test data
	- name: get test data
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.download_test_data.outputs.key }}
	path: test_data.hipo
	### dependencies
	###### system
	- name: install dependency packages
	run: iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }}
	###### ruby
	- name: install ruby from linux package manager
	if: ${{ inputs.id != 'macOS' && inputs.verset != 'minver' }}
	run: pacman -S --noconfirm ruby
	- name: install ruby from cached minver build
	if: ${{ inputs.id != 'macOS' && inputs.verset == 'minver' }}
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build_ruby_minver.outputs.key }}
	path: ruby.tar.zst
	- name: source ruby from cached minver build
	if: ${{ inputs.id != 'macOS' && inputs.verset == 'minver' }}
	run: \|
	tar xf ruby.tar.zst
	export RBENV_ROOT=$(pwd)/.rbenv
	echo RBENV_ROOT=$RBENV_ROOT >> $GITHUB_ENV
	eval "$($RBENV_ROOT/bin/rbenv init - bash)"
	echo "ruby --version:"
	ruby --version
	echo PATH=$PATH >> $GITHUB_ENV
	###### python bindings
	- name: install python bindings dependencies
	if: ${{ matrix.id == 'python' }}
	run: \|
	python -m venv .venv
	source .venv/bin/activate
	echo PATH=$PATH >> $GITHUB_ENV
	python -m pip install -r iguana_src/bind/python/requirements.txt
	###### install doxygen
	- name: install doxygen
	if: ${{ matrix.id == 'documentation' }}
	run: \|
	pacman -S --noconfirm doxygen graphviz
	echo "INSTALLED VERSIONS:"
	echo "doxygen: $(doxygen --version)"
	echo "$(dot --version)"
	###### hipo
	- name: get `hipo-withROOT` build
	if: ${{ matrix.id != 'noROOT' }}
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build_hipo.outputs.key }}---withROOT
	path: hipo.tar.zst
	- name: get `hipo-noROOT` build
	if: ${{ matrix.id == 'noROOT' }}
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build_hipo.outputs.key }}---noROOT
	path: hipo.tar.zst
	- name: untar hipo build
	run: tar xf hipo.tar.zst
	- run: tree hipo
	###### ROOT
	- name: get `ROOT` build
	if: ${{ matrix.id != 'noROOT' }}
	uses: actions/cache/restore@v4
	with:
	key: ${{ needs.build_root.outputs.key }}
	path: root.tar.zst
	- name: untar ROOT build
	if: ${{ matrix.id != 'noROOT' }}
	run: tar xf root.tar.zst
	- name: set ROOT environment
	if: ${{ matrix.id != 'noROOT' }}
	run: iguana_src/.github/source-ROOT.sh root
	###### RCDB
	- name: get RCDB
	- uses: actions/checkout@v4
	with:
	repository: JeffersonLab/rcdb
	ref: main
	path: rcdb_src
	- name: set RCDB env
	id: rcdb_env
	working-directory: rcdb_src
	run: \|
	echo RCDB_HOME=$(pwd) >> $GITHUB_ENV
	echo ver=$(git rev-parse HEAD \| cut -c1-7) >> $GITHUB_OUTPUT
	###### summarize dependencies
	- name: summarize dependencies
	if: ${{ matrix.id == 'cpp' }}
	run: \|
	echo '### Dependencies' >> $GITHUB_STEP_SUMMARY
	echo '\| Dependency \| Version \|' >> $GITHUB_STEP_SUMMARY
	echo '\| --- \| --- \|' >> $GITHUB_STEP_SUMMARY
	echo "\| \`hipo\` \| ${{ env.hipo_ref }} \|" >> $GITHUB_STEP_SUMMARY
	echo "\| \`rcdb\` \| commit hash \`${{ steps.rcdb_env.outputs.ver }}\` \|" >> $GITHUB_STEP_SUMMARY
	echo "\| \`root\` \| $(root --version 2>&1 \| head -n1) \|" >> $GITHUB_STEP_SUMMARY
	echo "\| \`ruby\` \| $(ruby --version) \|" >> $GITHUB_STEP_SUMMARY
	cat pkg_summary.md >> $GITHUB_STEP_SUMMARY
	### build
	- name: meson setup
	run: \|
	meson setup iguana_build iguana_src \
	--prefix=$(pwd)/iguana \
	--pkg-config-path=$(pwd)/hipo/lib/pkgconfig \
	-Drcdb:home=$RCDB_HOME \
	-Dwerror=true \
	-Dinstall_examples=true \
	-Dtest_data_file=$(pwd)/test_data.hipo \
	-Dtest_num_events=${{ env.num_events }} \
	-Dtest_output_dir=$(pwd)/validation_results \
	-Dtest_multithreading=${{ env.num_threads }} \
	${{ matrix.opts }}
	- name: dump all build options
	run: meson configure iguana_build --no-pager
	- name: dump project build options
	run: iguana_src/meson/dump-build-options.sh iguana_build
	- run: meson compile
	working-directory: iguana_build
	- run: meson install
	working-directory: iguana_build
	### dump info about this build
	- name: dump build log
	if: always()
	run: cat iguana_build/meson-logs/meson-log.txt
	- name: cat pkg-config pc files
	run: \|
	pcfiles=$(find iguana -type f -name "*.pc")
	for pcfile in $pcfiles; do
	echo "[+++] cat $pcfile"
	cat $pcfile
	done
	- run: tree iguana
	### run tests
	- name: meson test
	working-directory: iguana_build
	run: \|
	if [ "${{ env.verbose_test }}" = "true" ]; then
	[ ${{ inputs.runner }} = "macos-latest" ] && stdbuf_cmd=gstdbuf \|\| stdbuf_cmd=stdbuf
	$stdbuf_cmd -o0 meson test --print-errorlogs --verbose --no-stdsplit
	else
	meson test --print-errorlogs
	fi
	### run benchmarks
	- name: benchmark algorithms
	if: ${{ matrix.id == 'coverage' }} # use the coverage job's GITHUB_STEP_SUMMARY
	run: \|
	for suite in single_threaded memoize; do
	meson test --benchmark --suite $suite -C iguana_build \| tee benchmark_$suite.txt
	done
	iguana_src/.github/make-benchmark-table.rb benchmark_{single_threaded,memoize}.txt \| xargs -0 -I{} echo {} >> $GITHUB_STEP_SUMMARY
	### coverage
	- name: coverage
	if: ${{ matrix.id == 'coverage' }}
	run: \|
	ninja -C iguana_build coverage-html
	ninja -C iguana_build coverage-text
	mv iguana_build/meson-logs/coveragereport coverage-report
	echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY
	echo '```' >> $GITHUB_STEP_SUMMARY
	cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY
	echo '```' >> $GITHUB_STEP_SUMMARY
	echo '' >> $GITHUB_STEP_SUMMARY
	echo '- for details, see the `coverage-report` artifact' >> $GITHUB_STEP_SUMMARY
	echo '- to compare to the report from the `main` branch, see <https://jeffersonlab.github.io/iguana/coverage-report>' >> $GITHUB_STEP_SUMMARY
	### test relocatability
	- name: test relocatability
	run: \|
	mv iguana relocated
	source relocated/bin/this_iguana.sh --verbose # do not use --githubCI option, since we want this environment to be for only this step
	relocated/bin/iguana_ex_cpp_00_run_functions test_data.hipo ${{ env.num_events }}
	mv relocated iguana
	### set iguana environment, since the next steps will check the iguana installation
	- name: set iguana environment
	run: source iguana/bin/this_iguana.sh --verbose --githubCI
	### test installed examples
	###### cpp
	- run: iguana_ex_cpp_00_run_functions test_data.hipo ${{ env.num_events }}
	if: ${{ matrix.id == 'cpp' }}
	- run: iguana_ex_cpp_01_action_functions test_data.hipo ${{ env.num_events }}
	if: ${{ matrix.id == 'cpp' }}
	- run: iguana_ex_cpp_dataframes test_data.hipo ${{ env.num_events }}
	if: ${{ matrix.id == 'cpp' }}
	- run: iguana_ex_cpp_config_files iguana/etc/iguana/examples
	if: ${{ matrix.id == 'cpp' }}
	###### python
	- run: iguana_ex_python_00_run_functions.py test_data.hipo ${{ env.num_events }}
	if: ${{ matrix.id == 'python' }}
	- run: iguana_ex_python_01_action_functions.py test_data.hipo ${{ env.num_events }}
	if: ${{ matrix.id == 'python' }}
	- run: iguana_ex_python_hipopy.py test_data.hipo ${{ env.num_events }}
	if: ${{ matrix.id == 'python' }}
	###### fortran
	- run: iguana_ex_fortran_01_action_functions test_data.hipo ${{ env.num_events }}
	if: ${{ matrix.id == 'fortran' }}
	### test ROOT macro
	- name: test iguana_ex_cpp_ROOT_macro.C interpreted
	run: root -b -q iguana_src/examples/iguana_ex_cpp_ROOT_macro.C
	if: ${{ matrix.id == 'cpp' }}
	- name: test iguana_ex_cpp_ROOT_macro.C compiled
	run: root -b -q iguana_src/examples/iguana_ex_cpp_ROOT_macro.C+
	if: ${{ matrix.id == 'cpp' }}
	### test consumers
	- name: consumer test make
	if: ${{ matrix.id == 'cpp' }}
	run: \|
	make -C iguana_src/examples/build_with_make
	echo "========================================= TEST RUN ========================================="
	iguana_src/examples/build_with_make/bin/iguana_ex_cpp_00_run_functions test_data.hipo 10
	- name: consumer test meson
	if: ${{ matrix.id == 'cpp' }}
	run: \|
	meson setup build_consumer_meson iguana_src/examples/build_with_meson --prefix=$(pwd)/install_consumer_meson
	meson install -C build_consumer_meson
	echo "========================================= TEST RUN ========================================="
	install_consumer_meson/bin/iguana_ex_cpp_00_run_functions test_data.hipo 10
	- name: consumer test cmake
	if: ${{ matrix.id == 'cpp' }}
	run: \|
	cmake -B build_consumer_cmake -S iguana_src/examples/build_with_cmake -DCMAKE_PREFIX_PATH=$(pwd)/hipo -G Ninja --install-prefix=$(pwd)/install_consumer_cmake
	cmake --build build_consumer_cmake
	cmake --install build_consumer_cmake
	echo "========================================= TEST RUN ========================================="
	install_consumer_cmake/bin/iguana_ex_cpp_00_run_functions test_data.hipo 10
	### upload artifacts
	- name: upload build log artifacts
	uses: actions/upload-artifact@v4
	if: always()
	with:
	name: meson_logs_${{ matrix.id }}
	retention-days: 5
	path: iguana_build/meson-logs
	- name: upload coverage artifacts
	uses: actions/upload-artifact@v4
	if: ${{ matrix.id == 'coverage' }}
	with:
	name: coverage-report
	retention-days: 5
	path: coverage-report
	- name: upload validator artifacts
	uses: actions/upload-artifact@v4
	if: ${{ matrix.id == 'cpp' }}
	with:
	name: _validation_results
	retention-days: 5
	path: validation_results
	- name: upload documentation artifacts
	uses: actions/upload-artifact@v4
	if: ${{ matrix.id == 'documentation' }}
	with:
	name: doc_doxygen
	retention-days: 5
	path: iguana/share/doc/iguana/html/

	# deployment
	#########################################################

	collect_webpages:
	if: ${{ github.ref == 'refs/heads/main' && inputs.id == 'linux-latest' }}
	name: Collect webpages
	needs:
	- iguana
	runs-on: ${{ inputs.runner }}
	steps:
	- uses: actions/checkout@v4
	with:
	path: iguana_src
	- name: download doxygen documentation
	uses: actions/download-artifact@v4
	with:
	name: doc_doxygen
	path: doxygen
	- name: download coverage report
	uses: actions/download-artifact@v4
	with:
	name: coverage-report
	path: coverage-report
	- run: tree
	- name: collect
	run: \|
	mkdir pages
	cp iguana_src/.github/pages-index.html pages/index.html
	mv doxygen pages/
	mv coverage-report pages/
	- run: tree
	- uses: actions/upload-pages-artifact@v3
	with:
	retention-days: 5
	path: pages/

	deploy_webpages:
	if: ${{ github.ref == 'refs/heads/main' && inputs.id == 'linux-latest' }}
	name: Deploy webpages
	needs: collect_webpages
	permissions:
	pages: write
	id-token: write
	environment:
	name: github-pages
	url: ${{ steps.deployment.outputs.page_url }}
	runs-on: ${{ inputs.runner }}
	steps:
	- name: deployment
	id: deployment
	uses: actions/deploy-pages@v4

	# finalize
	#########################################################

	final:
	name: Final
	needs:
	- iguana
	runs-on: ${{ inputs.runner }}
	steps:
	- run: exit 0

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

ci: test with RCDB #6

Workflow file

ci: test with RCDB #6

Jobs

Run details

Workflow file for this run