Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CI-Cleanup #3

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

CI-Cleanup #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
28e7b5c
remove adios build and communicate builds as artifacts
t-erxleben Apr 23, 2023
6a694f5
enable workflow for current branch
t-erxleben Apr 23, 2023
7deffa7
switch to cache action
t-erxleben Apr 23, 2023
b83344d
add missing build dir to cache
t-erxleben Apr 24, 2023
13b5fb8
fix paths for caching
t-erxleben Apr 24, 2023
3ae3efe
activate test matrix and eliminate commented or useless steps
t-erxleben Apr 24, 2023
64829a2
disable mysql test
t-erxleben Jun 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
317 changes: 97 additions & 220 deletions .github/workflows/build-adios2-w-julea.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,16 @@ name: Compile ADIOS2 with JULEA

on:
push:
branches: [ "master" ]
branches: [ "master", "CI/cleanup" ]
pull_request:
branches: [ "master" ]
# schedule:
# # using a schedule instead of dispatch to trigger workflow when JULEA is updated
# - cron: '0 0 * * 0'

env:
BUILD_TYPE: Release

jobs:
build-adios2-julea:
name: Build ADIOS2 with JULEA
name: Build
runs-on: ${{ matrix.os.dist }}
timeout-minutes: 180
strategy:
Expand All @@ -24,14 +21,6 @@ jobs:
- dist: ubuntu-22.04
compiler: gcc
compiler_version: 11.3.0
# - dist: ubuntu-20.04
# compiler: gcc
# compiler_version: 9.4.0
# doesn't work: Compiler too old for C++ application
# "sorry, unimplemented: non-trivial designated initializers not supported"
# - dist: ubuntu-18.04
# compiler: gcc
# compiler_version: 7.5.0
steps:
- name: Checkout JULEA
uses: actions/checkout@v3
Expand All @@ -54,36 +43,21 @@ jobs:
JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }}
run: ./scripts/install-dependencies.sh

# https://github.com/actions/checkout/issues/209
- name: Get commit hash
id: julea-hash
working-directory: julea
run: |
echo ::set-output name=hash::$(git rev-parse --short HEAD)

- name: Cache JULEA
id: julea-cache
uses: actions/cache@v3
with:
path: julea-install
key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{steps.julea-hash.outputs.hash}}

# --- Build JULEA
- name: Configure JULEA with meson
if: ${{ steps.julea-cache.outputs.cache-hit != 'true' }}
- name: Configure JULEA
working-directory: julea
run: |
. scripts/environment.sh
meson setup --prefix="${GITHUB_WORKSPACE}/julea-install" --buildtype=release bld

- name: Build JULEA with ninja
- name: Build JULEA
if: ${{ steps.julea-cache.outputs.cache-hit != 'true' }}
working-directory: julea
run: |
. scripts/environment.sh
ninja -C bld

- name: Install
- name: Install JULEA
if: ${{ steps.julea-cache.outputs.cache-hit != 'true' }}
working-directory: julea
run: |
Expand All @@ -101,41 +75,36 @@ jobs:
sudo apt update || true
sudo apt --yes --no-install-recommends install libopenmpi-dev openmpi-bin

- name: Configure CMake
- name: Configure ADIOS2
working-directory: adios2
env:
JULEA_PREFIX: ${{github.workspace}}/julea-install
run: |
. ../julea/scripts/environment.sh
cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_MPI=ON -DADIOS2_USE_SST=OFF -DADIOS2_USE_SSC=OFF -DADIOS2_USE_DataMan=OFF -DADIOS2_USE_BZip2=OFF -DADIOS2_USE_PNG=OFF -DADIOS2_USE_JULEA=ON -DADIOS2_USE_NetCDF=OFF

# --- Cache ADIOS2
# Check whether only the workflow files and not the source files changed
# - name: Check changed files
# id: adios2-hash
# working-directory: adios2
# run: |
# echo ::set-output name=hash::$(git rev-parse --short HEAD)

# TODO: before or after configuring cmake? is build dir available before?
# - name: Cache ADIOS2
# id: adios2-cache
# uses: actions/cache@v3
# with:
# path: ${{github.workspace}}/build # TODO: does this work?
# key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{steps.adios2-hash.outputs.hash}}

- name: Build
- name: Build ADIOS2
working-directory: adios2
env:
JULEA_PREFIX: ${{github.workspace}}/julea-install
run: |
. ../julea/scripts/environment.sh
cd ${{github.workspace}}/build && make -j 2

- name: Cache build
uses: actions/cache/save@v3
with:
path: |
julea
${{github.workspace}}/julea-install
adios2
${{github.workspace}}/build
key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-testenv


test-application:
name: Run example application
# needs: build-adios2-julea
needs: build-adios2-julea
runs-on: ${{ matrix.os.dist }}
timeout-minutes: 180
strategy:
Expand All @@ -145,187 +114,95 @@ jobs:
- dist: ubuntu-22.04
compiler: gcc
compiler_version: 11.3.0
# - dist: ubuntu-20.04
# compiler: gcc
# compiler_version: 9.4.0
# doesn't work: Compiler too old for C++ application
# "sorry, unimplemented: non-trivial designated initializers not supported"
# - dist: ubuntu-18.04
# compiler: gcc
# compiler_version: 7.5.0
julea:
# Default
- object: posix
kv: lmdb
db: sqlite
# KV backends
# - object: posix
# kv: leveldb
# db: sqlite
# - object: posix
# kv: rocksdb
# db: sqlite
# - object: posix
# kv: sqlite
# db: sqlite
# # DB backends
- object: posix
kv: leveldb
db: sqlite
- object: posix
kv: rocksdb
db: sqlite
- object: posix
kv: sqlite
db: sqlite
# DB backends
# - object: posix
# kv: lmdb
# db: mysql
# db-server: mysql
# - object: posix
# kv: lmdb
# db: mysql
# db-server: mariadb
- object: posix
kv: lmdb
db: mysql
db-server: mariadb
steps:
- name: Checkout JULEA
uses: actions/checkout@v3
with:
repository: Bella42/julea
ref: dai-copy
path: julea

- name: Cache dependencies
id: dep-cache
uses: actions/cache@v3
with:
path: julea/dependencies
key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{ hashFiles('julea/scripts/spack', 'julea/scripts/install-dependencies.sh') }}-dependencies
- name: Install MPI
run: |
sudo apt update || true
sudo apt --yes --no-install-recommends install libopenmpi-dev openmpi-bin

- name: Install dependencies
if: ${{ steps.dep-cache.outputs.cache-hit != 'true' }}
working-directory: julea
env:
JULEA_SPACK_COMPILER: ${{ matrix.os.compiler }}@${{ matrix.os.compiler_version }}
run: ./scripts/install-dependencies.sh

# https://github.com/actions/checkout/issues/209
- name: Get commit hash
id: julea-hash
working-directory: julea
run: |
echo ::set-output name=hash::$(git rev-parse --short HEAD)

- name: Cache JULEA
id: julea-cache
uses: actions/cache@v3
with:
path: julea-install
key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-${{steps.julea-hash.outputs.hash}}

# --- Build JULEA
- name: Configure JULEA with meson
if: ${{ steps.julea-cache.outputs.cache-hit != 'true' }}
working-directory: julea
run: |
. scripts/environment.sh
meson setup --prefix="${GITHUB_WORKSPACE}/julea-install" --buildtype=release bld

- name: Build JULEA with ninja
if: ${{ steps.julea-cache.outputs.cache-hit != 'true' }}
working-directory: julea
run: |
. scripts/environment.sh
ninja -C bld

- name: Install
if: ${{ steps.julea-cache.outputs.cache-hit != 'true' }}
working-directory: julea
run: |
. scripts/environment.sh
ninja -C bld install

# --- Build ADIOS2
- name: Checkout ADIOS2
uses: actions/checkout@v3
with:
path: adios2

- name: Install MPI
run: |
sudo apt update || true
sudo apt --yes --no-install-recommends install libopenmpi-dev openmpi-bin

- name: Configure CMake
working-directory: adios2
env:
JULEA_PREFIX: ${{github.workspace}}/julea-install
run: |
. ../julea/scripts/environment.sh
cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_MPI=ON -DADIOS2_USE_SST=OFF -DADIOS2_USE_SSC=OFF -DADIOS2_USE_DataMan=OFF -DADIOS2_USE_BZip2=OFF -DADIOS2_USE_PNG=OFF -DADIOS2_USE_JULEA=ON -DADIOS2_USE_NetCDF=OFF

- name: Build
working-directory: adios2
env:
JULEA_PREFIX: ${{github.workspace}}/julea-install
run: |
. ../julea/scripts/environment.sh
cd ${{github.workspace}}/build && make -j 2

# --- Setup servers for JULEA database backends
- name: Set up MySQL
if: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mysql' }}
run: |
sudo systemctl start mysql.service
mysql --user=root --password=root --execute="CREATE DATABASE juleadb;"
mysql --user=root --password=root --execute="CREATE USER 'julea'@'localhost' IDENTIFIED WITH mysql_native_password BY 'aeluj';"
mysql --user=root --password=root --execute="GRANT ALL ON juleadb.* TO 'julea'@'localhost';"
- name: Cache build
uses: actions/cache/restore@v3
with:
fail-on-cache-miss: true
path: |
julea
${{github.workspace}}/julea-install
adios2
${{github.workspace}}/build
key: ${{ matrix.os.dist }}-${{ matrix.os.compiler }}-${{ matrix.os.compiler_version }}-testenv

# --- Setup servers for JULEA database backends
- name: Set up MySQL
if: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mysql' }}
run: |
sudo systemctl start mysql.service
mysql --user=root --password=root --execute="CREATE DATABASE juleadb;"
mysql --user=root --password=root --execute="CREATE USER 'julea'@'localhost' IDENTIFIED WITH mysql_native_password BY 'aeluj';"
mysql --user=root --password=root --execute="GRANT ALL ON juleadb.* TO 'julea'@'localhost';"

- name: Set up MariaDB
if: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mariadb' }}
# FIXME https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
# Ubuntu 18.04 has MySQL 5.7, Ubuntu 20.04 has MySQL 8.0
run: |
if test "${{ matrix.os.dist }}" = 'ubuntu-18.04'
then
sudo apt --yes purge mysql-client mysql-client-5.7 mysql-server mysql-server-5.7
else
- name: Set up MariaDB
if: ${{ matrix.julea.db == 'mysql' && matrix.julea.db-server == 'mariadb' }}
run: |
sudo apt --yes purge mysql-client mysql-client-8.0 mysql-server mysql-server-8.0
fi
sudo apt --yes --purge autoremove
sudo aa-remove-unknown
sudo rm --recursive --force /var/lib/mysql
sudo apt update || true
sudo apt --yes --no-install-recommends install mariadb-server
sudo systemctl start mariadb.service
sudo mariadb --execute="CREATE DATABASE juleadb;"
sudo mariadb --execute="CREATE USER 'julea'@'localhost' IDENTIFIED BY 'aeluj';"
sudo mariadb --execute="GRANT ALL ON juleadb.* TO 'julea'@'localhost';"

# --- Create JULEA configurations
- name: Create configuration
# FIXME We need to use 127.0.0.1 because localhost tries to use the socket (/tmp/mysql.sock), which does not exist.
working-directory: adios2
env:
JULEA_PREFIX: ${{github.workspace}}/julea-install
run: |
. ../julea/scripts/environment.sh
JULEA_KV_COMPONENT='server'
if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_COMPONENT='client'; fi
JULEA_KV_PATH="/tmp/julea/kv/${{ matrix.julea.kv }}"
JULEA_DB_COMPONENT='server'
if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_COMPONENT='client'; fi
JULEA_DB_PATH="/tmp/julea/db/${{ matrix.julea.db }}"
if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi
julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-component=server --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-component="${JULEA_KV_COMPONENT}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-component="${JULEA_DB_COMPONENT}" --db-path="${JULEA_DB_PATH}"

- name: Test application
working-directory: build
env:
JULEA_PREFIX: ${{github.workspace}}/julea-install
run: |
. ../julea/scripts/environment.sh
../julea/scripts/setup.sh start
mpirun -n 2 ./bin/thesis_eval_heatTransfer_write_adios2 ../adios2/examples/thesis/heatTransfer/heat_julea-db.xml heat_test-julea-db.bp 2 1 10 10 4 10 julea-db
# mpirun -n 2 ./bin/thesis_eval_heatTransfer_write_adios2 ../adios2/examples/thesis/heatTransfer/heat_julea-kv.xml heat_test-julea-kv.bp 2 1 10 10 4 10 julea-kv
mpirun -n 2 ./bin/thesis_eval_heatTransfer_write_adios2 ../adios2/examples/thesis/heatTransfer/heat_bp3_sync.xml heat_test-bp3.bp 2 1 10 10 4 10 bp3
mpirun -n 2 ./bin/thesis_eval_heatTransfer_write_adios2 ../adios2/examples/thesis/heatTransfer/heat_bp4_sync.xml heat_test-bp4.bp 2 1 10 10 4 10 bp4

- name: Test import_nc
working-directory: build
env:
JULEA_PREFIX: ${{github.workspace}}/julea-install
run: |
. ../julea/scripts/environment.sh
../julea/scripts/setup.sh start
#TODO: spack install netcdf-cxx
sudo apt --yes --purge autoremove
sudo aa-remove-unknown
sudo rm --recursive --force /var/lib/mysql
sudo apt update || true
sudo apt --yes --no-install-recommends install mariadb-server
sudo systemctl start mariadb.service
sudo mariadb --execute="CREATE DATABASE juleadb;"
sudo mariadb --execute="CREATE USER 'julea'@'localhost' IDENTIFIED BY 'aeluj';"
sudo mariadb --execute="GRANT ALL ON juleadb.* TO 'julea'@'localhost';"

# --- Create JULEA configurations
- name: Create configuration
# FIXME We need to use 127.0.0.1 because localhost tries to use the socket (/tmp/mysql.sock), which does not exist.
working-directory: adios2
env:
JULEA_PREFIX: ${{github.workspace}}/julea-install
run: |
. ../julea/scripts/environment.sh
JULEA_KV_COMPONENT='server'
if test "${{ matrix.julea.kv }}" = 'mongodb'; then JULEA_KV_COMPONENT='client'; fi
JULEA_KV_PATH="/tmp/julea/kv/${{ matrix.julea.kv }}"
JULEA_DB_COMPONENT='server'
if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_COMPONENT='client'; fi
JULEA_DB_PATH="/tmp/julea/db/${{ matrix.julea.db }}"
if test "${{ matrix.julea.db }}" = 'mysql'; then JULEA_DB_PATH='127.0.0.1:juleadb:julea:aeluj'; fi
julea-config --user --object-servers="$(hostname)" --kv-servers="$(hostname)" --db-servers="$(hostname)" --object-backend="${{ matrix.julea.object }}" --object-component=server --object-path="/tmp/julea/object/${{ matrix.julea.object }}" --kv-backend="${{ matrix.julea.kv }}" --kv-component="${JULEA_KV_COMPONENT}" --kv-path="${JULEA_KV_PATH}" --db-backend="${{ matrix.julea.db }}" --db-component="${JULEA_DB_COMPONENT}" --db-path="${JULEA_DB_PATH}"

- name: Test application
working-directory: build
env:
JULEA_PREFIX: ${{github.workspace}}/julea-install
run: |
. ../julea/scripts/environment.sh
../julea/scripts/setup.sh start
mpirun -n 2 ./bin/thesis_eval_heatTransfer_write_adios2 ../adios2/examples/thesis/heatTransfer/heat_julea-db.xml heat_test-julea-db.bp 2 1 10 10 4 10 julea-db
# mpirun -n 2 ./bin/thesis_eval_heatTransfer_write_adios2 ../adios2/examples/thesis/heatTransfer/heat_julea-kv.xml heat_test-julea-kv.bp 2 1 10 10 4 10 julea-kv
mpirun -n 2 ./bin/thesis_eval_heatTransfer_write_adios2 ../adios2/examples/thesis/heatTransfer/heat_bp3_sync.xml heat_test-bp3.bp 2 1 10 10 4 10 bp3
mpirun -n 2 ./bin/thesis_eval_heatTransfer_write_adios2 ../adios2/examples/thesis/heatTransfer/heat_bp4_sync.xml heat_test-bp4.bp 2 1 10 10 4 10 bp4
Loading