Skip to content

Commit

Permalink
Merge pull request #291 from EinsteinToolkit/eschnett/ubuntu-oneapi
Browse files Browse the repository at this point in the history
CI: Update scripts and settings
  • Loading branch information
eschnett authored Jun 18, 2024
2 parents a6de8f2 + bc84390 commit 0126ae0
Show file tree
Hide file tree
Showing 16 changed files with 271 additions and 25 deletions.
17 changes: 10 additions & 7 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,12 @@ jobs:
download-build-test:
strategy:
matrix:
accelerator: [cpu, cuda, rocm]
accelerator: [cpu, cuda, rocm, oneapi]
real-precision: [real32, real64]
mode: [debug, optimize]
exclude:
- {accelerator: rocm, real-precision: real32}
- {accelerator: oneapi, real-precision: real32}
- {real-precision: real32, mode: optimize}
env:
ACCELERATOR: ${{matrix.accelerator}}
Expand All @@ -36,13 +37,13 @@ jobs:
steps:
# Check out the repository under $GITHUB_WORKSPACE, so our job can access it
- uses: actions/checkout@v4

# # Check out the documentation and test results
# - uses: actions/checkout@v3
# with:
# ref: gh-pages
# path: gh-pages

- name: Log the triggering commit
run: echo ${{github.event.client_payload.trigger}}

Expand All @@ -51,12 +52,14 @@ jobs:

- name: Build Cactus
run: /bin/bash $GITHUB_WORKSPACE/scripts/build.sh

# The oneAPI setup does not build yet
continue-on-error: ${{matrix.accelerator == 'oneapi'}}

- name: Test Cactus
# We cannot run with cuda nor with rocm. Debug builds run too slowly.
if: ${{matrix.accelerator != 'cuda' && matrix.accelerator != 'rocm' && matrix.mode != 'debug'}}
# We cannot run on accelerators. Debug builds run too slowly.
if: ${{matrix.accelerator == 'cpu' && matrix.mode == 'optimize'}}
run: /bin/bash $GITHUB_WORKSPACE/scripts/test.sh

# Set exit code based on test status
- name: Set exit code
# Only look at the test results for `carpetx-cpu-real64` runs
Expand Down
11 changes: 10 additions & 1 deletion Algo/src/roots.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,16 @@ extern "C" void Test_roots(CCTK_ARGUMENTS) {
const int maxiters = 100;
int iters;
bool failed;
auto x = newton_raphson_nd(gnd<CCTK_REAL>, vec<CCTK_REAL, 2>{1.0, 1.0},

// SYCL cannot call the templated function above, so we define a
// local copy
auto gnd_CCTK_REAL = [](vec<CCTK_REAL, 2> x)
-> std::pair<vec<CCTK_REAL, 2>, mat<CCTK_REAL, 2> > {
return {vec<CCTK_REAL, 2>{x(0) * x(0) - 2, x(0) * x(1) - 2},
mat<CCTK_REAL, 2>{2 * x(0), x(1), 0, x(0)}};
};

auto x = newton_raphson_nd(gnd_CCTK_REAL, vec<CCTK_REAL, 2>{1.0, 1.0},
vec<CCTK_REAL, 2>{0.0, 0.0},
vec<CCTK_REAL, 2>{10.0, 10.0}, minbits,
maxiters, iters, failed);
Expand Down
2 changes: 1 addition & 1 deletion TestSymmetries/src/init.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ extern "C" void TestSymmetries_Check(CCTK_ARGUMENTS) {
auto &var, const auto &value) CCTK_ATTRIBUTE_ALWAYS_INLINE {
using std::abs;
if (abs(var - value) > 10 * std::numeric_limits<CCTK_REAL>::epsilon()) {
#if !defined __CUDACC__ && !defined __HIPCC__
#if !defined __CUDACC__ && !defined __HIPCC__ && !defined SYCL_LANGUAGE_VERSION
CCTK_VERROR(
"Grid function symmetry check failed: I=[%d,%d,%d] X=[%g,%g,%g] "
"centering=[%d,%d,%d] parity=[%d,%d,%d] var=%.17g value=%.17g",
Expand Down
4 changes: 2 additions & 2 deletions docker/carpetx-arm64v8-cpu.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
# docker push einsteintoolkit/carpetx:arm64v8-cpu-real32

# noble is ubuntu:24.04
# FROM arm64v8/ubuntu:noble-20240423
FROM arm64v8/ubuntu:noble-20240429
# FROM arm64v8/ubuntu:noble-20240429
FROM arm64v8/ubuntu:noble-20240530

ENV DEBIAN_FRONTEND=noninteractive \
LANGUAGE=en_US.en \
Expand Down
4 changes: 2 additions & 2 deletions docker/carpetx-cpu.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
# docker push einsteintoolkit/carpetx:cpu-real32

# noble is ubuntu:24.04
# FROM ubuntu:noble-20240423
FROM ubuntu:noble-20240429
# FROM ubuntu:noble-20240429
FROM ubuntu:noble-20240530

ENV DEBIAN_FRONTEND=noninteractive \
LANGUAGE=en_US.en \
Expand Down
22 changes: 17 additions & 5 deletions docker/carpetx-oneapi.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
# docker build --build-arg real_precision=real32 --file carpetx-oneapi.dockerfile --tag einsteintoolkit/carpetx:oneapi-real32 .
# docker push einsteintoolkit/carpetx:oneapi-real32

# FROM intel/oneapi-basekit:devel-ubuntu22.04
FROM intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04
# FROM intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04
FROM intel/oneapi-basekit:2024.1.1-devel-ubuntu22.04

ENV DEBIAN_FRONTEND=noninteractive \
LANGUAGE=en_US.en \
Expand Down Expand Up @@ -66,6 +66,18 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 28DA432DAAC8BAEA &&
&& \
rm -rf /var/lib/apt/lists/*

# Remove troublesome libraries
RUN find /opt/intel -name 'impi.pc' -delete && \
find /opt/intel -name 'libhwloc.a' -delete && \
find /opt/intel -name 'libhwloc.so' -delete && \
find /opt/intel -name 'libmpi*a' -delete && \
find /opt/intel -name 'libmpi*so' -delete && \
find /opt/intel -name 'mpi.h' -delete && \
find /opt/intel -name 'mpicc' -delete && \
find /opt/intel -name 'mpicxx' -delete && \
find /opt/intel -name 'mpiexec' -delete && \
find /opt/intel -name 'mpirun' -delete

# Install blosc2
# blosc2 is a compression library, comparable to zlib
RUN mkdir src && \
Expand Down Expand Up @@ -247,10 +259,10 @@ ARG real_precision=real64
# Should we keep the AMReX source tree around for debugging?
RUN mkdir src && \
(cd src && \
wget https://github.com/AMReX-Codes/amrex/archive/24.05.tar.gz && \
tar xzf 24.05.tar.gz && \
wget https://github.com/AMReX-Codes/amrex/archive/24.06.tar.gz && \
tar xzf 24.06.tar.gz && \
rm -rf /opt/intel/oneapi/mpi && \
cd amrex-24.05 && \
cd amrex-24.06 && \
case $real_precision in \
real32) precision=SINGLE;; \
real64) precision=DOUBLE;; \
Expand Down
4 changes: 2 additions & 2 deletions docker/carpetx-rocm.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
# docker build --build-arg real_precision=real32 --file carpetx-rocm.dockerfile --tag einsteintoolkit/carpetx:rocm-real32 .
# docker push einsteintoolkit/carpetx:rocm-real32

# FROM rocm/dev-ubuntu-22.04:6.1
FROM rocm/dev-ubuntu-22.04:6.1.1
# FROM rocm/dev-ubuntu-22.04:6.1.1
FROM rocm/dev-ubuntu-22.04:6.1.2

ENV DEBIAN_FRONTEND=noninteractive \
LANGUAGE=en_US.en \
Expand Down
2 changes: 1 addition & 1 deletion scripts/actions-cpu-real32.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ PTHREADS_DIR = NO_BUILD
# REPRIMAND_LIBS = RePrimAnd
# RNPLETAL_DIR = /usr/local
SILO_DIR = /usr/local
# SIMULATIONIO_DIR = /usr/local
SIMULATIONIO_DIR = /usr/local
SSHT_DIR = /usr/local
YAML_CPP_DIR = /usr/local
ZLIB_DIR = /usr
2 changes: 1 addition & 1 deletion scripts/actions-cpu-real64.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ PTHREADS_DIR = NO_BUILD
# REPRIMAND_LIBS = RePrimAnd
# RNPLETAL_DIR = /usr/local
SILO_DIR = /usr/local
# SIMULATIONIO_DIR = /usr/local
SIMULATIONIO_DIR = /usr/local
SSHT_DIR = /usr/local
YAML_CPP_DIR = /usr/local
ZLIB_DIR = /usr
2 changes: 1 addition & 1 deletion scripts/actions-cuda-real32.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ PTHREADS_DIR = NO_BUILD
# REPRIMAND_LIBS = RePrimAnd
# RNPLETAL_DIR = /usr/local
SILO_DIR = /usr/local
# SIMULATIONIO_DIR = /usr/local
SIMULATIONIO_DIR = /usr/local
SSHT_DIR = /usr/local
YAML_CPP_DIR = /usr/local
ZLIB_DIR = /usr
2 changes: 1 addition & 1 deletion scripts/actions-cuda-real64.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ PTHREADS_DIR = NO_BUILD
# REPRIMAND_LIBS = RePrimAnd
# RNPLETAL_DIR = /usr/local
SILO_DIR = /usr/local
# SIMULATIONIO_DIR = /usr/local
SIMULATIONIO_DIR = /usr/local
SSHT_DIR = /usr/local
YAML_CPP_DIR = /usr/local
ZLIB_DIR = /usr
102 changes: 102 additions & 0 deletions scripts/actions-oneapi-real64.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
# Option list for the Einstein Toolkit

# The "weird" options here should probably be made the default in the
# ET instead of being set here.

# Whenever this version string changes, the application is configured
# and rebuilt from scratch
VERSION = actions-oneapi-real64-2023-04-11

CPP = cpp
CC = /opt/intel/oneapi/compiler/2024.1/bin/icx
CXX = /opt/intel/oneapi/compiler/2024.1/bin/icpx
FPP = cpp
FC = gfortran
F90 = gfortran
LD = /opt/intel/oneapi/compiler/2024.1/bin/icpx

CPPFLAGS = -DSIMD_CPU
CFLAGS = -fp-model=precise -g -march=x86-64-v3 -pipe -std=gnu11
CXXFLAGS = -fp-model=precise -fsycl -g -march=x86-64-v3 -pipe -std=c++17
FPPFLAGS = -traditional
F90FLAGS = -fcray-pointer -ffixed-line-length-none -g -march=x86-64-v3 -pipe

SYS_INC_DIRS = /opt/intel/oneapi/2024.1/include
LIBDIRS = /usr/local/lib
LIBS = gfortran

C_LINE_DIRECTIVES = yes
F_LINE_DIRECTIVES = yes

INTEGER_PRECISION = 4
REAL_PRECISION = 8

# oneAPI does not support these
# DISABLE_INT16 = yes
DISABLE_REAL16 = yes

DEBUG = no
CPP_DEBUG_FLAGS = -DCARPET_DEBUG
C_DEBUG_FLAGS = -fsanitize=undefined -fstack-protector-all -ftrapv
CXX_DEBUG_FLAGS = -fsanitize=undefined -fstack-protector-all -ftrapv
FPP_DEBUG_FLAGS = -DCARPET_DEBUG
F90_DEBUG_FLAGS = -fcheck=bounds,do,mem,pointer,recursion -finit-character=65 -finit-integer=42424242 -finit-real=nan -ftrapv

OPTIMISE = yes
C_OPTIMISE_FLAGS = -O3 -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math
CXX_OPTIMISE_FLAGS = -O3 -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math
F90_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans

# Clang segfaults with OpenMP enabled
OPENMP = no
CPP_OPENMP_FLAGS = -fopenmp
FPP_OPENMP_FLAGS = -D_OPENMP
F90_OPENMP_FLAGS = -fopenmp

WARN = yes
CPP_WARN_FLAGS = -Wall
C_WARN_FLAGS =
CXX_WARN_FLAGS =
FPP_WARN_FLAGS = -Wall
F90_WARN_FLAGS = -Wall -Wshadow -Wsurprising

VECTORISE = no

ADIOS2_DIR = /usr/local
ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11
AMREX_DIR = /usr/local
# ASDF_CXX_DIR = /usr/local
BLAS_DIR = /usr
BOOST_DIR = /usr
FFTW3_DIR = /usr
GSL_DIR = /usr
HDF5_DIR = /usr/lib/x86_64-linux-gnu/hdf5/serial
HDF5_ENABLE_CXX = yes
HDF5_ENABLE_FORTRAN = yes
HDF5_INC_DIRS = /usr/lib/x86_64-linux-gnu/hdf5/serial/include
HDF5_LIB_DIRS = /usr/lib/x86_64-linux-gnu/hdf5/serial/lib
HDF5_LIBS = hdf5_hl_cpp hdf5_cpp hdf5hl_fortran hdf5_fortran hdf5_hl hdf5
HWLOC_DIR = /usr
# JEMALLOC_DIR = /usr/local
LAPACK_DIR = /usr
# LIBJPEG_DIR = /usr
# LORENE_DIR = /usr/local
MPI_DIR = /usr
NSIMD_DIR = /usr/local
NSIMD_SIMD = AVX2
OPENBLAS_DIR = /usr
OPENPMD_API_DIR = /usr/local
# OPENSSL_DIR = /usr
# PAPI_DIR = /usr/local
PETSC_DIR = /usr/lib/petsc
PETSC_LIBS = petsc_real
PETSC_ARCH_LIBS = m
PTHREADS_DIR = NO_BUILD
# REPRIMAND_DIR = /usr/local
# REPRIMAND_LIBS = RePrimAnd
# RNPLETAL_DIR = /usr/local
SILO_DIR = /usr/local
SIMULATIONIO_DIR = /usr/local
SSHT_DIR = /usr/local
YAML_CPP_DIR = /usr/local
ZLIB_DIR = /usr
72 changes: 72 additions & 0 deletions scripts/actions-oneapi-real64.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
[actions-oneapi-real64]

# Machine description
nickname = actions-oneapi-real64
name = Actions
location = Github
description = Github action for CarpetX CI
webpage = https://github.com/eschnett/CarpetX
status = production

# Access to this machine
hostname = actions
aliaspattern = ^actions$
# envsetup = <<EOT
# EOT

# Source tree management
sourcebasedir = /__w/CarpetX/workspace
optionlist = actions-oneapi-real64.cfg
submitscript = actions-oneapi-real64.sub
runscript = actions-oneapi-real64.run
make = make -j @MAKEJOBS@
makejobs = 2
disabled-thorns = <<EOT
# Not needed with oneAPI
ExternalLibraries/NSIMD
# long double exists but does not quite work
CactusTest/TestMath

ExternalLibraries/BLAS
ExternalLibraries/LAPACK
ExternalLibraries/LORENE
ExternalLibraries/PAPI
EOT
enabled-thorns = <<EOT
ExternalLibraries/OpenBLAS
ExternalLibraries/PETSc
EOT

# Simulation management
basedir = /__w/CarpetX/workspace/simulations
quota = 10 # don't use all disk space
cpu = Intel Core i9
cpufreq = 2.4
flop/cycle = 16
ppn = 8
spn = 1
mpn = 1
max-num-threads = 16
num-threads = 8
max-num-smt = 2
num-smt = 1
memory = 32768
nodes = 1
min-ppn = 1 # don't want this entry
queue = NOQUEUE # don't want this entry
allocation = NOALLOCATION # don't want this entry
maxwalltime = 8640:00:00 # don't want this entry
# submit = exec @SCRIPTFILE@ </dev/null >/dev/null 2>/dev/null & echo $!
submit = @SCRIPTFILE@ </dev/null >/dev/null 2>/dev/null; true & echo $!
getstatus = ps @JOB_ID@
stop = kill @JOB_ID@
submitpattern = (.*)
statuspattern = "^ *@JOB_ID@ "
queuedpattern = $^
runningpattern = ^
holdingpattern = $^
exechost = echo localhost
exechostpattern = (.*)
stdout = cat @[email protected]
stderr = cat @[email protected]
stdout-follow = tail -n 100 -f @[email protected] @[email protected]
Loading

0 comments on commit 0126ae0

Please sign in to comment.