CI: Update scripts and settings

.github/workflows/ci.yml

@@ -17,11 +17,12 @@ jobs:
   download-build-test:
     strategy:
       matrix:
-        accelerator: [cpu, cuda, rocm]
+        accelerator: [cpu, cuda, rocm, oneapi]
         real-precision: [real32, real64]
         mode: [debug, optimize]
         exclude:
           - {accelerator: rocm, real-precision: real32}
+          - {accelerator: oneapi, real-precision: real32}
           - {real-precision: real32, mode: optimize}
     env:
       ACCELERATOR: ${{matrix.accelerator}}
@@ -36,13 +37,13 @@ jobs:
     steps:
       # Check out the repository under $GITHUB_WORKSPACE, so our job can access it
       - uses: actions/checkout@v4
-          
+
       # # Check out the documentation and test results
       # - uses: actions/checkout@v3
       #   with:
       #     ref: gh-pages
       #     path: gh-pages
-          
+
       - name: Log the triggering commit
         run: echo ${{github.event.client_payload.trigger}}
 
@@ -51,12 +52,14 @@ jobs:
 
       - name: Build Cactus
         run: /bin/bash $GITHUB_WORKSPACE/scripts/build.sh
-
+        # The oneAPI setup does not build yet
+        continue-on-error: ${{matrix.accelerator == 'oneapi'}}
+
       - name: Test Cactus
-        # We cannot run with cuda nor with rocm. Debug builds run too slowly.
-        if: ${{matrix.accelerator != 'cuda' && matrix.accelerator != 'rocm' && matrix.mode != 'debug'}}
+        # We cannot run on accelerators. Debug builds run too slowly.
+        if: ${{matrix.accelerator == 'cpu' && matrix.mode == 'optimize'}}
         run: /bin/bash $GITHUB_WORKSPACE/scripts/test.sh
-      
+
       # Set exit code based on test status
       - name: Set exit code
         # Only look at the test results for `carpetx-cpu-real64` runs

Algo/src/roots.cxx

@@ -138,7 +138,16 @@ extern "C" void Test_roots(CCTK_ARGUMENTS) {
       const int maxiters = 100;
       int iters;
       bool failed;
-      auto x = newton_raphson_nd(gnd<CCTK_REAL>, vec<CCTK_REAL, 2>{1.0, 1.0},
+
+      // SYCL cannot call the templated function above, so we define a
+      // local copy
+      auto gnd_CCTK_REAL = [](vec<CCTK_REAL, 2> x)
+          -> std::pair<vec<CCTK_REAL, 2>, mat<CCTK_REAL, 2> > {
+        return {vec<CCTK_REAL, 2>{x(0) * x(0) - 2, x(0) * x(1) - 2},
+                mat<CCTK_REAL, 2>{2 * x(0), x(1), 0, x(0)}};
+      };
+
+      auto x = newton_raphson_nd(gnd_CCTK_REAL, vec<CCTK_REAL, 2>{1.0, 1.0},
                                  vec<CCTK_REAL, 2>{0.0, 0.0},
                                  vec<CCTK_REAL, 2>{10.0, 10.0}, minbits,
                                  maxiters, iters, failed);

TestSymmetries/src/init.cxx

@@ -275,7 +275,7 @@ extern "C" void TestSymmetries_Check(CCTK_ARGUMENTS) {
                   auto &var, const auto &value) CCTK_ATTRIBUTE_ALWAYS_INLINE {
         using std::abs;
         if (abs(var - value) > 10 * std::numeric_limits<CCTK_REAL>::epsilon()) {
-#if !defined __CUDACC__ && !defined __HIPCC__
+#if !defined __CUDACC__ && !defined __HIPCC__ && !defined SYCL_LANGUAGE_VERSION
           CCTK_VERROR(
               "Grid function symmetry check failed: I=[%d,%d,%d] X=[%g,%g,%g] "
               "centering=[%d,%d,%d] parity=[%d,%d,%d] var=%.17g value=%.17g",

docker/carpetx-arm64v8-cpu.dockerfile

@@ -7,8 +7,8 @@
 #     docker push einsteintoolkit/carpetx:arm64v8-cpu-real32
 
 # noble is ubuntu:24.04
-# FROM arm64v8/ubuntu:noble-20240423
-FROM arm64v8/ubuntu:noble-20240429
+# FROM arm64v8/ubuntu:noble-20240429
+FROM arm64v8/ubuntu:noble-20240530
 
 ENV DEBIAN_FRONTEND=noninteractive \
     LANGUAGE=en_US.en \

docker/carpetx-cpu.dockerfile

@@ -7,8 +7,8 @@
 #     docker push einsteintoolkit/carpetx:cpu-real32
 
 # noble is ubuntu:24.04
-# FROM ubuntu:noble-20240423
-FROM ubuntu:noble-20240429
+# FROM ubuntu:noble-20240429
+FROM ubuntu:noble-20240530
 
 ENV DEBIAN_FRONTEND=noninteractive \
     LANGUAGE=en_US.en \

docker/carpetx-oneapi.dockerfile

@@ -6,8 +6,8 @@
 #     docker build --build-arg real_precision=real32 --file carpetx-oneapi.dockerfile --tag einsteintoolkit/carpetx:oneapi-real32 .
 #     docker push einsteintoolkit/carpetx:oneapi-real32
 
-# FROM intel/oneapi-basekit:devel-ubuntu22.04
-FROM intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04
+# FROM intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04
+FROM intel/oneapi-basekit:2024.1.1-devel-ubuntu22.04
 
 ENV DEBIAN_FRONTEND=noninteractive \
     LANGUAGE=en_US.en \
@@ -66,6 +66,18 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 28DA432DAAC8BAEA &&
         && \
     rm -rf /var/lib/apt/lists/*
 
+# Remove troublesome libraries
+RUN find /opt/intel -name 'impi.pc' -delete && \
+    find /opt/intel -name 'libhwloc.a' -delete && \
+    find /opt/intel -name 'libhwloc.so' -delete && \
+    find /opt/intel -name 'libmpi*a' -delete && \
+    find /opt/intel -name 'libmpi*so' -delete && \
+    find /opt/intel -name 'mpi.h' -delete && \
+    find /opt/intel -name 'mpicc' -delete && \
+    find /opt/intel -name 'mpicxx' -delete && \
+    find /opt/intel -name 'mpiexec' -delete && \
+    find /opt/intel -name 'mpirun' -delete
+
 # Install blosc2
 # blosc2 is a compression library, comparable to zlib
 RUN mkdir src && \
@@ -247,10 +259,10 @@ ARG real_precision=real64
 # Should we keep the AMReX source tree around for debugging?
 RUN mkdir src && \
     (cd src && \
-    wget https://github.com/AMReX-Codes/amrex/archive/24.05.tar.gz && \
-    tar xzf 24.05.tar.gz && \
+    wget https://github.com/AMReX-Codes/amrex/archive/24.06.tar.gz && \
+    tar xzf 24.06.tar.gz && \
     rm -rf /opt/intel/oneapi/mpi && \
-    cd amrex-24.05 && \
+    cd amrex-24.06 && \
     case $real_precision in \
         real32) precision=SINGLE;; \
         real64) precision=DOUBLE;; \

docker/carpetx-rocm.dockerfile

@@ -6,8 +6,8 @@
 #     docker build --build-arg real_precision=real32 --file carpetx-rocm.dockerfile --tag einsteintoolkit/carpetx:rocm-real32 .
 #     docker push einsteintoolkit/carpetx:rocm-real32
 
-# FROM rocm/dev-ubuntu-22.04:6.1
-FROM rocm/dev-ubuntu-22.04:6.1.1
+# FROM rocm/dev-ubuntu-22.04:6.1.1
+FROM rocm/dev-ubuntu-22.04:6.1.2
 
 ENV DEBIAN_FRONTEND=noninteractive \
     LANGUAGE=en_US.en \

scripts/actions-cpu-real32.cfg

@@ -89,7 +89,7 @@ PTHREADS_DIR = NO_BUILD
 # REPRIMAND_LIBS = RePrimAnd
 # RNPLETAL_DIR = /usr/local
 SILO_DIR = /usr/local
-# SIMULATIONIO_DIR = /usr/local
+SIMULATIONIO_DIR = /usr/local
 SSHT_DIR = /usr/local
 YAML_CPP_DIR = /usr/local
 ZLIB_DIR = /usr

scripts/actions-cpu-real64.cfg

@@ -89,7 +89,7 @@ PTHREADS_DIR = NO_BUILD
 # REPRIMAND_LIBS = RePrimAnd
 # RNPLETAL_DIR = /usr/local
 SILO_DIR = /usr/local
-# SIMULATIONIO_DIR = /usr/local
+SIMULATIONIO_DIR = /usr/local
 SSHT_DIR = /usr/local
 YAML_CPP_DIR = /usr/local
 ZLIB_DIR = /usr

scripts/actions-cuda-real32.cfg

@@ -101,7 +101,7 @@ PTHREADS_DIR = NO_BUILD
 # REPRIMAND_LIBS = RePrimAnd
 # RNPLETAL_DIR = /usr/local
 SILO_DIR = /usr/local
-# SIMULATIONIO_DIR = /usr/local
+SIMULATIONIO_DIR = /usr/local
 SSHT_DIR = /usr/local
 YAML_CPP_DIR = /usr/local
 ZLIB_DIR = /usr

scripts/actions-cuda-real64.cfg

@@ -101,7 +101,7 @@ PTHREADS_DIR = NO_BUILD
 # REPRIMAND_LIBS = RePrimAnd
 # RNPLETAL_DIR = /usr/local
 SILO_DIR = /usr/local
-# SIMULATIONIO_DIR = /usr/local
+SIMULATIONIO_DIR = /usr/local
 SSHT_DIR = /usr/local
 YAML_CPP_DIR = /usr/local
 ZLIB_DIR = /usr

scripts/actions-oneapi-real64.cfg

@@ -0,0 +1,102 @@
+# Option list for the Einstein Toolkit
+
+# The "weird" options here should probably be made the default in the
+# ET instead of being set here.
+
+# Whenever this version string changes, the application is configured
+# and rebuilt from scratch
+VERSION = actions-oneapi-real64-2023-04-11
+
+CPP = cpp
+CC = /opt/intel/oneapi/compiler/2024.1/bin/icx
+CXX = /opt/intel/oneapi/compiler/2024.1/bin/icpx
+FPP = cpp
+FC = gfortran
+F90 = gfortran
+LD = /opt/intel/oneapi/compiler/2024.1/bin/icpx
+
+CPPFLAGS = -DSIMD_CPU
+CFLAGS = -fp-model=precise -g -march=x86-64-v3 -pipe -std=gnu11
+CXXFLAGS = -fp-model=precise -fsycl -g -march=x86-64-v3 -pipe -std=c++17
+FPPFLAGS = -traditional
+F90FLAGS = -fcray-pointer -ffixed-line-length-none -g -march=x86-64-v3 -pipe
+
+SYS_INC_DIRS = /opt/intel/oneapi/2024.1/include
+LIBDIRS = /usr/local/lib
+LIBS = gfortran
+
+C_LINE_DIRECTIVES = yes
+F_LINE_DIRECTIVES = yes
+
+INTEGER_PRECISION = 4
+REAL_PRECISION = 8
+
+# oneAPI does not support these
+# DISABLE_INT16 = yes
+DISABLE_REAL16 = yes
+
+DEBUG = no
+CPP_DEBUG_FLAGS = -DCARPET_DEBUG
+C_DEBUG_FLAGS = -fsanitize=undefined -fstack-protector-all -ftrapv
+CXX_DEBUG_FLAGS = -fsanitize=undefined -fstack-protector-all -ftrapv
+FPP_DEBUG_FLAGS = -DCARPET_DEBUG
+F90_DEBUG_FLAGS = -fcheck=bounds,do,mem,pointer,recursion -finit-character=65 -finit-integer=42424242 -finit-real=nan -ftrapv
+
+OPTIMISE = yes
+C_OPTIMISE_FLAGS = -O3 -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math
+CXX_OPTIMISE_FLAGS = -O3 -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math
+F90_OPTIMISE_FLAGS = -O3 -fcx-limited-range -fexcess-precision=fast -ffp-contract=fast -fno-math-errno -fno-rounding-math -fno-signaling-nans
+
+# Clang segfaults with OpenMP enabled
+OPENMP = no
+CPP_OPENMP_FLAGS = -fopenmp
+FPP_OPENMP_FLAGS = -D_OPENMP
+F90_OPENMP_FLAGS = -fopenmp
+
+WARN = yes
+CPP_WARN_FLAGS = -Wall
+C_WARN_FLAGS =
+CXX_WARN_FLAGS =
+FPP_WARN_FLAGS = -Wall
+F90_WARN_FLAGS = -Wall -Wshadow -Wsurprising
+
+VECTORISE = no
+
+ADIOS2_DIR = /usr/local
+ADIOS2_LIBS = adios2_cxx11_mpi adios2_cxx11
+AMREX_DIR = /usr/local
+# ASDF_CXX_DIR = /usr/local
+BLAS_DIR = /usr
+BOOST_DIR = /usr
+FFTW3_DIR = /usr
+GSL_DIR = /usr
+HDF5_DIR = /usr/lib/x86_64-linux-gnu/hdf5/serial
+HDF5_ENABLE_CXX = yes
+HDF5_ENABLE_FORTRAN = yes
+HDF5_INC_DIRS = /usr/lib/x86_64-linux-gnu/hdf5/serial/include
+HDF5_LIB_DIRS = /usr/lib/x86_64-linux-gnu/hdf5/serial/lib
+HDF5_LIBS = hdf5_hl_cpp hdf5_cpp hdf5hl_fortran hdf5_fortran hdf5_hl hdf5
+HWLOC_DIR = /usr
+# JEMALLOC_DIR = /usr/local
+LAPACK_DIR = /usr
+# LIBJPEG_DIR = /usr
+# LORENE_DIR = /usr/local
+MPI_DIR = /usr
+NSIMD_DIR = /usr/local
+NSIMD_SIMD = AVX2
+OPENBLAS_DIR = /usr
+OPENPMD_API_DIR = /usr/local
+# OPENSSL_DIR = /usr
+# PAPI_DIR = /usr/local
+PETSC_DIR = /usr/lib/petsc
+PETSC_LIBS = petsc_real
+PETSC_ARCH_LIBS = m
+PTHREADS_DIR = NO_BUILD
+# REPRIMAND_DIR = /usr/local
+# REPRIMAND_LIBS = RePrimAnd
+# RNPLETAL_DIR = /usr/local
+SILO_DIR = /usr/local
+SIMULATIONIO_DIR = /usr/local
+SSHT_DIR = /usr/local
+YAML_CPP_DIR = /usr/local
+ZLIB_DIR = /usr

scripts/actions-oneapi-real64.ini

@@ -0,0 +1,72 @@
+[actions-oneapi-real64]
+
+# Machine description
+nickname        = actions-oneapi-real64
+name            = Actions
+location        = Github
+description     = Github action for CarpetX CI
+webpage         = https://github.com/eschnett/CarpetX
+status          = production
+
+# Access to this machine
+hostname        = actions
+aliaspattern    = ^actions$
+# envsetup        = <<EOT
+# EOT
+
+# Source tree management
+sourcebasedir   = /__w/CarpetX/workspace
+optionlist      = actions-oneapi-real64.cfg
+submitscript    = actions-oneapi-real64.sub
+runscript       = actions-oneapi-real64.run
+make            = make -j @MAKEJOBS@
+makejobs        = 2
+disabled-thorns = <<EOT
+    # Not needed with oneAPI
+    ExternalLibraries/NSIMD
+    # long double exists but does not quite work
+    CactusTest/TestMath
+
+    ExternalLibraries/BLAS
+    ExternalLibraries/LAPACK
+    ExternalLibraries/LORENE
+    ExternalLibraries/PAPI
+EOT
+enabled-thorns = <<EOT
+    ExternalLibraries/OpenBLAS
+    ExternalLibraries/PETSc
+EOT
+
+# Simulation management
+basedir         = /__w/CarpetX/workspace/simulations
+quota           = 10            # don't use all disk space
+cpu             = Intel Core i9
+cpufreq         = 2.4
+flop/cycle      = 16
+ppn             = 8
+spn             = 1
+mpn             = 1
+max-num-threads = 16
+num-threads     = 8
+max-num-smt     = 2
+num-smt         = 1
+memory          = 32768
+nodes           = 1
+min-ppn         = 1             # don't want this entry
+queue           = NOQUEUE       # don't want this entry
+allocation      = NOALLOCATION  # don't want this entry
+maxwalltime     = 8640:00:00    # don't want this entry
+# submit          = exec @SCRIPTFILE@ </dev/null >/dev/null 2>/dev/null & echo $!
+submit          = @SCRIPTFILE@ </dev/null >/dev/null 2>/dev/null; true & echo $!
+getstatus       = ps @JOB_ID@
+stop            = kill @JOB_ID@
+submitpattern   = (.*)
+statuspattern   = "^ *@JOB_ID@ "
+queuedpattern   = $^
+runningpattern  = ^
+holdingpattern  = $^
+exechost        = echo localhost
+exechostpattern = (.*)
+stdout          = cat @[email protected]
+stderr          = cat @[email protected]
+stdout-follow   = tail -n 100 -f @[email protected] @[email protected]