From 3173c899477d9f5d07ba2e7b0e70ff029719caf9 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 30 Oct 2024 14:39:56 -0400 Subject: [PATCH] CI: Update Docker images --- docker/carpetx-arm64v8-cpu.dockerfile | 39 +++++++++++++++++++----- docker/carpetx-cpu.dockerfile | 39 +++++++++++++++++++----- docker/carpetx-cuda.dockerfile | 42 +++++++++++++++++++++----- docker/carpetx-oneapi.dockerfile | 42 +++++++++++++++++++++----- docker/carpetx-rocm.dockerfile | 43 ++++++++++++++++++++++----- 5 files changed, 167 insertions(+), 38 deletions(-) diff --git a/docker/carpetx-arm64v8-cpu.dockerfile b/docker/carpetx-arm64v8-cpu.dockerfile index c7912bcd6..3a4c9a3a3 100644 --- a/docker/carpetx-arm64v8-cpu.dockerfile +++ b/docker/carpetx-arm64v8-cpu.dockerfile @@ -8,7 +8,8 @@ # noble is ubuntu:24.04 # FROM arm64v8/ubuntu:noble-20240801 -FROM arm64v8/ubuntu:noble-20240904.1 +# FROM arm64v8/ubuntu:noble-20240904.1 +FROM arm64v8/ubuntu:noble-20241011 ENV DEBIAN_FRONTEND=noninteractive \ LANGUAGE=en_US.en \ @@ -54,6 +55,7 @@ RUN apt-get update && \ libopenmpi-dev \ libpapi-dev \ libpetsc-real-dev \ + libprotobuf-dev \ libtool \ libudev-dev \ libyaml-cpp-dev \ @@ -68,6 +70,7 @@ RUN apt-get update && \ patch \ perl \ pkgconf \ + protobuf-compiler \ python3 \ python3-pip \ python3-requests \ @@ -135,14 +138,35 @@ RUN mkdir src && \ true) && \ rm -rf src +# Install MGARD +# MGARD is a lossy compression library +RUN mkdir src && \ + (cd src && \ + wget https://github.com/CODARcode/MGARD/archive/refs/tags/1.5.2.tar.gz && \ + tar xzf 1.5.2.tar.gz && \ + cd MGARD-1.5.2 && \ + cmake -B build -G Ninja \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DMGARD_ENABLE_OPENMP=ON \ + -DMGARD_ENABLE_SERIAL=ON \ + && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + # Install ADIOS2 # ADIOS2 is a parallel I/O library, comparable to HDF5 # - depends on blosc2 +# - depends on MGARD RUN mkdir src && \ (cd src && \ - wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.10.1.tar.gz && \ - tar xzf v2.10.1.tar.gz && \ - cd ADIOS2-2.10.1 && \ + wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.10.2.tar.gz && \ + tar xzf v2.10.2.tar.gz && \ + cd ADIOS2-2.10.2 && \ cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ @@ -152,6 +176,7 @@ RUN mkdir src && \ -DADIOS2_Blosc2_PREFER_SHARED=ON \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_MGARD=ON \ && \ cmake --build build && \ cmake --install build && \ @@ -199,9 +224,9 @@ RUN mkdir src && \ # - depends on ADIOS2 RUN mkdir src && \ (cd src && \ - wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.15.2.tar.gz && \ - tar xzf 0.15.2.tar.gz && \ - cd openPMD-api-0.15.2 && \ + wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.16.0.tar.gz && \ + tar xzf 0.16.0.tar.gz && \ + cd openPMD-api-0.16.0 && \ cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ diff --git a/docker/carpetx-cpu.dockerfile b/docker/carpetx-cpu.dockerfile index 57d2223de..efa618f5f 100644 --- a/docker/carpetx-cpu.dockerfile +++ b/docker/carpetx-cpu.dockerfile @@ -8,7 +8,8 @@ # noble is ubuntu:24.04 # FROM amd64/ubuntu:noble-20240801 -FROM amd64/ubuntu:noble-20240904.1 +# FROM amd64/ubuntu:noble-20240904.1 +FROM amd64/ubuntu:noble-20241011 ENV DEBIAN_FRONTEND=noninteractive \ LANGUAGE=en_US.en \ @@ -54,6 +55,7 @@ RUN apt-get update && \ libopenmpi-dev \ libpapi-dev \ libpetsc-real-dev \ + libprotobuf-dev \ libtool \ libudev-dev \ libyaml-cpp-dev \ @@ -68,6 +70,7 @@ RUN apt-get update && \ patch \ perl \ pkgconf \ + protobuf-compiler \ python3 \ python3-pip \ python3-requests \ @@ -135,14 +138,35 @@ RUN mkdir src && \ true) && \ rm -rf src +# Install MGARD +# MGARD is a lossy compression library +RUN mkdir src && \ + (cd src && \ + wget https://github.com/CODARcode/MGARD/archive/refs/tags/1.5.2.tar.gz && \ + tar xzf 1.5.2.tar.gz && \ + cd MGARD-1.5.2 && \ + cmake -B build -G Ninja \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DMGARD_ENABLE_OPENMP=ON \ + -DMGARD_ENABLE_SERIAL=ON \ + && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + # Install ADIOS2 # ADIOS2 is a parallel I/O library, comparable to HDF5 # - depends on blosc2 +# - depends on MGARD RUN mkdir src && \ (cd src && \ - wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.10.1.tar.gz && \ - tar xzf v2.10.1.tar.gz && \ - cd ADIOS2-2.10.1 && \ + wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.10.2.tar.gz && \ + tar xzf v2.10.2.tar.gz && \ + cd ADIOS2-2.10.2 && \ cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ @@ -152,6 +176,7 @@ RUN mkdir src && \ -DADIOS2_Blosc2_PREFER_SHARED=ON \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_MGARD=ON \ && \ cmake --build build && \ cmake --install build && \ @@ -199,9 +224,9 @@ RUN mkdir src && \ # - depends on ADIOS2 RUN mkdir src && \ (cd src && \ - wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.15.2.tar.gz && \ - tar xzf 0.15.2.tar.gz && \ - cd openPMD-api-0.15.2 && \ + wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.16.0.tar.gz && \ + tar xzf 0.16.0.tar.gz && \ + cd openPMD-api-0.16.0 && \ cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ diff --git a/docker/carpetx-cuda.dockerfile b/docker/carpetx-cuda.dockerfile index 4b7fe33e8..5e417daba 100644 --- a/docker/carpetx-cuda.dockerfile +++ b/docker/carpetx-cuda.dockerfile @@ -6,8 +6,9 @@ # docker build --build-arg real_precision=real32 --file carpetx-cuda.dockerfile --tag einsteintoolkit/carpetx:cuda-real32 . # docker push einsteintoolkit/carpetx:cuda-real32 -# FROM amd64/nvidia/cuda:12.5.1-devel-ubuntu24.04 -FROM amd64/nvidia/cuda:12.6.1-devel-ubuntu24.04 +# FROM nvidia/cuda:12.5.1-devel-ubuntu24.04 +# FROM nvidia/cuda:12.6.1-devel-ubuntu24.04 +FROM nvidia/cuda:12.6.2-devel-ubuntu24.04 ENV DEBIAN_FRONTEND=noninteractive \ LANGUAGE=en_US.en \ @@ -48,6 +49,7 @@ RUN apt-get update && \ libopenblas-dev \ libopenmpi-dev \ libpetsc-real-dev \ + libprotobuf-dev \ libtool \ libudev-dev \ libyaml-cpp-dev \ @@ -59,6 +61,7 @@ RUN apt-get update && \ numactl \ perl \ pkgconf \ + protobuf-compiler \ python3 \ python3-pip \ python3-requests \ @@ -123,14 +126,36 @@ RUN mkdir src && \ true) && \ rm -rf src +# Install MGARD +# MGARD is a lossy compression library +# Note: -DMGARD_ENABLE_CUDA=ON requires nvcomp with a restrictive licence +RUN mkdir src && \ + (cd src && \ + wget https://github.com/CODARcode/MGARD/archive/refs/tags/1.5.2.tar.gz && \ + tar xzf 1.5.2.tar.gz && \ + cd MGARD-1.5.2 && \ + cmake -B build -G Ninja \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DMGARD_ENABLE_OPENMP=ON \ + -DMGARD_ENABLE_SERIAL=ON \ + && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + # Install ADIOS2 # ADIOS2 is a parallel I/O library, comparable to HDF5 # - depends on blosc2 +# - depends on MGARD RUN mkdir src && \ (cd src && \ - wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.10.1.tar.gz && \ - tar xzf v2.10.1.tar.gz && \ - cd ADIOS2-2.10.1 && \ + wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.10.2.tar.gz && \ + tar xzf v2.10.2.tar.gz && \ + cd ADIOS2-2.10.2 && \ cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ @@ -140,6 +165,7 @@ RUN mkdir src && \ -DADIOS2_Blosc2_PREFER_SHARED=ON \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_MGARD=ON \ && \ cmake --build build && \ cmake --install build && \ @@ -187,9 +213,9 @@ RUN mkdir src && \ # - depends on ADIOS2 RUN mkdir src && \ (cd src && \ - wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.15.2.tar.gz && \ - tar xzf 0.15.2.tar.gz && \ - cd openPMD-api-0.15.2 && \ + wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.16.0.tar.gz && \ + tar xzf 0.16.0.tar.gz && \ + cd openPMD-api-0.16.0 && \ cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ diff --git a/docker/carpetx-oneapi.dockerfile b/docker/carpetx-oneapi.dockerfile index 4a73d8e2d..599bf119e 100644 --- a/docker/carpetx-oneapi.dockerfile +++ b/docker/carpetx-oneapi.dockerfile @@ -6,8 +6,8 @@ # docker build --build-arg real_precision=real32 --file carpetx-oneapi.dockerfile --tag einsteintoolkit/carpetx:oneapi-real32 . # docker push einsteintoolkit/carpetx:oneapi-real32 -# FROM amd64/intel/oneapi-basekit:2024.2.0-1-devel-ubuntu22.04 -FROM amd64/intel/oneapi-basekit:2024.2.1-0-devel-ubuntu22.04 +# FROM intel/oneapi-basekit:2024.2.0-1-devel-ubuntu22.04 +FROM intel/oneapi-basekit:2024.2.1-0-devel-ubuntu22.04 ENV DEBIAN_FRONTEND=noninteractive \ LANGUAGE=en_US.en \ @@ -47,6 +47,7 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 28DA432DAAC8BAEA && libopenblas-dev \ libopenmpi-dev \ libpetsc-real-dev \ + libprotobuf-dev \ libtool \ libudev-dev \ libyaml-cpp-dev \ @@ -58,6 +59,7 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 28DA432DAAC8BAEA && numactl \ perl \ pkgconf \ + protobuf-compiler \ python3 \ python3-pip \ python3-requests \ @@ -102,13 +104,36 @@ RUN mkdir src && \ true) && \ rm -rf src +# Install MGARD +# MGARD is a lossy compression library +# Note: -DMGARD_ENABLE_SYCL=ON does not work +RUN mkdir src && \ + (cd src && \ + wget https://github.com/CODARcode/MGARD/archive/refs/tags/1.5.2.tar.gz && \ + tar xzf 1.5.2.tar.gz && \ + cd MGARD-1.5.2 && \ + cmake -B build -G Ninja \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DMGARD_ENABLE_OPENMP=ON \ + -DMGARD_ENABLE_SERIAL=ON \ + && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + # Install ADIOS2 # ADIOS2 is a parallel I/O library, comparable to HDF5 +# - depends on blosc2 +# - depends on MGARD RUN mkdir src && \ (cd src && \ - wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.10.1.tar.gz && \ - tar xzf v2.10.1.tar.gz && \ - cd ADIOS2-2.10.1 && \ + wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.10.2.tar.gz && \ + tar xzf v2.10.2.tar.gz && \ + cd ADIOS2-2.10.2 && \ cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ @@ -118,6 +143,7 @@ RUN mkdir src && \ -DADIOS2_Blosc2_PREFER_SHARED=ON \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_MGARD=ON \ && \ cmake --build build && \ cmake --install build && \ @@ -165,9 +191,9 @@ RUN mkdir src && \ # - depends on ADIOS2 RUN mkdir src && \ (cd src && \ - wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.15.2.tar.gz && \ - tar xzf 0.15.2.tar.gz && \ - cd openPMD-api-0.15.2 && \ + wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.16.0.tar.gz && \ + tar xzf 0.16.0.tar.gz && \ + cd openPMD-api-0.16.0 && \ cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ diff --git a/docker/carpetx-rocm.dockerfile b/docker/carpetx-rocm.dockerfile index d1e64b6db..b979a6dfa 100644 --- a/docker/carpetx-rocm.dockerfile +++ b/docker/carpetx-rocm.dockerfile @@ -6,8 +6,9 @@ # docker build --build-arg real_precision=real32 --file carpetx-rocm.dockerfile --tag einsteintoolkit/carpetx:rocm-real32 . # docker push einsteintoolkit/carpetx:rocm-real32 -# FROM amd64/rocm/dev-ubuntu-22.04:6.1.2 -FROM amd64/rocm/dev-ubuntu-24.04:6.2 +# FROM rocm/dev-ubuntu-22.04:6.1.2 +# FROM rocm/dev-ubuntu-24.04:6.2 +FROM rocm/dev-ubuntu-24.04:6.2 ENV DEBIAN_FRONTEND=noninteractive \ LANGUAGE=en_US.en \ @@ -49,6 +50,7 @@ RUN apt-get update && \ libopenblas-dev \ libopenmpi-dev \ libpetsc-real-dev \ + libprotobuf-dev \ libtool \ libudev-dev \ libyaml-cpp-dev \ @@ -60,6 +62,7 @@ RUN apt-get update && \ numactl \ perl \ pkgconf \ + protobuf-compiler \ python3 \ python3-pip \ python3-requests \ @@ -126,13 +129,36 @@ RUN mkdir src && \ true) && \ rm -rf src +# Install MGARD +# MGARD is a lossy compression library +# Note: -DMGARD_ENABLE_HIP=ON doesn't work +RUN mkdir src && \ + (cd src && \ + wget https://github.com/CODARcode/MGARD/archive/refs/tags/1.5.2.tar.gz && \ + tar xzf 1.5.2.tar.gz && \ + cd MGARD-1.5.2 && \ + cmake -B build -G Ninja \ + -DBUILD_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_PREFIX_PATH=/usr/local \ + -DMGARD_ENABLE_OPENMP=ON \ + -DMGARD_ENABLE_SERIAL=ON \ + && \ + cmake --build build && \ + cmake --install build && \ + true) && \ + rm -rf src + # Install ADIOS2 # ADIOS2 is a parallel I/O library, comparable to HDF5 +# - depends on blosc2 +# - depends on MGARD RUN mkdir src && \ (cd src && \ - wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.10.1.tar.gz && \ - tar xzf v2.10.1.tar.gz && \ - cd ADIOS2-2.10.1 && \ + wget https://github.com/ornladios/ADIOS2/archive/refs/tags/v2.10.2.tar.gz && \ + tar xzf v2.10.2.tar.gz && \ + cd ADIOS2-2.10.2 && \ cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \ @@ -142,6 +168,7 @@ RUN mkdir src && \ -DADIOS2_Blosc2_PREFER_SHARED=ON \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_MGARD=ON \ && \ cmake --build build && \ cmake --install build && \ @@ -189,9 +216,9 @@ RUN mkdir src && \ # - depends on ADIOS2 RUN mkdir src && \ (cd src && \ - wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.15.2.tar.gz && \ - tar xzf 0.15.2.tar.gz && \ - cd openPMD-api-0.15.2 && \ + wget https://github.com/openPMD/openPMD-api/archive/refs/tags/0.16.0.tar.gz && \ + tar xzf 0.16.0.tar.gz && \ + cd openPMD-api-0.16.0 && \ cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_INSTALL_PREFIX=/usr/local \