Add development container

.gitlab/image.yml

@@ -100,3 +100,8 @@
   tags:
     - private_ci
     - intel-dgpu
+
+.use_gko-unified:
+  image: ginkgodev/unified:latest
+  tags:
+    - unified

cuda/matrix/csr_kernels.cu

@@ -407,10 +407,9 @@ bool try_general_sparselib_spmv(std::shared_ptr<const CudaExecutor> exec,
         cusparse::destroy(vecc);
     } else {
 #if CUDA_VERSION >= 11060
-        if (b->get_size()[1] == 1 && exec->get_major_version() >= 7) {
+        if (b->get_size()[1] == 1) {
             // cusparseSpMM seems to take the single strided vector as column
-            // major without considering stride and row major (SM >= 70 and
-            // cuda 11.6)
+            // major without considering stride and row major (cuda 11.6)
             return false;
         }
 #endif  // CUDA_VERSION >= 11060

dev_tools/container/base/Dockerfile

@@ -0,0 +1,78 @@
+FROM ubuntu:22.04
+
+# install spack prerequisites and basics
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    coreutils \
+    curl \
+    environment-modules \
+    gfortran \
+    git \
+    lsb-release \
+    python3 \
+    python3-distutils \
+    python3-venv \
+    unzip \
+    zip \
+    vim-nox \
+    wget \
+    pkg-config \
+    ninja-build \
+    # for ginkgo
+    libopencv-dev \
+    # for rocm
+    gnupg2 \
+    gawk \
+    # for spack
+    perl \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN (echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main" > /etc/apt/sources.list.d/llvm15.list) && \
+    (curl https://apt.llvm.org/llvm-snapshot.gpg.key > /etc/apt/trusted.gpg.d/apt.llvm.org.asc) && apt-get update && \
+    apt-get install -y libllvm-15-ocaml-dev libllvm15 llvm-15 llvm-15-dev llvm-15-doc llvm-15-examples llvm-15-runtime \
+    clang-15 clang-tools-15 clang-15-doc libclang-common-15-dev libclang-15-dev libclang1-15 clang-format-15 python3-clang-15 \
+    clangd-15 clang-tidy-15 lldb-15 lld-15 libc++-15-dev libc++abi-15-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && \
+    wget https://repo.radeon.com/amdgpu-install/5.5/ubuntu/jammy/amdgpu-install_5.5.50500-1_all.deb && \
+    apt-get install -y --no-install-recommends ./amdgpu-install_5.5.50500-1_all.deb && \
+    apt-get update && apt-get install -y --no-install-recommends \
+    rocm-dev hipblas-dev hipfft-dev hipsparse-dev rocfft-dev rocrand-dev rocsolver-dev rocthrust-dev roctracer-dev && \
+    rm ./amdgpu-install_5.5.50500-1_all.deb && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN (echo "deb https://apt.repos.intel.com/oneapi all main" > /etc/apt/sources.list.d/oneapi.list) && \
+    (curl https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+    > /etc/apt/trusted.gpg.d/apt.repos.intel.com.asc) && apt-get update && \
+    apt-get install -y \
+    intel-oneapi-ccl-devel \
+    intel-oneapi-common-licensing \
+    intel-oneapi-common-vars \
+    intel-oneapi-compiler-dpcpp-cpp \
+    intel-oneapi-dev-utilities \
+    intel-oneapi-dpcpp-ct \
+    intel-oneapi-dpcpp-debugger \
+    intel-oneapi-ipp-devel \
+    intel-oneapi-libdpstd-devel \
+    intel-oneapi-mkl-devel \
+    intel-oneapi-tbb-devel \
+    intel-oneapi-vtune \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN cd /usr/ && wget -O cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.26.4/cmake-3.26.4-linux-x86_64.tar.gz && \
+    tar xzf cmake.tar.gz --strip-components 1 && rm cmake.tar.gz
+
+RUN cd /tmp && wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.13822.6/intel-igc-core_1.0.13822.6_amd64.deb && \
+    wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.13822.6/intel-igc-opencl_1.0.13822.6_amd64.deb && \
+    wget https://github.com/intel/compute-runtime/releases/download/23.17.26241.22/intel-level-zero-gpu-dbgsym_1.3.26241.22_amd64.ddeb && \
+    wget https://github.com/intel/compute-runtime/releases/download/23.17.26241.22/intel-level-zero-gpu_1.3.26241.22_amd64.deb && \
+    wget https://github.com/intel/compute-runtime/releases/download/23.17.26241.22/intel-opencl-icd-dbgsym_23.17.26241.22_amd64.ddeb && \
+    wget https://github.com/intel/compute-runtime/releases/download/23.17.26241.22/intel-opencl-icd_23.17.26241.22_amd64.deb && \
+    wget https://github.com/intel/compute-runtime/releases/download/23.17.26241.22/libigdgmm12_22.3.0_amd64.deb && \
+    dpkg -i *.deb *.ddeb && rm *.deb *.ddeb
+
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /

dev_tools/container/prebuilt/Dockerfile

@@ -0,0 +1,32 @@
+FROM ginkgodev/unified:latest
+
+SHELL ["/root/entrypoint.sh", "-c"]
+
+RUN git clone https://github.com/ginkgo-project/ginkgo && cd ginkgo && mkdir build && cd build && \
+    cmake .. \
+    -DCMAKE_BUILD_TYPE=Debug \
+    -DCMAKE_LINK_DEPENDS_NO_SHARED=ON \
+    -DCMAKE_CXX_COMPILER=g++ \
+    -DCMAKE_C_COMPILER=gcc \
+    -DCMAKE_CUDA_COMPILER=`spack location -i cuda`/bin/nvcc \
+    #-DCMAKE_CXX_FLAGS= \
+    #-DCMAKE_C_FLAGS= \
+    #-DCMAKE_CUDA_FLAGS= \
+    -DGINKGO_HIP_AMDGPU=gfx906 \
+    -DCMAKE_CUDA_ARCHITECTURES=OFF \
+    -DGINKGO_CUDA_ARCHITECTURES='61;70' \
+    -DGINKGO_BUILD_OMP=ON \
+    -DGINKGO_BUILD_CUDA=ON \
+    -DGINKGO_BUILD_HIP=ON \
+    -DGINKGO_BUILD_MPI=ON \
+    -DGINKGO_BUILD_TESTS=ON \
+    -DGINKGO_BUILD_EXAMPLES=ON \
+    -DGINKGO_BUILD_BENCHMARKS=ON \
+    -DGINKGO_MIXED_PRECISION=ON \
+    -DGINKGO_DEVEL_TOOLS=ON \
+    -DGINKGO_WITH_CCACHE=ON \
+    -GNinja
+
+RUN cd ginkgo/build && ninja install
+
+WORKDIR /root/ginkgo

dev_tools/container/unified/Dockerfile

@@ -0,0 +1,32 @@
+FROM ginkgodev/base:latest
+
+RUN git clone --depth=1 -c feature.manyFiles=true https://github.com/spack/spack.git /spack
+
+RUN /spack/bin/spack compiler find && \
+    /spack/bin/spack external find --not-buildable opencv openssh perl llvm cmake ninja && \
+    /spack/bin/spack env create ginkgo && \
+    /spack/bin/spack bootstrap now && \
+    /spack/bin/spack gpg create buildcache build@cache
+
+COPY spack.yaml /spack/var/spack/environments/ginkgo/spack.yaml
+COPY packages.yaml /spack/etc/spack/packages.yaml
+COPY entrypoint.sh /usr/bin/
+
+ENV CMAKE_GENERATOR=Ninja CMAKE_EXPORT_COMPILE_COMMANDS=ON 
+ENV OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+RUN --mount=type=cache,target=/cache \
+    . /spack/share/spack/setup-env.sh && \
+    if [ -d "/cache/build_cache" ]; then \
+        echo Reusing buildcache && \
+        spack mirror add buildcache /cache && \
+        spack buildcache keys --install --trust; \
+    fi && \
+    spack env activate ginkgo && \
+    spack install --use-cache --reuse && \
+    spack buildcache create -a /cache
+
+# remove the generated key to prevent malicious package injection
+RUN rm -rf /spack/opt/spack/gpg
+
+ENTRYPOINT ["/usr/bin/entrypoint.sh"]

dev_tools/container/unified/build.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker build -t ginkgodev/unified $(dirname -- $0)

dev_tools/container/unified/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+. /spack/share/spack/setup-env.sh
+spack env activate ginkgo
+export CC=gcc CXX=g++ CUDACXX=`spack location -i cuda`/bin/nvcc PATH=$PATH:`spack location -i cuda`/bin
+exec "$@"

dev_tools/container/unified/packages.yaml

@@ -0,0 +1,63 @@
+packages:
+  all:
+    target: ['x86_64']
+  hip:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.0
+    buildable: false
+  hipblas:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.0
+    buildable: false
+  hipsparse:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.0
+    buildable: false
+  hipfft:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.0
+    buildable: false
+  rocrand:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.0
+    buildable: false
+  rocthrust:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/rocm-5.5.0
+    buildable: false
+  intel-oneapi-compilers:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/intel/oneapi
+    buildable: false
+  intel-oneapi-mkl:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/intel/oneapi
+    buildable: false
+  intel-oneapi-vtune:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/intel/oneapi
+    buildable: false
+  intel-oneapi-tbb:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/intel/oneapi
+    buildable: false
+  intel-oneapi-dpl:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/intel/oneapi
+    buildable: false
+  intel-oneapi-dpct:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/intel/oneapi
+    buildable: false

dev_tools/container/unified/spack.yaml

@@ -0,0 +1,61 @@
+spack:
+  specs:
+    # ginkgo
+    - cmake
+    - ninja
+    - googletest +gmock cxxstd=14
+    - rapidjson
+    - gflags
+    # ginkgo examples and features
+    - papi +sde
+    - perfstubs
+    - tau
+    - kokkos
+    - "hwloc @2.0:"
+    - benchmark
+    - opencv +videoio
+    - python
+    # ginkgo_mpi
+    - "openmpi @4.1: +cuda"
+    # development environment
+    - valgrind
+    - gdb +source-highlight +tui +python
+    - git
+    - ccache
+    - llvm
+    # ginkgo_cuda
+    - cuda
+    # ginkgo_hip
+    - hip
+    - hipsparse
+    - hipblas
+    - hipfft
+    - rocrand
+    - rocthrust
+    # ginkgo_dpcpp
+    - intel-oneapi-compilers
+    - intel-oneapi-mkl
+    - intel-oneapi-vtune
+    - intel-oneapi-tbb
+    - intel-oneapi-dpl
+    - intel-oneapi-dpct
+    # build dependencies of the above packages,
+    # which would otherwise be missing from the cache
+    - pkgconf
+    - libsigsegv
+    - util-macros
+    - diffutils
+    - m4
+    - libtool
+    - bison
+    - autoconf
+    - automake
+    - texinfo
+  view:
+    default:
+      root: /spack/env/
+      # we need to exclude CUDA from the view,
+      # since it contains Thrust, which conflicts with rocThrust
+      exclude: ["cuda"]
+  concretizer:
+    unify: true

examples/CMakeLists.txt

@@ -40,13 +40,6 @@ if(GINKGO_BUILD_EXTLIB_EXAMPLE)
     list(APPEND EXAMPLES_LIST external-lib-interfacing)
 endif()
 
-find_package(OpenCV QUIET)
-if(OpenCV_FOUND)
-    list(APPEND EXAMPLES_LIST heat-equation schroedinger-splitting)
-else()
-    message(STATUS "No OpenCV found, disabling examples with video output")
-endif()
-
 if(GINKGO_HAVE_PAPI_SDE)
     list(APPEND EXAMPLES_LIST papi-logging)
 endif()

examples/kokkos_assembly/kokkos_assembly.cpp

@@ -124,9 +124,8 @@ double calculate_error(int discretization_points,
         KOKKOS_LAMBDA(int i, double& lsum) {
             const auto h = 1.0 / (discretization_points + 1);
             const auto xi = (i + 1) * h;
-            lsum += Kokkos::Experimental::abs(
-                (v_u(i) - correct_u(xi)) /
-                Kokkos::Experimental::abs(correct_u(xi)));
+            lsum += Kokkos::abs((v_u(i) - correct_u(xi)) /
+                                Kokkos::abs(correct_u(xi)));
         },
         error);
     return error;

include/ginkgo/core/base/math.hpp

@@ -47,13 +47,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <ginkgo/core/base/utils.hpp>
 
 
-// Using SYCL_LANGUAGE_VERSION will lead the mismatch sycl namespace from 6.0.0
-// when using dpcpp compiler without dpcpp module
-#if GINKGO_DPCPP_MAJOR_VERSION
-#include <CL/sycl.hpp>
-#endif
-
-
 namespace gko {
 
 

test/matrix/fft_kernels.cpp

@@ -138,6 +138,9 @@ TYPED_TEST(Fft, Apply1DIsEqualToReference)
 
 TYPED_TEST(Fft, ApplyStrided1DIsEqualToReference)
 {
+#if defined(GKO_COMPILING_HIP) && GINKGO_HIP_PLATFORM_HCC
+    GTEST_SKIP() << "rocFFT has a bug related to strided 1D FFT";
+#endif
     using T = typename TestFixture::value_type;
 
     this->fft->apply(this->data_strided, this->out_strided);
@@ -160,6 +163,9 @@ TYPED_TEST(Fft, Apply1DInverseIsEqualToReference)
 
 TYPED_TEST(Fft, ApplyStrided1DInverseIsEqualToReference)
 {
+#if defined(GKO_COMPILING_HIP) && GINKGO_HIP_PLATFORM_HCC
+    GTEST_SKIP() << "rocFFT has a bug related to strided 1D FFT";
+#endif
     using T = typename TestFixture::value_type;
 
     this->ifft->apply(this->data_strided, this->out_strided);

test/mpi/CMakeLists.txt

@@ -1,2 +1 @@
 add_subdirectory(distributed)
-add_subdirectory(solver)

test/mpi/distributed/CMakeLists.txt

@@ -1,4 +1,5 @@
-ginkgo_create_common_and_reference_test(matrix MPI_SIZE 3)
-ginkgo_create_common_and_reference_test(vector MPI_SIZE 3)
+ginkgo_create_common_and_reference_test(matrix MPI_SIZE 3 DISABLE_EXECUTORS dpcpp)
+ginkgo_create_common_and_reference_test(vector MPI_SIZE 3 DISABLE_EXECUTORS dpcpp)
 
 add_subdirectory(preconditioner)
+add_subdirectory(solver)

test/mpi/distributed/preconditioner/CMakeLists.txt

@@ -1 +1 @@
-ginkgo_create_common_and_reference_test(schwarz MPI_SIZE 3)
+ginkgo_create_common_and_reference_test(schwarz MPI_SIZE 3 DISABLE_EXECUTORS dpcpp)

test/mpi/distributed/solver/CMakeLists.txt

@@ -0,0 +1 @@
+ginkgo_create_common_and_reference_test(solver MPI_SIZE 3 DISABLE_EXECUTORS dpcpp)

test/solver/idr_kernels.cpp

@@ -40,6 +40,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <gtest/gtest.h>
 
 
+#ifdef GKO_COMPILING_DPCPP
+#include <CL/sycl.hpp>
+#endif
+
+
 #include <ginkgo/core/base/array.hpp>
 #include <ginkgo/core/base/exception.hpp>
 #include <ginkgo/core/base/executor.hpp>