qe7.3.1 for gh200 (#92)

config.yaml

@@ -187,6 +187,11 @@ uenvs:
         a100: v7.1/a100
       deploy:
         eiger: [zen2]
+    "v7.3.1":
+      recipes:
+        gh200: v7.3.1/gh200
+      deploy:
+        santis: [gh200]
   sirius:
     "v7.5":
       recipes:

docs/uenv-linaro-forge.md

@@ -135,7 +135,7 @@ Open the *Remote Launch* menu and click on *configure* then *Add*. Examples of t
 
 Some notes on the examples above:
 
-* SSH Forwarding via `ela.scscs.ch` is used to access the cluster.
+* SSH Forwarding via `ela.cscs.ch` is used to access the cluster.
 * the replace the username `bsmith` with your CSCS user name that you would normally use to open an SSH connection to CSCS.
 * the Remote Installation Path is a little bit more complicated than
 * the private keys should be the ones generated for CSCS MFA, and this field does not need to be set if you have added the key to your SSH agent.

docs/uenv-qe.md

@@ -1,42 +1,148 @@
 # Quantum ESPRESSO
 
-https://www.quantum-espresso.org/
+https://www.quantum-espresso.org
 
 An environment that provides the latest version of Quantum ESPRESSO, along with the libraries and tools required to build a different or custom version of Quantum ESPRESSO.
 At the moment a GPU-build environment is provided without a ScaLAPACK.
 
 The following environment views are provided:
 
-* default : QuantumESPRESSO/7.1 itself + dependencies
-* develop : only dependencies
+=== "GH200"
 
-The following modules are provided:
+     * default : QuantumESPRESSO/7.3.1 itself + dependencies
+     * develop : only dependencies
 
-* cmake/3.26.3
-* cray-mpich/8.1.25-nvhpc
-* cuda/11.8.0
-* fftw/3.3.10
-* gcc/11.3.0
-* libxc/5.2.3
-* nvhpc/22.11
-* openblas/0.3.23
-* patchelf/0.17.2
-* quantum-espresso/7.1
+     The following modules are provided:
 
-## Building a custom version
+     * cmake/3.27.7
+     * nvhpc/24.1
+     * quantum-espresso/7.3.1
+     * cray-mpich/8.1.29
+     * fftw/3.3.10
+     * git/2.42.0
+     * nvpl-lapack/0.2.0
+     * gcc/12.3.0
+     * libxc/6.2.2
+     * nvpl-blas/0.1.0
 
-### using modules
+=== "A100"
 
-To build your version of QE do the following steps:
+    * default : QuantumESPRESSO/7.1 itself + dependencies
+    * develop : only dependencies
 
+    The following modules are provided:
+
+    * cmake/3.26.3
+    * cray-mpich/8.1.25-nvhpc
+    * cuda/11.8.0
+    * fftw/3.3.10
+    * gcc/11.3.0
+    * libxc/5.2.3
+    * nvhpc/22.11
+    * openblas/0.3.23
+    * patchelf/0.17.2
+    * quantum-espresso/7.1
+
+
+# Building a custom version
+
+## Using modules
+
+=== "GH200"
+
+    ```bash
+    uenv start quantumespresso/v7.3.1
+    uenv modules use
+    module load cmake \
+        fftw \
+        nvhpc \
+        nvpl-lapack \
+        nvpl-blas \
+        cray-mpich \
+        netlib-scalapack \
+        libxc
+
+    mkdir build && cd build
+    FC=mpif90 CXX=mpic++ CC=mpicc cmake .. \
+        -DQE_ENABLE_MPI=ON \
+        -DQE_ENABLE_OPENMP=ON \
+        -DQE_ENABLE_SCALAPACK:BOOL=OFF \
+        -DQE_ENABLE_LIBXC=ON \
+        -DQE_ENABLE_CUDA=ON \
+        -DQE_ENABLE_PROFILE_NVTX=ON \
+        -DQE_CLOCK_SECONDS:BOOL=OFF \
+        -DQE_ENABLE_MPI_GPU_AWARE:BOOL=OFF \
+        -DQE_ENABLE_OPENACC=ON
+    make -j20
+    ```
+
+=== "A100"
+
+    ```bash
+    uenv start quantumespresso/v7.1
+    uenv modules use
+    module load cmake \
+        cray-mpich
+        cuda \
+        fftw \
+        gcc \
+        libxc \
+        nvhpc \
+        openblas
+    mkdir build && cd build
+    FC=mpif90 CXX=mpic++ CC=mpicc cmake .. \
+        -DQE_ENABLE_MPI=ON \
+        -DQE_ENABLE_OPENMP=ON \
+        -DQE_ENABLE_SCALAPACK:BOOL=OFF \
+        -DQE_ENABLE_LIBXC=ON \
+        -DQE_ENABLE_CUDA=ON \
+        -DQE_CLOCK_SECONDS:BOOL=OFF \
+        -DQE_ENABLE_MPI_GPU_AWARE:BOOL=OFF \
+        -DQE_ENABLE_OPENACC=ON
+    make -j20
+    ```
+
+## Using spack
+
+1. Clone spack using the same version that has been used to build the uenv. 
 ```bash
-module load cmake cray-mpich/8.1.25-nvhpc cuda fftw libxc/5.2.3-nvhpc nvhpc openblas
-cd qe-71-dev
-mkdir build && cd build
-cmake .. -DQE_ENABLE_OPENMP=1 -DQE_ENABLE_SCALAPACK=0 -DQE_ENABLE_LIBXC=1 -DQE_ENABLE_CUDA=1
-make -j20
+uenv start quantumespresso/v7.3.1
+# clone the same spack version as has been used to build the uenv
+git clone -b $(jq -r .spack.commit /user-environment/meta/configure.json) $(jq -r .spack.repo /user-environment/meta/configure.json) $SCRATCH/spack
 ```
 
-### using Spack
+2. Activate spack with the uenv configured as upstream
+```bash
+# ensure spack is using the uenv as upstream repository (always required)
+export SPACK_SYSTEM_CONFIG_PATH=/user-environment/config
+# active spack (always required)
+. $SCRATCH/spack/share/spack/setup-env.sh
+```
+
+3. Create an anonymous environment for QE
+```bash
+spack env create -d $SCRATCH/qe-env
+spack -e $SCRATCH/qe-env add quantum-espresso%nvhpc +cuda
+spack -e $SCRATCH/qe-env config add packages:all:prefer:cuda_arch=90
+spack -e $SCRATCH/qe-env develop -p /path/to/your/QE-src quantum-espresso@=develop
+spack -e $SCRATCH/qe-env concretize -f
+```
+Check the output of `spack concretize -f`. All dependencies should have been picked up from spack upstream, marked eiter by a green `[^]` or `[e]`.
+Next we create a local filesystem view, this instructs spack to create symlinks for binaries and libraries in a local directory `view`.
+```bash
+spack -e $SCRATCH/qe-env env view enable view
+spack -e $SCRATCH/qe-env install
+```
+To recompile QE after editing the source code re-run `spack -e $SCRATCH/qe-env install`.
+
+4. Run `pw.x` using the filesystem view generated in 3.
+```bash
+uenv start quantumespresso/v7.3.1
+srun [...] $SCRATCH/qe-env/view/bin/pw.x < pw.in
+```
+Note: The `pw.x` is linked to the uenv, it won't work without activating the uenv, also it will only work with the exact same version of the uenv. The physical installation path is in `$SCRATCH/spack`, deleting this directory will leave the anonymous spack environment created in 3. with dangling symlinks.
+
+
+
+
 
-todo.

mkdocs.yml

@@ -8,6 +8,7 @@ nav:
     - 'linaro-forge': uenv-linaro-forge.md
     - 'namd': uenv-namd.md
     - 'quantumespresso': uenv-qe.md
+    - 'quantumespresso-gh200': uenv-qe-gh200.md
     - 'vasp': uenv-vasp.md
   - 'Developers':
     - 'tutorial: application': pkg-application-tutorial.md

recipes/quantumespresso/v7.3.1/gh200/compilers.yaml

@@ -0,0 +1,9 @@
+bootstrap:
+  spec: gcc@12
+gcc:
+  specs:
+  - gcc@12
+llvm:
+  requires: gcc@12
+  specs:
+  - [email protected]

recipes/quantumespresso/v7.3.1/gh200/config.yaml

@@ -0,0 +1,6 @@
+name: quantumespresso
+store: /user-environment
+description: QuantumESPRESSO 7.3.1 +cuda
+spack:
+  repo: https://github.com/spack/spack.git
+  commit: v0.21.2

recipes/quantumespresso/v7.3.1/gh200/environments.yaml

@@ -0,0 +1,32 @@
+nvhpc-env:
+  compiler:
+    - toolchain: llvm
+      spec: nvhpc
+    - toolchain: gcc
+      spec: gcc@12
+  mpi:
+    spec: [email protected]
+    gpu: cuda
+  unify: true
+  specs:
+  - [email protected] %nvhpc +libxc +cuda +nvtx +scalapack
+  - cmake%gcc
+  - fftw%gcc
+  - netlib-scalapack%gcc
+  - nvpl-blas threads=openmp
+  - nvpl-lapack threads=openmp
+  - libxc%nvhpc~cuda
+  - libxml2%gcc
+  - patchelf%gcc
+  packages:
+  - git
+  variants:
+  - +mpi
+  - +cuda
+  - cuda_arch=90
+  views:
+    default:
+      link: roots
+    develop:
+      exclude: ['quantum-espresso', 'patchelf', 'git']
+      link: roots

recipes/quantumespresso/v7.3.1/gh200/modules.yaml

@@ -0,0 +1,23 @@
+modules:
+  # Paths to check when creating modules for all module sets
+  prefix_inspections:
+    bin:
+      - PATH
+    lib:
+      - LD_LIBRARY_PATH
+    lib64:
+      - LD_LIBRARY_PATH
+
+  default:
+    arch_folder: false
+    # Where to install modules
+    roots:
+      tcl: /user-environment/modules
+    tcl:
+      all:
+        autoload: none
+      hash_length: 0
+      exclude_implicits: true
+      exclude: ['%[email protected]', 'gcc %[email protected]']
+      projections:
+        all: '{name}/{version}'

recipes/quantumespresso/v7.3.1/gh200/repo/packages/libxc/0001-Bugfix-avoid-implicit-pointer-cast-to-make-libxc-com.patch

@@ -0,0 +1,26 @@
+From 205a08a30ee0d057d173f9eaa6fc6414f6960c22 Mon Sep 17 00:00:00 2001
+From: Xavier Andrade <[email protected]>
+Date: Wed, 27 May 2020 15:42:32 -0700
+Subject: [PATCH] Bugfix: avoid implicit pointer cast to make libxc compile
+ with a C++ compiler.
+
+---
+ src/functionals.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/functionals.c b/src/functionals.c
+index 3f668292..293f7f9b 100644
+--- a/src/functionals.c
++++ b/src/functionals.c
+@@ -324,7 +324,7 @@ xc_func_set_ext_params_name(xc_func_type *p, const char *name, double par)
+
+   assert(p != NULL && p->info->ext_params.n > 0);
+
+-  ext_params = libxc_malloc(p->info->ext_params.n*sizeof(double));
++  ext_params = (double *) libxc_malloc(p->info->ext_params.n*sizeof(double));
+   for(ii=0; ii<p->info->ext_params.n; ii++){
+     if(strcmp(p->info->ext_params.names[ii], name) == 0)
+       ext_params[ii] = par;
+-- 
+2.25.1
+

recipes/quantumespresso/v7.3.1/gh200/repo/packages/libxc/0002-Mark-xc_erfcx-a-GPU_FUNCTION.patch

@@ -0,0 +1,33 @@
+From 11b90a0a477925f19dfeb7e1f672058f25d83387 Mon Sep 17 00:00:00 2001
+From: Harmen Stoppels <[email protected]>
+Date: Fri, 31 Jul 2020 13:54:39 +0200
+Subject: [PATCH] Mark xc_erfcx a GPU_FUNCTION
+
+---
+ src/faddeeva.c | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/src/faddeeva.c b/src/faddeeva.c
+index ceec4736..03484713 100644
+--- a/src/faddeeva.c
++++ b/src/faddeeva.c
+@@ -29,6 +29,8 @@
+
+ #include "config.h"
+
++#include "util.h"
++
+ #include <math.h>
+
+ /* erfcx(x) = exp(x^2) erfc(x) function, for real x, written by
+@@ -477,6 +479,7 @@ return 0.97771701335885035464e0 + (0.22000938572830479551e-1 + (0.27951610702682
+   return 1.0;
+ }
+
++GPU_FUNCTION
+ double xc_erfcx(double x)
+ {
+   if (x >= 0) {
+-- 
+2.25.1
+