Merge branch 'main' into git-date

.github/workflows/makezip.yaml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Build modules.zip
       run: |
         [[ $(uname) == Linux ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev

.github/workflows/tests.yaml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: ShellCheck
       run: |
         shellcheck -x $(find -name '*.sh')
@@ -24,13 +24,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest, macos-13, macos-latest]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Install prerequisites
       run: |
-        [[ $(uname) == Linux ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev
+        [[ $(uname) == Linux ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev libgl1-mesa-glx libegl1
         [[ $(uname) == Darwin ]] && brew install openmpi
         ./install.sh
     - name: Version info
@@ -41,12 +41,20 @@ jobs:
         # Don't build pyopencl as that is difficult
         cat myreq.txt | sed /pyopencl/d > req.txt
         pip install -r req.txt
+    # FIXME: disabled until https://github.com/conda-forge/pocl-feedstock/pull/96 is resolved
+    # - name: Run update
+    #   run: |
+    #     ./pull-packages.sh -x
     - name: Reinstall emirge env with new requirements.txt
       run: |
+        source miniforge3/bin/activate ceesd
+        conda clean -a
+        conda deactivate
         ./install.sh --env-name=reinstall_env --pip-pkgs=req.txt
-    - name: Reinstall conda env with new env file
-      run: |
-        ./install.sh --conda-env=myenv.yml --pip-pkgs=req.txt
+    # FIXME: Disabled until the next mpi4py release (v4)
+    # - name: Reinstall conda env with new env file
+    #   run: |
+    #     ./install.sh --conda-env=myenv.yml --pip-pkgs=req.txt
     - name: Run examples
       run: |
         source miniforge3/bin/activate ceesd

.gitignore

@@ -131,6 +131,7 @@ dmypy.json
 
 # emirge stuff
 Miniforge*.sh
+Mambaforge*.sh
 *.vtu
 
 # File generated by makezip.sh
@@ -155,6 +156,7 @@ pyrometheus/
 pytato/
 pytools/
 myenv/
+feinsum/
 
 # Default path for conda installation
 miniforge3/

README.md

@@ -19,89 +19,20 @@ In most cases, running `./install.sh` should be sufficient to install all packag
 - `--env-name=NAME`: Create conda environment named `NAME` instead of the default (ceesd).
 - `--modules`: Install a modules.zip file that contains a copy of all python packages that are installed through git (see below for details).
 - `--branch=NAME`: Install the `NAME`d branch of mirgecom instead of the default branch (main).
+- `--fork=NAME` : Install mirgecom from a fork (default=illinois-ceesd).
 - `--conda-pkgs=FILE`: Install additional conda packages from the list of package names specified in `FILE`.
 - `--conda-env=FILE`: Obtain conda package versions from conda environment file FILE.
 - `--pip-pkgs=FILE`: Install additional pip packages from the pip requirements file specified in `FILE`.
 - `--git-ssh`: Use SSH-based URL to clone mirgecom.
 - `--debug`: Show debugging output of this script (set -x).
 - `--skip-clone`: Skip cloning mirgecom, assume it will be manually copied to the selected installation prefix.
+- `--py-ver=VERSION`: Replace the Python version specified in the conda environment file with `VERSION` (e.g., `--py-ver=3.10`).
 - `--help`: Print this help text.
 
-## Testing the installation
+# Testing the installation
 
 Testing can be done by:
 
 ```bash
 $ mirgecom/examples/run_examples.sh mirgecom/examples/
 ```
-
-## Running on systems with lots of nodes (>256)
-On large systems, the file system can become a bottleneck for loading Python
-packages. On these systems, it is recommended to create a zip file with the
-modules to speed up the startup process. This can be done by specifying the
-`--modules` parameter to `install.sh`, or by running `makezip.sh` after
-installation.
-
-See https://github.com/illinois-ceesd/planning/issues/26 for more details.
-
-
-# Manual installation
-
-Please use the instructions above instead.
-
-## Running wavelet0
-
-
-### Prerequesites
-
-#### Install POCL
-
-##### Installation with conda-miniforge
-
-```bash
-$ wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh
-# For Power8/9:
-# $ wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-ppc64le.sh
-# For MacOS:
-# $ wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh
-
-# Install Miniforge/conda:
-$ bash ./Miniforge3-Linux-x86_64.sh
-
-# Optional: create conda environment
-$ export MY_CONDA=/path/to/installed/conda # Default installation path: $HOME/miniforge3
-$ $MY_CONDA/bin/conda create -n ceesd
-$ . $MY_CONDA/bin/activate ceesd
-
-# Install required conda packages:
-$ conda install pip pocl numpy pyopencl islpy flake8 mypy pudb
-
-# Install optional conda packages:
-$ conda install clinfo
-
-# In a new session, you may reactivate this environment using:
-. $MY_CONDA/bin/activate ceesd
-```
-
-##### Installation with Spack
-
-```bash
-$ git clone [email protected]:spack/spack
-$ source spack/share/spack/setup-env.sh
-# Maybe edit your Spack config
-# $ spack config edit packages
-$ spack install pocl
-```
-
-#### Install Python packages
-
-```bash
-$ pip install pyvisfile
-$ for m in pytools pymbolic dagrt leap loopy meshmode grudge mirgecom; do cd $m && pip install -e . && cd ..; done
-```
-
-### Run wavelet0
-
-```bash
-$ cd mirgecom/examples; python wave-eager.py
-```

install-pip-dependencies.sh

@@ -9,7 +9,7 @@
 # Usage: install-pip-dependencies <requirements_file> <install_location>
 #
 
-set -o nounset -o errexit
+set -o errexit
 
 origin=$(pwd)
 requirements_file="${1-mirgecom/requirements.txt}"
@@ -27,8 +27,8 @@ python -m pip install pybind11
 
 # Some nice-to haves for development
 python -m pip install pytest pudb flake8 pep8-naming flake8-quotes flake8-bugbear \
-                      pytest-pudb sphinx \
-                      sphinx_math_dollar sphinx_copybutton furo
+                      flake8-comprehensions pytest-pudb sphinx \
+                      sphinx_math_dollar sphinx_copybutton furo ruff
 
 
 if [[ $(mpicc --version) == "IBM XL"* ]]; then
@@ -37,4 +37,14 @@ if [[ $(mpicc --version) == "IBM XL"* ]]; then
     exit 1
 fi
 
+
+# Install the packages from the requirements file
+export MPI4PY_BUILD_CONFIGURE=1
+
+if [[ $(hostname) == tioga* ]]; then
+    # Need extra flags for the mpi4py build
+    LDFLAGS="$LDFLAGS -Wl,-rpath $CRAYLIBS_X86_64" \
+        pip install 'mpi4py>=4'
+fi
+
 pip install --src . -r "$requirements_file"

install.sh

@@ -21,6 +21,15 @@ function version
 }
 
 
+if [[ $(hostname) == tioga* ]]; then
+  if [[ -z $ROCM_PATH ]]; then
+    # ROCM_PATH is needed below to install the AMD OpenCL ICD link
+    echo "**** Error: No ROCM_PATH environment variable set."
+    echo "**** Please load the appropriate 'rocm' module."
+    exit 3
+  fi
+fi
+
 usage()
 {
   echo "Usage: $0 [--install-prefix=DIR] [--branch=NAME] [--conda-prefix=DIR]"
@@ -34,6 +43,7 @@ usage()
   echo "  --conda-pkgs=FILE     Install these additional packages with conda."
   echo "  --conda-env=FILE      Obtain conda package versions from conda environment file FILE."
   echo "  --pip-pkgs=FILE       Install these additional packages with pip."
+  echo "  --py-ver=VERSION      Replace the Python version specified in the conda environment file with VERSION (e.g., --py-ver=3.10)."
   echo "  --git-ssh             Use SSH-based URL to clone mirgecom."
   echo "  --debug               Show debugging output of this script (set -x)."
   echo "  --skip-clone          Skip cloning mirgecom, assume it will be manually copied."
@@ -68,6 +78,9 @@ opt_skip_clone=0
 # Check out version of the branch before a certain date
 opt_before_date=
 
+
+opt_py_ver=
+
 while [[ $# -gt 0 ]]; do
   arg=$1
   shift
@@ -122,6 +135,10 @@ while [[ $# -gt 0 ]]; do
     --before=*)
         opt_before_date=${arg#*=}
         ;;
+    --py-ver=*)
+        # Install this python version instead of the version specified in the conda env file.
+        opt_py_ver=${arg#*=}
+        ;;
     --help)
         usage
         exit 0
@@ -143,7 +160,7 @@ echo "==== Conda installation"
 
 # Make sure we get the just installed conda.
 # See https://github.com/conda/conda/issues/10133 for details.
-#shellcheck disable=SC1090
+#shellcheck disable=SC1091
 source "$MY_CONDA_DIR"/bin/activate
 
 export PATH=$MY_CONDA_DIR/bin:$PATH
@@ -181,7 +198,26 @@ echo "==== Create $env_name conda environment"
 
 [[ -z $conda_env_file ]] && conda_env_file="$mcsrc/conda-env.yml"
 
-conda env create --name "$env_name" --force --file="$conda_env_file"
+if [[ -n $opt_py_ver ]]; then
+  echo "=== Overriding Python version with $opt_py_ver"
+  sed -i.bak "s,- python=3[0-9\.]*,- python=$opt_py_ver," "$conda_env_file"
+fi
+
+# Due to https://github.com/conda/conda/issues/8089, we have to install these
+# packages manually on specific operating systems:
+# Required to use pocl on macOS Big Sur+:
+# https://github.com/illinois-ceesd/emirge/issues/114
+# https://github.com/conda-forge/pocl-feedstock/pull/96
+if [[ $(uname) == "Darwin" ]]; then
+  echo "=== adding clang_osx and ld64 to conda environment file"
+  echo "- ld64=609" >> "$conda_env_file"
+  [[ $(uname -m) == "x86_64" ]] && echo "- clang_osx-64" >> "$conda_env_file"
+  [[ $(uname -m) == "arm64" ]] &&  echo "- clang_osx-arm64" >> "$conda_env_file"
+fi
+
+cat "$conda_env_file"
+
+mamba env create --name "$env_name" --yes --file="$conda_env_file"
 
 # Avoid a 'frankenconda' env that uses Python from the base env.
 # See https://github.com/illinois-ceesd/emirge/pull/132 for details.
@@ -190,49 +226,32 @@ conda deactivate
 # Strike 2: deactivate conda base env:
 conda deactivate
 # Strike 3: activate the desired env, which now actually works:
-#shellcheck disable=SC1090
+#shellcheck disable=SC1091
 source "$MY_CONDA_DIR"/bin/activate "$env_name"
 
 if [[ -n "$conda_pkg_file" ]]; then
   echo "==== Installing custom packages from file ($conda_pkg_file)."
   # shellcheck disable=SC2013
   for package in $(cat "$conda_pkg_file"); do
     echo "=== Installing user-custom package ($package)."
-    conda install --yes "$package"
+    mamba install --yes "$package"
   done
 fi
 
-# Due to https://github.com/conda/conda/issues/8089, we have to install these
-# packages manually on specific operating systems:
-
-# Required for Nvidia GPU support on Linux (package does not exist on macOS)
-[[ $(uname) == "Linux" ]] && conda install --yes pocl-cuda
+# shellcheck disable=SC2153
+echo "==== Creating pin file for conda packages: $CONDA_PREFIX/conda-meta/pinned"
+echo 'pocl=5.0=*_6' > "$CONDA_PREFIX"/conda-meta/pinned
 
-# Required to use pocl on macOS Big Sur
-# (https://github.com/illinois-ceesd/emirge/issues/114)
 if [[ $(uname) == "Darwin" ]]; then
-  [[ $(uname -m) == "x86_64" ]] && conda install --yes clang_osx-64
-  [[ $(uname -m) == "arm64" ]] && conda install --yes clang_osx-arm64
+  echo 'ld64=609' >> "$CONDA_PREFIX"/conda-meta/pinned
 fi
 
-# Remove spurious (and almost empty) sysroot caused by a bug in the 'qt' package
-# (at least version 5.12.9). See https://github.com/conda-forge/qt-feedstock/issues/208
-# for details.
-(
-BROKEN_SYSROOT="$MY_CONDA_DIR/envs/$env_name/x86_64-conda-linux-gnu/sysroot/"
-if [[ -d $BROKEN_SYSROOT ]]; then
-  cd "$BROKEN_SYSROOT"
-  nFiles=$(find .//. ! -name . -print | grep -c //)
-  if [[ $nFiles != "4" ]]; then
-    echo "**** WARNING: SYSROOT at $BROKEN_SYSROOT not empty, refusing to remove it."
-    echo "**** Installation of mpi4py might fail."
-    echo "**** See https://github.com/conda-forge/qt-feedstock/issues/208 for details."
-  else
-    echo "**** Removing SYSROOT at $BROKEN_SYSROOT"
-    rm -rf "$BROKEN_SYSROOT"
-  fi
+
+if [[ $(hostname) == tioga* ]]; then
+  echo "**** Installing AMD OpenCL ICD (rocm) for Tioga"
+  #shellcheck disable=SC2153
+  echo "$ROCM_PATH/lib/libamdocl64.so" > "$CONDA_PREFIX/etc/OpenCL/vendors/amd_ceesd.icd"
 fi
-)
 
 # Install an environment activation script
 rm -rf "$mcprefix"/config

makezip.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 set -o errexit -o nounset
 
@@ -21,6 +21,10 @@ for name in */; do
     MY_MODULES+="${name/\//} "
 
     cd "$install_loc/$name"
+
+    # Feinsum has a different directory scheme
+    [[ $name == "feinsum/" ]] && cd src
+
     echo "=== Zipping $name"
     zip -r "$zipfile" "$name"
     cd "$origin"