diff --git a/.github/workflows/makezip.yaml b/.github/workflows/makezip.yaml index fcc1371..8eefa72 100644 --- a/.github/workflows/makezip.yaml +++ b/.github/workflows/makezip.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Build modules.zip run: | [[ $(uname) == Linux ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 06eca90..f5e3dfb 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: ShellCheck run: | shellcheck -x $(find -name '*.sh') @@ -24,13 +24,13 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-13, macos-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install prerequisites run: | - [[ $(uname) == Linux ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev + [[ $(uname) == Linux ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev libgl1-mesa-glx libegl1 [[ $(uname) == Darwin ]] && brew install openmpi ./install.sh - name: Version info @@ -41,12 +41,20 @@ jobs: # Don't build pyopencl as that is difficult cat myreq.txt | sed /pyopencl/d > req.txt pip install -r req.txt + # FIXME: disabled until https://github.com/conda-forge/pocl-feedstock/pull/96 is resolved + # - name: Run update + # run: | + # ./pull-packages.sh -x - name: Reinstall emirge env with new requirements.txt run: | + source miniforge3/bin/activate ceesd + conda clean -a + conda deactivate ./install.sh --env-name=reinstall_env --pip-pkgs=req.txt - - name: Reinstall conda env with new env file - run: | - ./install.sh --conda-env=myenv.yml --pip-pkgs=req.txt + # FIXME: Disabled until the next mpi4py release (v4) + # - name: Reinstall conda env with new env file + # run: | + # ./install.sh --conda-env=myenv.yml --pip-pkgs=req.txt - name: Run examples run: | source miniforge3/bin/activate ceesd diff --git a/.gitignore b/.gitignore index 4ae493c..b36a826 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,7 @@ dmypy.json # emirge stuff Miniforge*.sh +Mambaforge*.sh *.vtu # File generated by makezip.sh @@ -155,6 +156,7 @@ pyrometheus/ pytato/ pytools/ myenv/ +feinsum/ # Default path for conda installation miniforge3/ diff --git a/README.md b/README.md index 72b8716..92dac16 100644 --- a/README.md +++ b/README.md @@ -19,89 +19,20 @@ In most cases, running `./install.sh` should be sufficient to install all packag - `--env-name=NAME`: Create conda environment named `NAME` instead of the default (ceesd). - `--modules`: Install a modules.zip file that contains a copy of all python packages that are installed through git (see below for details). - `--branch=NAME`: Install the `NAME`d branch of mirgecom instead of the default branch (main). +- `--fork=NAME` : Install mirgecom from a fork (default=illinois-ceesd). - `--conda-pkgs=FILE`: Install additional conda packages from the list of package names specified in `FILE`. - `--conda-env=FILE`: Obtain conda package versions from conda environment file FILE. - `--pip-pkgs=FILE`: Install additional pip packages from the pip requirements file specified in `FILE`. - `--git-ssh`: Use SSH-based URL to clone mirgecom. - `--debug`: Show debugging output of this script (set -x). - `--skip-clone`: Skip cloning mirgecom, assume it will be manually copied to the selected installation prefix. +- `--py-ver=VERSION`: Replace the Python version specified in the conda environment file with `VERSION` (e.g., `--py-ver=3.10`). - `--help`: Print this help text. -## Testing the installation +# Testing the installation Testing can be done by: ```bash $ mirgecom/examples/run_examples.sh mirgecom/examples/ ``` - -## Running on systems with lots of nodes (>256) -On large systems, the file system can become a bottleneck for loading Python -packages. On these systems, it is recommended to create a zip file with the -modules to speed up the startup process. This can be done by specifying the -`--modules` parameter to `install.sh`, or by running `makezip.sh` after -installation. - -See https://github.com/illinois-ceesd/planning/issues/26 for more details. - - -# Manual installation - -Please use the instructions above instead. - -## Running wavelet0 - - -### Prerequesites - -#### Install POCL - -##### Installation with conda-miniforge - -```bash -$ wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh -# For Power8/9: -# $ wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-ppc64le.sh -# For MacOS: -# $ wget https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-MacOSX-x86_64.sh - -# Install Miniforge/conda: -$ bash ./Miniforge3-Linux-x86_64.sh - -# Optional: create conda environment -$ export MY_CONDA=/path/to/installed/conda # Default installation path: $HOME/miniforge3 -$ $MY_CONDA/bin/conda create -n ceesd -$ . $MY_CONDA/bin/activate ceesd - -# Install required conda packages: -$ conda install pip pocl numpy pyopencl islpy flake8 mypy pudb - -# Install optional conda packages: -$ conda install clinfo - -# In a new session, you may reactivate this environment using: -. $MY_CONDA/bin/activate ceesd -``` - -##### Installation with Spack - -```bash -$ git clone git@github.com:spack/spack -$ source spack/share/spack/setup-env.sh -# Maybe edit your Spack config -# $ spack config edit packages -$ spack install pocl -``` - -#### Install Python packages - -```bash -$ pip install pyvisfile -$ for m in pytools pymbolic dagrt leap loopy meshmode grudge mirgecom; do cd $m && pip install -e . && cd ..; done -``` - -### Run wavelet0 - -```bash -$ cd mirgecom/examples; python wave-eager.py -``` diff --git a/install-pip-dependencies.sh b/install-pip-dependencies.sh index 1b59c2c..d5dffb6 100755 --- a/install-pip-dependencies.sh +++ b/install-pip-dependencies.sh @@ -9,7 +9,7 @@ # Usage: install-pip-dependencies # -set -o nounset -o errexit +set -o errexit origin=$(pwd) requirements_file="${1-mirgecom/requirements.txt}" @@ -27,8 +27,8 @@ python -m pip install pybind11 # Some nice-to haves for development python -m pip install pytest pudb flake8 pep8-naming flake8-quotes flake8-bugbear \ - pytest-pudb sphinx \ - sphinx_math_dollar sphinx_copybutton furo + flake8-comprehensions pytest-pudb sphinx \ + sphinx_math_dollar sphinx_copybutton furo ruff if [[ $(mpicc --version) == "IBM XL"* ]]; then @@ -37,4 +37,14 @@ if [[ $(mpicc --version) == "IBM XL"* ]]; then exit 1 fi + +# Install the packages from the requirements file +export MPI4PY_BUILD_CONFIGURE=1 + +if [[ $(hostname) == tioga* ]]; then + # Need extra flags for the mpi4py build + LDFLAGS="$LDFLAGS -Wl,-rpath $CRAYLIBS_X86_64" \ + pip install 'mpi4py>=4' +fi + pip install --src . -r "$requirements_file" diff --git a/install.sh b/install.sh index e113a94..1dc067c 100755 --- a/install.sh +++ b/install.sh @@ -21,6 +21,15 @@ function version } +if [[ $(hostname) == tioga* ]]; then + if [[ -z $ROCM_PATH ]]; then + # ROCM_PATH is needed below to install the AMD OpenCL ICD link + echo "**** Error: No ROCM_PATH environment variable set." + echo "**** Please load the appropriate 'rocm' module." + exit 3 + fi +fi + usage() { echo "Usage: $0 [--install-prefix=DIR] [--branch=NAME] [--conda-prefix=DIR]" @@ -34,6 +43,7 @@ usage() echo " --conda-pkgs=FILE Install these additional packages with conda." echo " --conda-env=FILE Obtain conda package versions from conda environment file FILE." echo " --pip-pkgs=FILE Install these additional packages with pip." + echo " --py-ver=VERSION Replace the Python version specified in the conda environment file with VERSION (e.g., --py-ver=3.10)." echo " --git-ssh Use SSH-based URL to clone mirgecom." echo " --debug Show debugging output of this script (set -x)." echo " --skip-clone Skip cloning mirgecom, assume it will be manually copied." @@ -68,6 +78,9 @@ opt_skip_clone=0 # Check out version of the branch before a certain date opt_before_date= + +opt_py_ver= + while [[ $# -gt 0 ]]; do arg=$1 shift @@ -122,6 +135,10 @@ while [[ $# -gt 0 ]]; do --before=*) opt_before_date=${arg#*=} ;; + --py-ver=*) + # Install this python version instead of the version specified in the conda env file. + opt_py_ver=${arg#*=} + ;; --help) usage exit 0 @@ -143,7 +160,7 @@ echo "==== Conda installation" # Make sure we get the just installed conda. # See https://github.com/conda/conda/issues/10133 for details. -#shellcheck disable=SC1090 +#shellcheck disable=SC1091 source "$MY_CONDA_DIR"/bin/activate export PATH=$MY_CONDA_DIR/bin:$PATH @@ -181,7 +198,26 @@ echo "==== Create $env_name conda environment" [[ -z $conda_env_file ]] && conda_env_file="$mcsrc/conda-env.yml" -conda env create --name "$env_name" --force --file="$conda_env_file" +if [[ -n $opt_py_ver ]]; then + echo "=== Overriding Python version with $opt_py_ver" + sed -i.bak "s,- python=3[0-9\.]*,- python=$opt_py_ver," "$conda_env_file" +fi + +# Due to https://github.com/conda/conda/issues/8089, we have to install these +# packages manually on specific operating systems: +# Required to use pocl on macOS Big Sur+: +# https://github.com/illinois-ceesd/emirge/issues/114 +# https://github.com/conda-forge/pocl-feedstock/pull/96 +if [[ $(uname) == "Darwin" ]]; then + echo "=== adding clang_osx and ld64 to conda environment file" + echo "- ld64=609" >> "$conda_env_file" + [[ $(uname -m) == "x86_64" ]] && echo "- clang_osx-64" >> "$conda_env_file" + [[ $(uname -m) == "arm64" ]] && echo "- clang_osx-arm64" >> "$conda_env_file" +fi + +cat "$conda_env_file" + +mamba env create --name "$env_name" --yes --file="$conda_env_file" # Avoid a 'frankenconda' env that uses Python from the base env. # See https://github.com/illinois-ceesd/emirge/pull/132 for details. @@ -190,7 +226,7 @@ conda deactivate # Strike 2: deactivate conda base env: conda deactivate # Strike 3: activate the desired env, which now actually works: -#shellcheck disable=SC1090 +#shellcheck disable=SC1091 source "$MY_CONDA_DIR"/bin/activate "$env_name" if [[ -n "$conda_pkg_file" ]]; then @@ -198,41 +234,24 @@ if [[ -n "$conda_pkg_file" ]]; then # shellcheck disable=SC2013 for package in $(cat "$conda_pkg_file"); do echo "=== Installing user-custom package ($package)." - conda install --yes "$package" + mamba install --yes "$package" done fi -# Due to https://github.com/conda/conda/issues/8089, we have to install these -# packages manually on specific operating systems: - -# Required for Nvidia GPU support on Linux (package does not exist on macOS) -[[ $(uname) == "Linux" ]] && conda install --yes pocl-cuda +# shellcheck disable=SC2153 +echo "==== Creating pin file for conda packages: $CONDA_PREFIX/conda-meta/pinned" +echo 'pocl=5.0=*_6' > "$CONDA_PREFIX"/conda-meta/pinned -# Required to use pocl on macOS Big Sur -# (https://github.com/illinois-ceesd/emirge/issues/114) if [[ $(uname) == "Darwin" ]]; then - [[ $(uname -m) == "x86_64" ]] && conda install --yes clang_osx-64 - [[ $(uname -m) == "arm64" ]] && conda install --yes clang_osx-arm64 + echo 'ld64=609' >> "$CONDA_PREFIX"/conda-meta/pinned fi -# Remove spurious (and almost empty) sysroot caused by a bug in the 'qt' package -# (at least version 5.12.9). See https://github.com/conda-forge/qt-feedstock/issues/208 -# for details. -( -BROKEN_SYSROOT="$MY_CONDA_DIR/envs/$env_name/x86_64-conda-linux-gnu/sysroot/" -if [[ -d $BROKEN_SYSROOT ]]; then - cd "$BROKEN_SYSROOT" - nFiles=$(find .//. ! -name . -print | grep -c //) - if [[ $nFiles != "4" ]]; then - echo "**** WARNING: SYSROOT at $BROKEN_SYSROOT not empty, refusing to remove it." - echo "**** Installation of mpi4py might fail." - echo "**** See https://github.com/conda-forge/qt-feedstock/issues/208 for details." - else - echo "**** Removing SYSROOT at $BROKEN_SYSROOT" - rm -rf "$BROKEN_SYSROOT" - fi + +if [[ $(hostname) == tioga* ]]; then + echo "**** Installing AMD OpenCL ICD (rocm) for Tioga" + #shellcheck disable=SC2153 + echo "$ROCM_PATH/lib/libamdocl64.so" > "$CONDA_PREFIX/etc/OpenCL/vendors/amd_ceesd.icd" fi -) # Install an environment activation script rm -rf "$mcprefix"/config diff --git a/makezip.sh b/makezip.sh index de23f63..a3c0760 100755 --- a/makezip.sh +++ b/makezip.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash set -o errexit -o nounset @@ -21,6 +21,10 @@ for name in */; do MY_MODULES+="${name/\//} " cd "$install_loc/$name" + + # Feinsum has a different directory scheme + [[ $name == "feinsum/" ]] && cd src + echo "=== Zipping $name" zip -r "$zipfile" "$name" cd "$origin" diff --git a/pull-packages.sh b/pull-packages.sh index 3dfb264..716f672 100755 --- a/pull-packages.sh +++ b/pull-packages.sh @@ -1,13 +1,13 @@ #!/bin/bash -set -o nounset -o errexit +set -o nounset -o errexit -o pipefail SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" if [[ $# -ne 1 || $1 != "-x" ]]; then echo "WARNING: This script is for advanced users only. It updates the emirge" - echo "pip packages." + echo "pip, conda, and development (git) packages." echo "Execute this script with the '-x' option if you want to run it: '$0 -x'" echo "Exiting." exit 1 @@ -57,4 +57,29 @@ else echo "==== Conda not found, not updating conda packages." fi + +echo "==== Updating pip packages." + +# Note that 'conda list' and 'pip list' generally contain both +# conda-installed and pip-installed packages, so we need to do some +# filtering to make sure we don't override conda packages with newer +# pip packages. + +# Packages conda thinks were installed via pip/pypi +conda_pypi=$(conda list | awk '/pypi/ {print $1}') + +# Names of outdated packages according to pip +pip_outdated=$(pip list --local --outdated | tail -n +3 | awk '{print $1}') + +# For each outdated package, make sure it was actually installed by pip +# before updating it. +for p in $pip_outdated; do + for pp in $conda_pypi; do + if [[ $p == "$pp" ]]; then + echo "=== Updating $p" + pip install --upgrade "$p" + fi + done +done + echo "==== Done." diff --git a/version.sh b/version.sh index 716fb07..218e15f 100755 --- a/version.sh +++ b/version.sh @@ -78,21 +78,32 @@ uname -a echo -echo "*** Emirge modules" +echo "*** OS packages" + +set +e + +command -v brew && echo "brew list --versions" && brew list --versions 2>/dev/null +command -v apt && echo "apt list --installed" && apt list --installed 2>/dev/null +command -v rpm && echo "rpm -qa" && rpm -qa 2>/dev/null + +set -e + + +echo +echo "*** OS modules" + +command -v module && module --redirect list + + +echo +echo "*** Emirge git modules" res="Package|Branch|Commit|Date|URL\n" res+="=======|======|======|======|======\n" for name in */; do if [[ -d $name ]]; then - if [[ $name == "loopy" && -d loo-py ]]; then - # FIXME: this is a hack that was needed before loo-py was renamed to loopy; - # remove this by 12/2021. - # https://github.com/illinois-ceesd/emirge/pull/85 - cd "loo-py" || exit 1 - else - cd "$name" || exit 1 - fi + cd "$name" || exit 1 url=$(git config --get remote.origin.url) branch=$(git branch --show-current) @@ -142,14 +153,7 @@ for name in */; do continue fi - if [[ $name == "loopy" && -d loo-py ]]; then - # FIXME: this is a hack that was needed before loo-py was renamed to loopy; - # remove this by 12/2021. - # https://github.com/illinois-ceesd/emirge/pull/85 - cd "loo-py" || exit 1 - else - cd "$name" || exit 1 - fi + cd "$name" || exit 1 giturl=$(git config --get remote.origin.url) if [[ $giturl == https://* ]]; then @@ -170,7 +174,7 @@ for name in */; do echo "--editable $giturl@$commit$egg" | tee -a $output_requirements done -# Record mirgecom version as well, if it is not part of the requirements.txt +# Record mirgecom version as well, if it is not part of the requirements.txt file if [[ $seen_mirgecom -eq 0 ]]; then mcremote=$(cd mirgecom && git checkout | cut -d "'" -f 2 | cut -d "/" -f 1) mcurl=$(cd mirgecom && git config --get remote."$mcremote".url) @@ -188,9 +192,9 @@ fi echo echo "*** Conda env file with current conda package versions" -# remove f2py since it can' t be pip install'ed +# remove f2py/feinsum/minikanren since they can't be pip install'ed #shellcheck disable=SC2086 -conda env export | grep -v f2py | tee $output_conda_env +conda env export | grep -v f2py | grep -v feinsum | grep -v minikanren | tee $output_conda_env # If output is a conda environment file, tell user how to install it if [[ -f "$output_conda_env" ]]; then