Updates required for full rebuild of analysis3-24.04

environments/analysis3/config.sh

@@ -17,6 +17,7 @@ export ENVIRONMENT=analysis3
 export FULLENV="${ENVIRONMENT}-${VERSION_TO_MODIFY}"
 
 declare -a rpms_to_remove=( "openssh-clients" "openssh-server" "openssh" )
-declare -a replace_from_apps=( "openmpi/4.1.5" "ucx/1.14.0" )
+declare -a replace_from_apps=( "ucx/1.15.0" )
 declare -a outside_commands_to_include=( "pbs_tmrsh" "ssh" )
-declare -a outside_files_to_copy=( "/g/data/hh5/public/apps/nci-intake-catalogue/catalogue_new.yaml" )
+declare -a outside_files_to_copy=( "/g/data/hh5/public/apps/nci-intake-catalogue/catalogue_new.yaml" "/g/data/hh5/public/apps/openmpi/4.1.6" )
+declare -a replace_with_external=( "/g/data/hh5/public/apps/openmpi/4.1.6" )

environments/analysis3/environment.yml

@@ -10,26 +10,26 @@ dependencies:
 - python>=3.10
 - pip
 - sqlalchemy<2.0 ### cosima cookbook does not import with sqlalchemy 2
-- libnetcdf>=4.7.4=mpi_openmpi* # pinned for solver stability
+- libnetcdf=*=mpi_openmpi*
 - coecms-nci
-- cdo<2.1.1 ### https://code.mpimet.mpg.de/boards/1/topics/14349, later versions not compatible with netcdf4 restrictions
+- cdo
 - hdf5
  #x basemap # Deprecated, see https://matplotlib.org/basemap/users/intro.html#deprecation-notice
  #x basemap-data-hires
-- cartopy>=0.19 # pinned for solver stability
+- cartopy
 - dask
 - distributed
 - ipython
 - iris
-- netcdf4<=1.6.0 ### Workaround for https://github.com/pydata/xarray/issues/7079
-- numpy>=1.21 # Mule dependency
+- netcdf4
+- numpy
 - pandas
 - pygrib
 - pyhdf
 - pytest
 - scipy
 - seaborn
-- xarray>2023.9 ### py-cordex fixes xarray at != 2023.9.0 - set this to prevent downgrade
+- xarray
 - jupyter
 - notebook
 - metpy
@@ -69,12 +69,12 @@ dependencies:
 - matplotlib
 #- matplotlib>=3.5.2|<=3.4.3  # pinned to solve  https://github.com/matplotlib/matplotlib/issues/21917
 - dreqpy
-- pint<0.20 ### Pint 0.22 breaks geocat
+- pint
 - pycodestyle 
 - nc-time-axis
 - jq
 - pvlib-python
-- parcels>=2.4 ### Required for cosima recipes
+- parcels
 - siphon
 - scikit-learn
 - filelock
@@ -95,7 +95,7 @@ dependencies:
 - h5py
 - line_profiler
 - bottleneck
-- jupyterlab<4.2.0a0
+- jupyterlab
 - dask-jobqueue
 - jupyterlab_server
 - docrep
@@ -128,8 +128,8 @@ dependencies:
 - panel
 - xrft
 - git
-- openmpi==4.1.5 ### Pin to match latest version on Gadi
-- mpi4py==3.1.4 ### Prevent installation of Intel MPI
+- openmpi
+- mpi4py
 - palettable
 - xlrd
 - nccompress
@@ -153,8 +153,8 @@ dependencies:
 - ninja
 - hvplot
 - cython
-- esmf>=8.0.1=mpi_openmpi*
-- esmpy>=8.0.1=mpi_openmpi*
+- esmf=*=mpi_openmpi*
+- esmpy
 - xesmf
 - sharppy
 - docrep
@@ -263,10 +263,10 @@ dependencies:
 - xskillscore
 - geocat-comp
 - ecgtools
-- gcc_linux-64>=12.2.0 ### Pin otherwise parcels brings in a very old version that breaks the module file
-- ipywidgets>=8
+- gcc_linux-64
+- ipywidgets
 - flox
-- erddapy>2 ### Pin to fix argopy bringing in very old version of erddapy
+- erddapy
 - tintx
 - climpred
 - odc-geo
@@ -279,7 +279,7 @@ dependencies:
 - pysteps
 - ucx-py
 - geocat-viz
-- icclim>=6.4.0 ### Prevent downgrade
+- icclim
 - py-cordex
 - pdbufr
 - matplotlib-scalebar
@@ -289,5 +289,6 @@ dependencies:
 - crick ### Unlisted dependency for dask-expr
 - regional-mom6
 - u8darts-all
+- xwrf
 - pip:
   - railroad-diagrams ### Unlisted dependency of pip and pyparsing

scripts/build.sh

@@ -79,7 +79,7 @@ function inner() {
     ### Replace things from apps
     for pkg in "${replace_from_apps[@]}"; do
         for dir in bin etc lib include; do
-	    if [[ -d "${dir}" ]]; then
+	        if [[ -d "${dir}" ]]; then
                 pushd $dir
                 apps_subdir=/apps/"${pkg}"/"${dir}"
                 for i in $( find "${apps_subdir}" -type f ); do
@@ -89,7 +89,26 @@ function inner() {
                     ln -sf "${i}" "${fn}"
                 done
                 popd
-	    fi
+	        fi
+        done
+    done
+
+    ### Replace things from somewhere else - note, if 'somewhere else'
+    ### is on /g/data, the path will also need to appear in the
+    ### $outside_files_to_copy array
+    for path in "${replace_with_external[@]}"; do
+        for dir in bin etc lib include; do
+            if [[ -d "${dir}" ]]; then
+                pushd $dir
+                external_subdir="${path}"/"${dir}"
+                for i in $( find "${external_subdir}" -type f ); do
+                    fn="${i//$external_subdir\//}"
+                    [[ -e "${fn}" ]] && rm "${fn}"
+                    [[ "${fn}" != "${fn%/*}" ]] && mkdir -p "${fn%/*}"
+                    ln -sf "${i}" "${fn}"
+                done
+                popd
+            fi
         done
     done
     popd
@@ -152,7 +171,7 @@ fi
 echo "Copying external files"
 for f in "${outside_files_to_copy[@]}"; do
     mkdir -p "${OVERLAY_BASE}"/$( dirname "${f#/g/}" )
-    cp "${f}" "${OVERLAY_BASE}"/"${f#/g/}"
+    cp -r "${f}" "${OVERLAY_BASE}"/"${f#/g/}"
 done
 
 if [[ -e  "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}.sqsh" ]]; then

scripts/deploy.sh

@@ -35,7 +35,7 @@ echo "Sync across any changes in the base conda environment"
 rsync --archive --verbose --partial --progress --one-file-system --itemize-changes --hard-links --acls --relative -- "${CONDA_TEMP_PATH}"/./"${APPS_SUBDIR}"/"${CONDA_INSTALL_BASENAME}" "${CONDA_TEMP_PATH}"/./"${MODULE_SUBDIR}" "${CONDA_TEMP_PATH}"/./"${SCRIPT_SUBDIR}" "${CONDA_BASE}"
 
 echo "Make sure anything deleted from this environments scripts directory is also deleted from the prod copy"
-rsync --archive --verbose --partial --progress --one-file-system --itemize-changes --hard-links --acls --relative --delete -- "${CONDA_TEMP_PATH}"/./"${SCRIPT_SUBDIR}"/"${CONDA_ENVIRONMENT}".d "${CONDA_BASE}"
+rsync --archive --verbose --partial --progress --one-file-system --itemize-changes --hard-links --acls --relative --delete -- "${CONDA_TEMP_PATH}"/./"${SCRIPT_SUBDIR}"/"${FULLENV}".d "${CONDA_BASE}"
 set -e
 
 [[ -e "${CONDA_INSTALLATION_PATH}"/envs/"${FULLENV}".sqsh ]] && cp "${CONDA_INSTALLATION_PATH}"/envs/"${FULLENV}".sqsh "${ADMIN_DIR}"/"${FULLENV}".sqsh.bak

scripts/install_config.sh

@@ -42,3 +42,4 @@ fi
 [[ -z ${replace_from_apps+x} ]]           && declare -a replace_from_apps=()           || true
 [[ -z ${outside_commands_to_include+x} ]] && declare -a outside_commands_to_include=() || true
 [[ -z ${outside_files_to_copy+x} ]]       && declare -a outside_files_to_copy=()       || true
+[[ -z ${replace_with_external+x} ]]       && declare -a replace_with_external=()       || true