From 7bd23ba4b7d37f12bd64a6a841e5d8208f3c2f1a Mon Sep 17 00:00:00 2001
From: Romain Beucher <romain.beucher@anu.edu.au>
Date: Thu, 8 Feb 2024 12:03:08 +1100
Subject: [PATCH 01/16] Try this

---
 container/container.def                 | 3 ++-
 environments/access-med/environment.yml | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/container/container.def b/container/container.def
index f97cb0c..cb64142 100644
--- a/container/container.def
+++ b/container/container.def
@@ -47,4 +47,5 @@ Bootstrap: scratch
         mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-24.01
 
 %runscript
-        /usr/bin/bash -l
\ No newline at end of file
+        /usr/bin/bash -l
+        
\ No newline at end of file
diff --git a/environments/access-med/environment.yml b/environments/access-med/environment.yml
index c32051c..190b0ec 100644
--- a/environments/access-med/environment.yml
+++ b/environments/access-med/environment.yml
@@ -71,6 +71,7 @@ dependencies:
 - pip:
   - railroad-diagrams ### Unlisted dependency of pip and pyparsing
   - access-med-utils==0.1.6
+ 
   
 # Julia (dependencies installed by separate script)
 - julia

From ad1fc4f8dca0915c9b85f6bbbb7eb91c9690809c Mon Sep 17 00:00:00 2001
From: Romain Beucher <romain.beucher@anu.edu.au>
Date: Thu, 8 Feb 2024 12:10:08 +1100
Subject: [PATCH 02/16] Try again

---
 environments/access-med/config.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/environments/access-med/config.sh b/environments/access-med/config.sh
index fca0abb..619311a 100644
--- a/environments/access-med/config.sh
+++ b/environments/access-med/config.sh
@@ -9,9 +9,9 @@
 
 
 ### Optional config for custom deploy script
-export VERSION_TO_MODIFY=0.4
+export VERSION_TO_MODIFY=0.6
 export STABLE_VERSION=0.3
-export UNSTABLE_VERSION=0.4
+export UNSTABLE_VERSION=0.6
 
 ### Version settings
 export ENVIRONMENT=access-med

From 6f7e02ed19d985577b4d5047b6698cac3c17b945 Mon Sep 17 00:00:00 2001
From: Romain Beucher <romain.beucher@anu.edu.au>
Date: Thu, 8 Feb 2024 13:00:58 +1100
Subject: [PATCH 03/16] Updates

---
 container/container.def         |  3 +--
 modules/access-med-are-unstable | 15 ---------------
 scripts/build.sh                | 18 ++++++++++--------
 3 files changed, 11 insertions(+), 25 deletions(-)
 delete mode 100644 modules/access-med-are-unstable

diff --git a/container/container.def b/container/container.def
index cb64142..f97cb0c 100644
--- a/container/container.def
+++ b/container/container.def
@@ -47,5 +47,4 @@ Bootstrap: scratch
         mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-24.01
 
 %runscript
-        /usr/bin/bash -l
-        
\ No newline at end of file
+        /usr/bin/bash -l
\ No newline at end of file
diff --git a/modules/access-med-are-unstable b/modules/access-med-are-unstable
deleted file mode 100644
index 545df13..0000000
--- a/modules/access-med-are-unstable
+++ /dev/null
@@ -1,15 +0,0 @@
-#%Module1.0
-source /opt/Modules/extensions/extensions.tcl
-### Set this to the real path later
-set-basedir -root __CONDA_BASE__/__APPS_SUBDIR__ -package __CONDA_INSTALL_BASENAME__ -version envs
-inhibit-self-conflict
-inhibit-base-envvar
-inhibit-version-envvar
-
-### Loads the conda/access-med module, but brings all squashfs into the container
-module load singularity
-module load conda/access-med
-
-foreach sqshfs [ glob $::basedir/*sqsh ] {
-    prepend-path CONTAINER_OVERLAY_PATH $sqshfs
-}
\ No newline at end of file
diff --git a/scripts/build.sh b/scripts/build.sh
index f9a2527..1cc3954 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -64,14 +64,16 @@ function inner() {
 
     pushd "${ENV_INSTALLATION_PATH}"
     ### Get rid of stuff from packages we don't want
-    for dir in bin lib etc libexec include; do 
-        pushd $dir
-        for i in $( rpm -qli "${rpms_to_remove[@]}" ); do 
-            fn=$( basename $i )
-            [[ -f $fn ]] && rm $fn
-            [[ -d $fn ]] && rm -rf $fn
-        done
-        popd
+    for dir in bin lib etc libexec include; do
+	if [[ -d "${dir}" ]]; then
+            pushd $dir
+            for i in $( rpm -qli "${rpms_to_remove[@]}" ); do
+                fn=$( basename $i )
+                [[ -f $fn ]] && rm $fn
+                [[ -d $fn ]] && rm -rf $fn
+            done
+            popd
+	fi
     done
 
     ### Replace things from apps

From e1f682c109e57a04ede202fcdd8677cd436eb80a Mon Sep 17 00:00:00 2001
From: Romain Beucher <romain.beucher@anu.edu.au>
Date: Thu, 8 Feb 2024 13:12:03 +1100
Subject: [PATCH 04/16] Try

---
 environments/romain_test/build_inner.sh  |  8 +++
 environments/romain_test/config.sh       | 23 +++++++
 environments/romain_test/deploy.sh       | 18 ++++++
 environments/romain_test/environment.yml | 77 ++++++++++++++++++++++++
 environments/romain_test/testconfig.yml  | 64 ++++++++++++++++++++
 5 files changed, 190 insertions(+)
 create mode 100644 environments/romain_test/build_inner.sh
 create mode 100644 environments/romain_test/config.sh
 create mode 100644 environments/romain_test/deploy.sh
 create mode 100644 environments/romain_test/environment.yml
 create mode 100644 environments/romain_test/testconfig.yml

diff --git a/environments/romain_test/build_inner.sh b/environments/romain_test/build_inner.sh
new file mode 100644
index 0000000..32e9d5f
--- /dev/null
+++ b/environments/romain_test/build_inner.sh
@@ -0,0 +1,8 @@
+### Custom install inner to build jupyter lab extensions
+
+set +u
+eval "$( ${MAMBA} shell hook --shell bash)"
+micromamba activate "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}"
+set -u
+
+jupyter lab build
\ No newline at end of file
diff --git a/environments/romain_test/config.sh b/environments/romain_test/config.sh
new file mode 100644
index 0000000..619311a
--- /dev/null
+++ b/environments/romain_test/config.sh
@@ -0,0 +1,23 @@
+### config.sh MUST provide the following:
+### $FULLENV
+###
+### Arrays used by the build system (optional, can be empty)
+### rpms_to_remove
+### replace_from_apps
+### outside_commands_to_include
+### outside_files_to_copy
+
+
+### Optional config for custom deploy script
+export VERSION_TO_MODIFY=0.6
+export STABLE_VERSION=0.3
+export UNSTABLE_VERSION=0.6
+
+### Version settings
+export ENVIRONMENT=access-med
+export FULLENV="${ENVIRONMENT}-${VERSION_TO_MODIFY}"
+
+declare -a rpms_to_remove=( "openssh-clients" "openssh-server" "openssh" )
+declare -a replace_from_apps=( "openmpi/4.1.5" "ucx/1.14.0" )
+declare -a outside_commands_to_include=( "pbs_tmrsh" "ssh" )
+declare -a outside_files_to_copy=( "/g/data/hh5/public/apps/nci-intake-catalogue/catalogue_new.yaml" )
\ No newline at end of file
diff --git a/environments/romain_test/deploy.sh b/environments/romain_test/deploy.sh
new file mode 100644
index 0000000..00bdd51
--- /dev/null
+++ b/environments/romain_test/deploy.sh
@@ -0,0 +1,18 @@
+### Update stable/unstable if necessary
+CURRENT_STABLE=$( get_aliased_module "${MODULE_NAME}"/romain-test "${CONDA_MODULE_PATH}" )
+NEXT_STABLE="${ENVIRONMENT}-${STABLE_VERSION}"
+CURRENT_UNSTABLE=$( get_aliased_module "${MODULE_NAME}"/romain-test-unstable "${CONDA_MODULE_PATH}" )
+NEXT_UNSTABLE="${ENVIRONMENT}-${UNSTABLE_VERSION}"
+
+if ! [[ "${CURRENT_STABLE}" == "${MODULE_NAME}/${NEXT_STABLE}" ]]; then
+    echo "Updating stable environment to ${NEXT_STABLE}"
+    write_modulerc "${NEXT_STABLE}" "${NEXT_UNSTABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}"
+    symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}" "${NEXT_STABLE}"
+    symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}".d "${NEXT_STABLE}".d
+fi
+if ! [[ "${CURRENT_UNSTABLE}" == "${MODULE_NAME}/${NEXT_UNSTABLE}" ]]; then
+    echo "Updating unstable environment to ${NEXT_UNSTABLE}"
+    write_modulerc "${NEXT_STABLE}" "${NEXT_UNSTABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}"
+    symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}"-unstable "${NEXT_UNSTABLE}"
+    symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}"-unstable.d "${NEXT_UNSTABLE}".d
+fi
\ No newline at end of file
diff --git a/environments/romain_test/environment.yml b/environments/romain_test/environment.yml
new file mode 100644
index 0000000..190b0ec
--- /dev/null
+++ b/environments/romain_test/environment.yml
@@ -0,0 +1,77 @@
+name: med-condaenv
+channels:
+- conda-forge
+- accessnri
+variables:
+  CARTOPY_USER_BACKGROUNDS: /g/data/xp65/public/apps/cartopy-data/backgrounds
+dependencies:
+- apscheduler
+- fs
+- python>=3.9
+- libnetcdf>=4.7.4=mpi_openmpi* # pinned for solver stability
+- pip
+- pytest >=3.9,!=6.0.0rc1,!=6.0.0
+- pytest-cov
+- autopep8
+- cartopy
+- pyyaml
+- cdo>=1.9.5 # pinned for solver stability
+- matplotlib
+- jinja2
+- ilamb 2.7.*
+- dask
+- distributed
+- dask-labextension
+- panel
+- hvplot
+- datashader
+- iris>=3.1.0
+- iris-esmf-regrid
+- iris-grib
+- intake
+- access-nri-intake==0.0.9 # Keep pinned to latest version while in development
+- netcdf4<=1.6.0 ### Workaround for https://github.com/pydata/xarray/issues/7079
+- mo_pack
+- nodejs
+- numpy>=1.21, !=1.24.3  # severe masking bug
+- pandas
+- scipy
+- h5py
+- numba
+- esmvaltool 2.9.*
+- esmvaltool-python
+- esmvaltool-ncl
+- scikit-image
+- jupyter
+- notebook<6.5.3
+- jupyter_nbextensions_configurator
+- jupyter_contrib_nbextensions
+- jupyterlab
+- jsonschema>=4.18.1
+- jupyterlab_server
+- jupyter-server-proxy
+- jupyter-resource-usage
+- greenlet ### Dependency of... something?
+- objgraph ### Unlisted dependency of greenlet
+- asyncssh
+- gevent
+- openmpi
+- mpi4py
+- esmf>=8.0.1=mpi_openmpi*
+- esmpy>=8.0.1=mpi_openmpi*
+- pytest-json-report ### Needed for esmpy
+- xesmf >=0.7.1
+- xarray >=0.12.0
+- shapely
+- cf_xarray
+- sparse
+- coecms::ucx-py
+- xgboost >1.6.1  # github.com/ESMValGroup/ESMValTool/issues/2779
+- med-diagnostics==0.0.1 # pining for now
+- pip:
+  - railroad-diagrams ### Unlisted dependency of pip and pyparsing
+  - access-med-utils==0.1.6
+ 
+  
+# Julia (dependencies installed by separate script)
+- julia
diff --git a/environments/romain_test/testconfig.yml b/environments/romain_test/testconfig.yml
new file mode 100644
index 0000000..8b7e6ef
--- /dev/null
+++ b/environments/romain_test/testconfig.yml
@@ -0,0 +1,64 @@
+# Modules test will not try to import (should be cleaned)
+skip:
+# Cause errors and don't throw exceptions cleanly
+- access_nri_intake
+- psutil.tests
+- pandas.io.clipboard
+- matplotlib.backends
+- pbr.tests
+- plotly.plotly
+- bokeh.server.django
+- xesmf
+# Too many, or not necessary
+- iris.tests
+- cartopy.tests
+- plotly.validators
+- xarray.tests
+- pyresample.test
+- pyferret.eofanal
+- ants.tests
+- alembic.testing
+- sqlalchemy.testing
+- httpx
+- sanic
+- tests # random tests from black?
+ # - cupy # Disable when testing locally
+ # - cupyx # ditto
+ # - nci_intake_catalogue # ditto
+ # - wrf # Prints garbage at v1.3.2.5
+- matplotlib.tests # No test data
+- prometheus_client.twisted # No twisted
+- pyface
+- qt
+- traits
+- traitsui
+- vtk
+- pyparsing
+- tensorflow_estimator
+- acs_replica_intake  # can't load catalogue file
+- access_nri_intake  # can't load catalogue file
+- pip._vendor.chardet # encounters a null byte, doesn't seem to be an error
+- pykrige # AttributeError: module 'scipy.linalg' has no attribute 'pinv2' -- rk4417
+# for both hdbscan and cosima_cookbook see https://accessdev.nci.org.au/jenkins/job/conda/job/analysis3-unstable/1351/consoleFull
+- hdbscan # test errors but can import
+- cosima_cookbook  # test error 
+  #- plotnine # can't import fails on "import matplotlib._contour as _contour" which seems to work otherwise
+  #- nctoolkit # calls plotnine
+- xgboost.spark ### Don't think we support spark as a parallel backend
+- skimage.future.graph ### Deprecated
+- send2trash
+- esmf_regrid
+- esmvaltool.diag_scripts
+- med_diagnostics
+
+# Preload these modules before testing to avoid weird python issues
+preload:
+- xarray
+- setuptools.command.build_ext ### Strange issue with numba.pycc
+- setuptools.command.build_ext ### Strange issue with numba.pycc
+
+# Allow loading, but pass exceptions. When exceptions no longer triggered
+# can remove
+exception:
+- matplotlib.tests
+- zmq.backend.cffi

From 8bf98220b07c5d935e325732afe327ee538fe63d Mon Sep 17 00:00:00 2001
From: Romain Beucher <romain.beucher@anu.edu.au>
Date: Thu, 8 Feb 2024 13:25:46 +1100
Subject: [PATCH 05/16] Try

---
 scripts/install_config.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/install_config.sh b/scripts/install_config.sh
index 53151d3..8d3c2fc 100644
--- a/scripts/install_config.sh
+++ b/scripts/install_config.sh
@@ -4,9 +4,9 @@ export ADMIN_DIR="${ADMIN_DIR:-/g/data/xp65/admin/med_conda/admin}"
 export CONDA_TEMP_PATH="${PBS_JOBFS:-${CONDA_TEMP_PATH}}"
 export SCRIPT_DIR="${SCRIPT_DIR:-$PWD}"
 
-export SCRIPT_SUBDIR="apps/med_conda_scripts"
+export SCRIPT_SUBDIR="apps1/med_conda_scripts"
 export MODULE_SUBDIR="modules"
-export APPS_SUBDIR="apps"
+export APPS_SUBDIR="apps1"
 export CONDA_INSTALL_BASENAME="med_conda"
 export MODULE_NAME="conda"
 

From b7bf2aef5ccb9397455186c29480290e9339f381 Mon Sep 17 00:00:00 2001
From: rbeucher <romain.beucher@anu.edu.au>
Date: Mon, 12 Feb 2024 14:24:25 +1100
Subject: [PATCH 06/16] Fix

---
 environments/romain_test/build_inner.sh  |  8 ---
 environments/romain_test/config.sh       | 23 -------
 environments/romain_test/deploy.sh       | 18 ------
 environments/romain_test/environment.yml | 77 ------------------------
 environments/romain_test/testconfig.yml  | 64 --------------------
 scripts/functions.sh                     | 19 ++++--
 scripts/install_config.sh                |  2 +-
 7 files changed, 15 insertions(+), 196 deletions(-)
 delete mode 100644 environments/romain_test/build_inner.sh
 delete mode 100644 environments/romain_test/config.sh
 delete mode 100644 environments/romain_test/deploy.sh
 delete mode 100644 environments/romain_test/environment.yml
 delete mode 100644 environments/romain_test/testconfig.yml

diff --git a/environments/romain_test/build_inner.sh b/environments/romain_test/build_inner.sh
deleted file mode 100644
index 32e9d5f..0000000
--- a/environments/romain_test/build_inner.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-### Custom install inner to build jupyter lab extensions
-
-set +u
-eval "$( ${MAMBA} shell hook --shell bash)"
-micromamba activate "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}"
-set -u
-
-jupyter lab build
\ No newline at end of file
diff --git a/environments/romain_test/config.sh b/environments/romain_test/config.sh
deleted file mode 100644
index 619311a..0000000
--- a/environments/romain_test/config.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-### config.sh MUST provide the following:
-### $FULLENV
-###
-### Arrays used by the build system (optional, can be empty)
-### rpms_to_remove
-### replace_from_apps
-### outside_commands_to_include
-### outside_files_to_copy
-
-
-### Optional config for custom deploy script
-export VERSION_TO_MODIFY=0.6
-export STABLE_VERSION=0.3
-export UNSTABLE_VERSION=0.6
-
-### Version settings
-export ENVIRONMENT=access-med
-export FULLENV="${ENVIRONMENT}-${VERSION_TO_MODIFY}"
-
-declare -a rpms_to_remove=( "openssh-clients" "openssh-server" "openssh" )
-declare -a replace_from_apps=( "openmpi/4.1.5" "ucx/1.14.0" )
-declare -a outside_commands_to_include=( "pbs_tmrsh" "ssh" )
-declare -a outside_files_to_copy=( "/g/data/hh5/public/apps/nci-intake-catalogue/catalogue_new.yaml" )
\ No newline at end of file
diff --git a/environments/romain_test/deploy.sh b/environments/romain_test/deploy.sh
deleted file mode 100644
index 00bdd51..0000000
--- a/environments/romain_test/deploy.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-### Update stable/unstable if necessary
-CURRENT_STABLE=$( get_aliased_module "${MODULE_NAME}"/romain-test "${CONDA_MODULE_PATH}" )
-NEXT_STABLE="${ENVIRONMENT}-${STABLE_VERSION}"
-CURRENT_UNSTABLE=$( get_aliased_module "${MODULE_NAME}"/romain-test-unstable "${CONDA_MODULE_PATH}" )
-NEXT_UNSTABLE="${ENVIRONMENT}-${UNSTABLE_VERSION}"
-
-if ! [[ "${CURRENT_STABLE}" == "${MODULE_NAME}/${NEXT_STABLE}" ]]; then
-    echo "Updating stable environment to ${NEXT_STABLE}"
-    write_modulerc "${NEXT_STABLE}" "${NEXT_UNSTABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}"
-    symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}" "${NEXT_STABLE}"
-    symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}".d "${NEXT_STABLE}".d
-fi
-if ! [[ "${CURRENT_UNSTABLE}" == "${MODULE_NAME}/${NEXT_UNSTABLE}" ]]; then
-    echo "Updating unstable environment to ${NEXT_UNSTABLE}"
-    write_modulerc "${NEXT_STABLE}" "${NEXT_UNSTABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}"
-    symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}"-unstable "${NEXT_UNSTABLE}"
-    symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}"-unstable.d "${NEXT_UNSTABLE}".d
-fi
\ No newline at end of file
diff --git a/environments/romain_test/environment.yml b/environments/romain_test/environment.yml
deleted file mode 100644
index 190b0ec..0000000
--- a/environments/romain_test/environment.yml
+++ /dev/null
@@ -1,77 +0,0 @@
-name: med-condaenv
-channels:
-- conda-forge
-- accessnri
-variables:
-  CARTOPY_USER_BACKGROUNDS: /g/data/xp65/public/apps/cartopy-data/backgrounds
-dependencies:
-- apscheduler
-- fs
-- python>=3.9
-- libnetcdf>=4.7.4=mpi_openmpi* # pinned for solver stability
-- pip
-- pytest >=3.9,!=6.0.0rc1,!=6.0.0
-- pytest-cov
-- autopep8
-- cartopy
-- pyyaml
-- cdo>=1.9.5 # pinned for solver stability
-- matplotlib
-- jinja2
-- ilamb 2.7.*
-- dask
-- distributed
-- dask-labextension
-- panel
-- hvplot
-- datashader
-- iris>=3.1.0
-- iris-esmf-regrid
-- iris-grib
-- intake
-- access-nri-intake==0.0.9 # Keep pinned to latest version while in development
-- netcdf4<=1.6.0 ### Workaround for https://github.com/pydata/xarray/issues/7079
-- mo_pack
-- nodejs
-- numpy>=1.21, !=1.24.3  # severe masking bug
-- pandas
-- scipy
-- h5py
-- numba
-- esmvaltool 2.9.*
-- esmvaltool-python
-- esmvaltool-ncl
-- scikit-image
-- jupyter
-- notebook<6.5.3
-- jupyter_nbextensions_configurator
-- jupyter_contrib_nbextensions
-- jupyterlab
-- jsonschema>=4.18.1
-- jupyterlab_server
-- jupyter-server-proxy
-- jupyter-resource-usage
-- greenlet ### Dependency of... something?
-- objgraph ### Unlisted dependency of greenlet
-- asyncssh
-- gevent
-- openmpi
-- mpi4py
-- esmf>=8.0.1=mpi_openmpi*
-- esmpy>=8.0.1=mpi_openmpi*
-- pytest-json-report ### Needed for esmpy
-- xesmf >=0.7.1
-- xarray >=0.12.0
-- shapely
-- cf_xarray
-- sparse
-- coecms::ucx-py
-- xgboost >1.6.1  # github.com/ESMValGroup/ESMValTool/issues/2779
-- med-diagnostics==0.0.1 # pining for now
-- pip:
-  - railroad-diagrams ### Unlisted dependency of pip and pyparsing
-  - access-med-utils==0.1.6
- 
-  
-# Julia (dependencies installed by separate script)
-- julia
diff --git a/environments/romain_test/testconfig.yml b/environments/romain_test/testconfig.yml
deleted file mode 100644
index 8b7e6ef..0000000
--- a/environments/romain_test/testconfig.yml
+++ /dev/null
@@ -1,64 +0,0 @@
-# Modules test will not try to import (should be cleaned)
-skip:
-# Cause errors and don't throw exceptions cleanly
-- access_nri_intake
-- psutil.tests
-- pandas.io.clipboard
-- matplotlib.backends
-- pbr.tests
-- plotly.plotly
-- bokeh.server.django
-- xesmf
-# Too many, or not necessary
-- iris.tests
-- cartopy.tests
-- plotly.validators
-- xarray.tests
-- pyresample.test
-- pyferret.eofanal
-- ants.tests
-- alembic.testing
-- sqlalchemy.testing
-- httpx
-- sanic
-- tests # random tests from black?
- # - cupy # Disable when testing locally
- # - cupyx # ditto
- # - nci_intake_catalogue # ditto
- # - wrf # Prints garbage at v1.3.2.5
-- matplotlib.tests # No test data
-- prometheus_client.twisted # No twisted
-- pyface
-- qt
-- traits
-- traitsui
-- vtk
-- pyparsing
-- tensorflow_estimator
-- acs_replica_intake  # can't load catalogue file
-- access_nri_intake  # can't load catalogue file
-- pip._vendor.chardet # encounters a null byte, doesn't seem to be an error
-- pykrige # AttributeError: module 'scipy.linalg' has no attribute 'pinv2' -- rk4417
-# for both hdbscan and cosima_cookbook see https://accessdev.nci.org.au/jenkins/job/conda/job/analysis3-unstable/1351/consoleFull
-- hdbscan # test errors but can import
-- cosima_cookbook  # test error 
-  #- plotnine # can't import fails on "import matplotlib._contour as _contour" which seems to work otherwise
-  #- nctoolkit # calls plotnine
-- xgboost.spark ### Don't think we support spark as a parallel backend
-- skimage.future.graph ### Deprecated
-- send2trash
-- esmf_regrid
-- esmvaltool.diag_scripts
-- med_diagnostics
-
-# Preload these modules before testing to avoid weird python issues
-preload:
-- xarray
-- setuptools.command.build_ext ### Strange issue with numba.pycc
-- setuptools.command.build_ext ### Strange issue with numba.pycc
-
-# Allow loading, but pass exceptions. When exceptions no longer triggered
-# can remove
-exception:
-- matplotlib.tests
-- zmq.backend.cffi
diff --git a/scripts/functions.sh b/scripts/functions.sh
index a7f6ea7..1f3b16e 100644
--- a/scripts/functions.sh
+++ b/scripts/functions.sh
@@ -206,11 +206,20 @@ function initialise_tmp_dirs() {
     if [[ "${PBS_JOBFS}" ]]; then
         relink_cmds=""
         for dir in "$@"; do
-            relink_cmds="${relink_cmds}rm ~/${dir}; ln -s $( readlink ~/${dir} ) ~/${dir}; "
-            rm ~/"${dir}"
-            mkdir -p "${PBS_JOBFS}"/"${dir}"
-            ln -s "${PBS_JOBFS}"/"${dir}" ~
-            ### Race condition
+            if [[ -h ~/"${dir}" ]]; then
+                relink_cmds="${relink_cmds}rm ~/${dir}; ln -s $( readlink ~/${dir} ) ~/${dir}; "
+                rm ~/"${dir}"
+                mkdir -p "${PBS_JOBFS}"/"${dir}"
+                ln -s "${PBS_JOBFS}"/"${dir}" ~
+                ### Race condition
+            elif [[ -d ~/"${dir}" ]]; then
+                echo "Warning! ~/${dir} is a directory (not a symlink). This can significantly increase "
+                echo "installation time of conda environments."
+            else
+                relink_cmds="${relink_cmds}rm ~/${dir}; "
+                mkdir -p "${PBS_JOBFS}"/"${dir}"
+                ln -s "${PBS_JOBFS}"/"${dir}" ~
+            fi
         done
         trap "${relink_cmds}" EXIT
     fi
diff --git a/scripts/install_config.sh b/scripts/install_config.sh
index 8d3c2fc..80c13cb 100644
--- a/scripts/install_config.sh
+++ b/scripts/install_config.sh
@@ -6,7 +6,7 @@ export SCRIPT_DIR="${SCRIPT_DIR:-$PWD}"
 
 export SCRIPT_SUBDIR="apps1/med_conda_scripts"
 export MODULE_SUBDIR="modules"
-export APPS_SUBDIR="apps1"
+export APPS_SUBDIR="apps"
 export CONDA_INSTALL_BASENAME="med_conda"
 export MODULE_NAME="conda"
 

From 1e135c226587bcbcc2ef902b503c323be86db29e Mon Sep 17 00:00:00 2001
From: Romain Beucher <romain.beucher@anu.edu.au>
Date: Mon, 12 Feb 2024 15:12:30 +1100
Subject: [PATCH 07/16] Update install_config.sh

---
 scripts/install_config.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/install_config.sh b/scripts/install_config.sh
index 80c13cb..53151d3 100644
--- a/scripts/install_config.sh
+++ b/scripts/install_config.sh
@@ -4,7 +4,7 @@ export ADMIN_DIR="${ADMIN_DIR:-/g/data/xp65/admin/med_conda/admin}"
 export CONDA_TEMP_PATH="${PBS_JOBFS:-${CONDA_TEMP_PATH}}"
 export SCRIPT_DIR="${SCRIPT_DIR:-$PWD}"
 
-export SCRIPT_SUBDIR="apps1/med_conda_scripts"
+export SCRIPT_SUBDIR="apps/med_conda_scripts"
 export MODULE_SUBDIR="modules"
 export APPS_SUBDIR="apps"
 export CONDA_INSTALL_BASENAME="med_conda"

From 7d6389a89692c685d34411ae1749fbdf51c1c7ef Mon Sep 17 00:00:00 2001
From: rbeucher <romain.beucher@anu.edu.au>
Date: Wed, 14 Feb 2024 15:28:38 +1100
Subject: [PATCH 08/16] Try to pin openmpi

---
 environments/access-med/environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/environments/access-med/environment.yml b/environments/access-med/environment.yml
index 190b0ec..3d2deba 100644
--- a/environments/access-med/environment.yml
+++ b/environments/access-med/environment.yml
@@ -55,7 +55,7 @@ dependencies:
 - objgraph ### Unlisted dependency of greenlet
 - asyncssh
 - gevent
-- openmpi
+- openmpi==4.1.5
 - mpi4py
 - esmf>=8.0.1=mpi_openmpi*
 - esmpy>=8.0.1=mpi_openmpi*

From 1ac7e6ac87a20a2183d81b6b2538c10f9c65041c Mon Sep 17 00:00:00 2001
From: rbeucher <romain.beucher@anu.edu.au>
Date: Wed, 14 Feb 2024 15:51:39 +1100
Subject: [PATCH 09/16] Try to pin python 3.11

---
 environments/access-med/environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/environments/access-med/environment.yml b/environments/access-med/environment.yml
index 3d2deba..ca751c9 100644
--- a/environments/access-med/environment.yml
+++ b/environments/access-med/environment.yml
@@ -7,7 +7,7 @@ variables:
 dependencies:
 - apscheduler
 - fs
-- python>=3.9
+- python==3.11.*
 - libnetcdf>=4.7.4=mpi_openmpi* # pinned for solver stability
 - pip
 - pytest >=3.9,!=6.0.0rc1,!=6.0.0

From a5b0e4325f39095036f554d0e6fc122f386dc01b Mon Sep 17 00:00:00 2001
From: rbeucher <romain.beucher@anu.edu.au>
Date: Wed, 14 Feb 2024 16:00:21 +1100
Subject: [PATCH 10/16] Revert "Try to pin python 3.11"

This reverts commit 1ac7e6ac87a20a2183d81b6b2538c10f9c65041c.
---
 environments/access-med/environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/environments/access-med/environment.yml b/environments/access-med/environment.yml
index ca751c9..3d2deba 100644
--- a/environments/access-med/environment.yml
+++ b/environments/access-med/environment.yml
@@ -7,7 +7,7 @@ variables:
 dependencies:
 - apscheduler
 - fs
-- python==3.11.*
+- python>=3.9
 - libnetcdf>=4.7.4=mpi_openmpi* # pinned for solver stability
 - pip
 - pytest >=3.9,!=6.0.0rc1,!=6.0.0

From ce43d1d650688d1b2fb25ac39c15458b6b130350 Mon Sep 17 00:00:00 2001
From: rbeucher <romain.beucher@anu.edu.au>
Date: Wed, 14 Feb 2024 16:01:33 +1100
Subject: [PATCH 11/16] Skip rsvdg_frontend testing

---
 environments/access-med/testconfig.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/environments/access-med/testconfig.yml b/environments/access-med/testconfig.yml
index 8b7e6ef..5263a88 100644
--- a/environments/access-med/testconfig.yml
+++ b/environments/access-med/testconfig.yml
@@ -21,6 +21,7 @@ skip:
 - sqlalchemy.testing
 - httpx
 - sanic
+- rsvdg_frontend
 - tests # random tests from black?
  # - cupy # Disable when testing locally
  # - cupyx # ditto

From f2cf8e7de5e0d3a92be8778f1b17cce8c61585b1 Mon Sep 17 00:00:00 2001
From: rbeucher <romain.beucher@anu.edu.au>
Date: Wed, 14 Feb 2024 16:17:32 +1100
Subject: [PATCH 12/16] Revert "Skip rsvdg_frontend testing"

This reverts commit ce43d1d650688d1b2fb25ac39c15458b6b130350.
---
 environments/access-med/testconfig.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/environments/access-med/testconfig.yml b/environments/access-med/testconfig.yml
index 5263a88..8b7e6ef 100644
--- a/environments/access-med/testconfig.yml
+++ b/environments/access-med/testconfig.yml
@@ -21,7 +21,6 @@ skip:
 - sqlalchemy.testing
 - httpx
 - sanic
-- rsvdg_frontend
 - tests # random tests from black?
  # - cupy # Disable when testing locally
  # - cupyx # ditto

From d9daa410c8ec2454e54615cd8dc5856369e91930 Mon Sep 17 00:00:00 2001
From: rbeucher <romain.beucher@anu.edu.au>
Date: Wed, 14 Feb 2024 16:17:57 +1100
Subject: [PATCH 13/16] Skip numba testing

---
 environments/access-med/testconfig.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/environments/access-med/testconfig.yml b/environments/access-med/testconfig.yml
index 8b7e6ef..502a523 100644
--- a/environments/access-med/testconfig.yml
+++ b/environments/access-med/testconfig.yml
@@ -50,6 +50,7 @@ skip:
 - esmf_regrid
 - esmvaltool.diag_scripts
 - med_diagnostics
+- numba
 
 # Preload these modules before testing to avoid weird python issues
 preload:

From fc792631b2f0d14b6ee7ecc0555ff9b7fa6cfa2f Mon Sep 17 00:00:00 2001
From: rbeucher <romain.beucher@anu.edu.au>
Date: Wed, 14 Feb 2024 16:34:27 +1100
Subject: [PATCH 14/16] Add js somewhat needed

---
 environments/access-med/environment.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/environments/access-med/environment.yml b/environments/access-med/environment.yml
index 3d2deba..d0b30a0 100644
--- a/environments/access-med/environment.yml
+++ b/environments/access-med/environment.yml
@@ -7,6 +7,7 @@ variables:
 dependencies:
 - apscheduler
 - fs
+- js
 - python>=3.9
 - libnetcdf>=4.7.4=mpi_openmpi* # pinned for solver stability
 - pip

From 4dbcbfa5ea50fb5872c33100df0fe3b7183739d3 Mon Sep 17 00:00:00 2001
From: rbeucher <romain.beucher@anu.edu.au>
Date: Wed, 14 Feb 2024 16:47:24 +1100
Subject: [PATCH 15/16] Revert "Add js somewhat needed"

This reverts commit fc792631b2f0d14b6ee7ecc0555ff9b7fa6cfa2f.
---
 environments/access-med/environment.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/environments/access-med/environment.yml b/environments/access-med/environment.yml
index d0b30a0..3d2deba 100644
--- a/environments/access-med/environment.yml
+++ b/environments/access-med/environment.yml
@@ -7,7 +7,6 @@ variables:
 dependencies:
 - apscheduler
 - fs
-- js
 - python>=3.9
 - libnetcdf>=4.7.4=mpi_openmpi* # pinned for solver stability
 - pip

From 029c30d14f6f21e5d65569bb1aa4e7bc9273ac8c Mon Sep 17 00:00:00 2001
From: rbeucher <romain.beucher@anu.edu.au>
Date: Wed, 14 Feb 2024 16:52:19 +1100
Subject: [PATCH 16/16] Try

---
 environments/access-med/testconfig.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/environments/access-med/testconfig.yml b/environments/access-med/testconfig.yml
index 502a523..c980048 100644
--- a/environments/access-med/testconfig.yml
+++ b/environments/access-med/testconfig.yml
@@ -51,6 +51,7 @@ skip:
 - esmvaltool.diag_scripts
 - med_diagnostics
 - numba
+- urllib3
 
 # Preload these modules before testing to avoid weird python issues
 preload: