diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9c00e03..2bc7023 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,17 +9,34 @@ jobs: build: name: Build environment runs-on: ubuntu-latest + environment: Gadi steps: - - name: Build environment - uses: appleboy/ssh-action@v0.1.7 + - name: Setup SSH + uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh with: - host: gadi.nci.org.au - username: ${{secrets.GADI_USER}} - key: ${{secrets.DEPLOY_KEY}} - command_timeout: 120m - script: | - export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts - eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh ) - eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh ) - cd $JOB_LOG_DIR - qsub -N build_${{ inputs.environment }} -lncpus=1,mem=20GB,walltime=2:00:00,jobfs=50GB,storage=gdata/v45+scratch/v45+gdata/hh5+scratch/hh5 -v SCRIPT_DIR,CONDA_ENVIRONMENT=${{ inputs.environment }} -P kr06 -q copyq -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/build.sh + hosts: ${{ secrets.HOST }} + private-key: ${{ secrets.SSH_KEY }} + + - name: Build environment + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + set -e + export SCRIPT_DIR="${{ secrets.REPO_PATH }}/scripts" + export CONDA_ENVIRONMENT="${{ inputs.environment }}" + export ADMIN_DIR="${{ vars.ADMIN_DIR }}" + export CONDA_BASE="${{ vars.CONDA_BASE }}" + export APPS_USERS_GROUP="${{ vars.APPS_USERS_GROUP }}" + export APPS_OWNERS_GROUP="${{ vars.APPS_OWNERS_GROUP }}" + PROJECT=${{ vars.PROJECT }} + STORAGE=${{ vars.STORAGE }} + + source "${SCRIPT_DIR}"/install_config.sh + cd "${JOB_LOG_DIR}" + + qsub -N build_"${CONDA_ENVIRONMENT}" -lncpus=1,mem=20GB,walltime=2:00:00,jobfs=50GB,storage="${STORAGE}" \ + -v SCRIPT_DIR,CONDA_ENVIRONMENT,ADMIN_DIR,CONDA_BASE,APPS_USERS_GROUP,APPS_OWNERS_GROUP \ + -P "${PROJECT}" -q copyq -Wblock=true -Wumask=037 \ + "${SCRIPT_DIR}"/build.sh + + EOT \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2a77cbe..14714d9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -9,16 +9,28 @@ jobs: deploy: name: Deploy environment runs-on: ubuntu-latest + environment: Gadi steps: - - name: Deploy environment - uses: appleboy/ssh-action@v0.1.7 + - name: Setup SSH + uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh with: - host: gadi.nci.org.au - username: ${{secrets.GADI_USER}} - key: ${{secrets.DEPLOY_KEY}} - script: | - export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts - export CONDA_ENVIRONMENT=${{ inputs.environment }} - eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh ) - eval $( grep BUILD_STAGE_DIR $SCRIPT_DIR/install_config.sh ) - "${SCRIPT_DIR}"/deploy.sh + hosts: ${{ secrets.HOST }} + private-key: ${{ secrets.SSH_KEY }} + + - name: Deploy environment + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + set -e + export SCRIPT_DIR="${{ secrets.REPO_PATH }}/scripts" + export CONDA_ENVIRONMENT="${{ inputs.environment }}" + export ADMIN_DIR="${{ vars.ADMIN_DIR }}" + export CONDA_BASE="${{ vars.CONDA_BASE }}" + export APPS_USERS_GROUP="${{ vars.APPS_USERS_GROUP }}" + export APPS_OWNERS_GROUP="${{ vars.APPS_OWNERS_GROUP }}" + + source "${SCRIPT_DIR}"/install_config.sh + + "${SCRIPT_DIR}"/deploy.sh + + EOT \ No newline at end of file diff --git a/.github/workflows/get_changed_env.yml b/.github/workflows/get_changed_env.yml index 71a0ef9..ce81c22 100644 --- a/.github/workflows/get_changed_env.yml +++ b/.github/workflows/get_changed_env.yml @@ -32,7 +32,7 @@ jobs: for line in $DIFF; do if [[ $line =~ environments ]]; then env_name=$( basename ${line%/*} ) - if ! [[ $json =~ $env_name ]]; then + if ! [[ $json =~ $env_name ]] && [ -d "environments/$env_name" ]; then json="$json{\"environment\":\"$env_name\"}," fi fi diff --git a/.github/workflows/manual_trigger.yml b/.github/workflows/manual_trigger.yml index 71a800a..8661392 100644 --- a/.github/workflows/manual_trigger.yml +++ b/.github/workflows/manual_trigger.yml @@ -10,27 +10,42 @@ on: jobs: setup: runs-on: ubuntu-latest + environment: Gadi steps: - name: Checkout repository uses: actions/checkout@v4 + + - name: Setup SSH + uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh + with: + hosts: | + ${{ secrets.HOST }} + ${{ secrets.HOST_DATA }} + private-key: ${{ secrets.SSH_KEY }} + - name: Sync repository to Gadi - uses: up9cloud/action-rsync@v1.3 - env: - HOST: gadi.nci.org.au - TARGET: ${{secrets.GADI_REPO_PATH}} - KEY: ${{secrets.DEPLOY_KEY}} - USER: ${{secrets.GADI_USER}} + run: | + rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ + -avz --delete --exclude=/.git/ --exclude=/.github/ \ + ./ \ + ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ secrets.REPO_PATH }} + - name: Create Admin dirs on Gadi - uses: appleboy/ssh-action@v0.1.7 - with: - host: gadi.nci.org.au - username: ${{secrets.GADI_USER}} - key: ${{secrets.DEPLOY_KEY}} - script: | - source ${{secrets.GADI_REPO_PATH}}/scripts/install_config.sh - source ${{secrets.GADI_REPO_PATH}}/scripts/functions.sh - mkdir -p $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR - set_admin_perms $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + set -e + export ADMIN_DIR="${{ vars.ADMIN_DIR }}" + export CONDA_BASE="${{ vars.CONDA_BASE }}" + export APPS_USERS_GROUP="${{ vars.APPS_USERS_GROUP }}" + export APPS_OWNERS_GROUP="${{ vars.APPS_OWNERS_GROUP }}" + + source "${{ secrets.REPO_PATH }}/scripts/install_config.sh" + source "${{ secrets.REPO_PATH }}/scripts/functions.sh" + mkdir -p "${ADMIN_DIR}" "${JOB_LOG_DIR}" "${BUILD_STAGE_DIR}" + set_admin_perms "${ADMIN_DIR}" "${JOB_LOG_DIR}" "${BUILD_STAGE_DIR}" + + EOT build: needs: setup diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5b6f8b5..7e05a32 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1,51 +1,135 @@ name: Build and test conda env + on: pull_request + jobs: generate_matrix: uses: ./.github/workflows/get_changed_env.yml - - setup: + + build_base_image: runs-on: ubuntu-latest needs: generate_matrix if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }} container: image: quay.io/singularity/singularity:v3.11.4 options: --privileged + outputs: + built-container: ${{ steps.changed-container-def.outputs.containerdef_any_changed }} steps: - name: Checkout repository ### Latest at time of writing uses: actions/checkout@v4 + - name: Check if container definition has changed id: changed-container-def - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@48d8f15b2aaa3d255ca5af3eba4870f807ce6b3c # v43.0.2 with: files_yaml: | containerdef: - container/container.def + - name: Build container if definition has changed if: steps.changed-container-def.outputs.containerdef_any_changed == 'true' run: | echo ${{ needs.generate_matrix.outputs.matrix }} sudo -E singularity build container/base.sif container/container.def + + - name: Upload base container + if: steps.changed-container-def.outputs.containerdef_any_changed == 'true' + uses: actions/upload-artifact@v4 + with: + name: base-container + path: container/base.sif + + setup: + runs-on: ubuntu-latest + needs: [ generate_matrix, build_base_image ] + if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }} + environment: Gadi + steps: + - name: Check Gadi Github Environment configuration + run: | + vars_unset=false + if [ -z "${{ vars.ADMIN_DIR }}" ]; then + echo "::error::vars.ADMIN_DIR is unset. ADMIN_DIR is the base directory which will contain the staging and logs sub-directories" + vars_unset=true + fi + if [ -z "${{ vars.CONDA_BASE }}" ]; then + echo "::error::vars.CONDA_BASE is unset. CONDA_BASE is the base deployment directory which will contain apps and module sub-directories" + vars_unset=true + fi + if [ -z "${{ vars.PROJECT }}" ]; then + echo "::error::vars.PROJECT is unset. PROJECT is the project code passed to build+test qsub calls" + vars_unset=true + fi + if [ -z "${{ vars.STORAGE }}" ]; then + echo "::error::vars.STORAGE is unset. STORAGE is the storage flags passed to build+test qsub calls" + vars_unset=true + fi + if [ -z "${{ vars.APPS_USERS_GROUP }}" ]; then + echo "::error::vars.APPS_USERS_GROUP is unset. APPS_USERS_GROUP is used for setting file permissions" + vars_unset=true + fi + if [ -z "${{ vars.APPS_OWNERS_GROUP }}" ]; then + echo "::error::APPS_OWNERS_GROUP is unset. APPS_OWNERS_GROUP is used for setting file permissions" + vars_unset=true + fi + if [ -z "${{ secrets.REPO_PATH }}" ]; then + echo "::error::secrets.REPO_PATH is unset. REPO_PATH is the path where this repository is synced to" + vars_unset=true + fi + + if [ "$vars_unset" == "true" ]; then + echo "::error::Required vars in Gadi Github Environment are unset." + exit 1 + fi + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download base container + if: needs.build_base_image.outputs.built-container == 'true' + uses: actions/download-artifact@v4 + with: + name: base-container + path: container + + - name: Setup SSH + uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh + with: + hosts: | + ${{ secrets.HOST }} + ${{ secrets.HOST_DATA }} + private-key: ${{ secrets.SSH_KEY }} + + - name: Create parent repository directories + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + mkdir -p ${{ secrets.REPO_PATH }} + EOT + - name: Sync repository to Gadi - ### Latest at time of writing - uses: up9cloud/action-rsync@v1.3 - env: - HOST: gadi.nci.org.au - TARGET: ${{secrets.GADI_REPO_PATH}} - KEY: ${{secrets.DEPLOY_KEY}} - USER: ${{secrets.GADI_USER}} + run: | + rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ + -avz --delete --exclude=/.git/ --exclude=/.github/ \ + ./ \ + ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ secrets.REPO_PATH }} + - name: Create Admin dirs on Gadi - uses: appleboy/ssh-action@v0.1.7 - with: - host: gadi.nci.org.au - username: ${{secrets.GADI_USER}} - key: ${{secrets.DEPLOY_KEY}} - script: | - source ${{secrets.GADI_REPO_PATH}}/scripts/install_config.sh - source ${{secrets.GADI_REPO_PATH}}/scripts/functions.sh - mkdir -p $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR - set_admin_perms $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + set -e + export ADMIN_DIR="${{ vars.ADMIN_DIR }}" + export CONDA_BASE="${{ vars.CONDA_BASE }}" + export APPS_USERS_GROUP="${{ vars.APPS_USERS_GROUP }}" + export APPS_OWNERS_GROUP="${{ vars.APPS_OWNERS_GROUP }}" + + source "${{ secrets.REPO_PATH }}/scripts/install_config.sh" + source "${{ secrets.REPO_PATH }}/scripts/functions.sh" + mkdir -p "${ADMIN_DIR}" "${JOB_LOG_DIR}" "${BUILD_STAGE_DIR}" + set_admin_perms "${ADMIN_DIR}" "${JOB_LOG_DIR}" "${BUILD_STAGE_DIR}" + EOT build: needs: [ generate_matrix, setup ] diff --git a/.github/workflows/push_to_main.yml b/.github/workflows/push_to_main.yml index 49a7e3c..5a0f791 100644 --- a/.github/workflows/push_to_main.yml +++ b/.github/workflows/push_to_main.yml @@ -8,19 +8,26 @@ jobs: setup: runs-on: ubuntu-latest + environment: Gadi steps: - name: Checkout repository ### Latest at time of writing uses: actions/checkout@v4 + + - name: Setup SSH + uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh + with: + hosts: ${{ secrets.HOST_DATA }} + private-key: ${{ secrets.SSH_KEY }} + - name: Sync repository to Gadi - ### Latest at time of writing - uses: up9cloud/action-rsync@v1.3 - env: - HOST: gadi.nci.org.au - TARGET: ${{secrets.GADI_REPO_PATH}} - KEY: ${{secrets.DEPLOY_KEY}} - USER: ${{secrets.GADI_USER}} - + run: | + rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \ + -avz --delete --exclude=/.git/ --exclude=/.github/ \ + ./ \ + ${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ secrets.REPO_PATH }} + deploy: needs: [ generate_matrix, setup ] uses: ./.github/workflows/deploy.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c6a65cc..8591075 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,16 +9,34 @@ jobs: test: name: Test environment runs-on: ubuntu-latest + environment: Gadi steps: - - name: Test environment - uses: appleboy/ssh-action@v0.1.7 + - name: Setup SSH + uses: access-nri/actions/.github/actions/setup-ssh@main + id: ssh with: - host: gadi.nci.org.au - username: ${{secrets.GADI_USER}} - key: ${{secrets.DEPLOY_KEY}} - script: | - export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts - eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh ) - eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh ) - cd $JOB_LOG_DIR - qsub -N test_${{ inputs.environment }} -lncpus=4,mem=20GB,walltime=0:20:00,jobfs=50GB,storage=gdata/v45+scratch/v45+gdata/hh5+scratch/hh5 -v SCRIPT_DIR,CONDA_ENVIRONMENT=${{ inputs.environment }} -P kr06 -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/test.sh \ No newline at end of file + hosts: ${{ secrets.HOST }} + private-key: ${{ secrets.SSH_KEY }} + + - name: Test environment + run: | + ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT' + set -e + export SCRIPT_DIR="${{secrets.REPO_PATH}}/scripts" + export CONDA_ENVIRONMENT="${{ inputs.environment }}" + export ADMIN_DIR="${{ vars.ADMIN_DIR }}" + export CONDA_BASE="${{ vars.CONDA_BASE }}" + export APPS_USERS_GROUP="${{ vars.APPS_USERS_GROUP }}" + export APPS_OWNERS_GROUP="${{ vars.APPS_OWNERS_GROUP }}" + PROJECT=${{ vars.PROJECT }} + STORAGE=${{ vars.STORAGE }} + + source "${SCRIPT_DIR}"/install_config.sh + cd "${JOB_LOG_DIR}" + + qsub -N test_"${CONDA_ENVIRONMENT}" -lncpus=4,mem=20GB,walltime=0:20:00,jobfs=50GB,storage="${STORAGE}" \ + -v SCRIPT_DIR,CONDA_ENVIRONMENT,ADMIN_DIR,CONDA_BASE,APPS_USERS_GROUP,APPS_OWNERS_GROUP \ + -P "${PROJECT}" -Wblock=true -Wumask=037 \ + "${SCRIPT_DIR}"/test.sh + + EOT \ No newline at end of file diff --git a/container/container.def b/container/container.def index f97cb0c..082a074 100644 --- a/container/container.def +++ b/container/container.def @@ -38,13 +38,8 @@ Bootstrap: scratch mkdir -p ${SINGULARITY_ROOTFS}/var/lib/sss mkdir -p ${SINGULARITY_ROOTFS}/var/run/munge - mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-22.07 - mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-22.10 - mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.01 - mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.04 - mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.07 - mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-23.10 - mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/analysis3-24.01 + mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/payu-1.1.5 + mkdir -p ${SINGULARITY_ROOTFS}/opt/conda/payu-dev %runscript /usr/bin/bash -l \ No newline at end of file diff --git a/environments/analysis3/build_inner.sh b/environments/analysis3/build_inner.sh deleted file mode 100644 index 32e9d5f..0000000 --- a/environments/analysis3/build_inner.sh +++ /dev/null @@ -1,8 +0,0 @@ -### Custom install inner to build jupyter lab extensions - -set +u -eval "$( ${MAMBA} shell hook --shell bash)" -micromamba activate "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" -set -u - -jupyter lab build \ No newline at end of file diff --git a/environments/analysis3/config.sh b/environments/analysis3/config.sh deleted file mode 100644 index 60a2aca..0000000 --- a/environments/analysis3/config.sh +++ /dev/null @@ -1,23 +0,0 @@ -### config.sh MUST provide the following: -### $FULLENV -### -### Arrays used by the build system (optional, can be empty) -### rpms_to_remove -### replace_from_apps -### outside_commands_to_include -### outside_files_to_copy - -### Optional config for custom deploy script -export VERSION_TO_MODIFY=24.07 -export STABLE_VERSION=24.04 -export UNSTABLE_VERSION=24.07 - -### Version settings -export ENVIRONMENT=analysis3 -export FULLENV="${ENVIRONMENT}-${VERSION_TO_MODIFY}" - -declare -a rpms_to_remove=( "openssh-clients" "openssh-server" "openssh" ) -declare -a replace_from_apps=( "ucx/1.15.0" ) -declare -a outside_commands_to_include=( "pbs_tmrsh" "ssh" ) -declare -a outside_files_to_copy=( "/g/data/hh5/public/apps/nci-intake-catalogue/catalogue_new.yaml" "/g/data/hh5/public/apps/openmpi/4.1.6" ) -declare -a replace_with_external=( "/g/data/hh5/public/apps/openmpi/4.1.6" ) diff --git a/environments/analysis3/deploy.sh b/environments/analysis3/deploy.sh deleted file mode 100644 index b466cbc..0000000 --- a/environments/analysis3/deploy.sh +++ /dev/null @@ -1,18 +0,0 @@ -### Update stable/unstable if necessary -CURRENT_STABLE=$( get_aliased_module "${MODULE_NAME}"/analysis "${CONDA_MODULE_PATH}" ) -NEXT_STABLE="${ENVIRONMENT}-${STABLE_VERSION}" -CURRENT_UNSTABLE=$( get_aliased_module "${MODULE_NAME}"/analysis3-unstable "${CONDA_MODULE_PATH}" ) -NEXT_UNSTABLE="${ENVIRONMENT}-${UNSTABLE_VERSION}" - -if ! [[ "${CURRENT_STABLE}" == "${MODULE_NAME}/${NEXT_STABLE}" ]]; then - echo "Updating stable environment to ${NEXT_STABLE}" - write_modulerc "${NEXT_STABLE}" "${NEXT_UNSTABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}" - symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}" "${NEXT_STABLE}" - symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}".d "${NEXT_STABLE}".d -fi -if ! [[ "${CURRENT_UNSTABLE}" == "${MODULE_NAME}/${NEXT_UNSTABLE}" ]]; then - echo "Updating unstable environment to ${NEXT_UNSTABLE}" - write_modulerc "${NEXT_STABLE}" "${NEXT_UNSTABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}" - symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}"-unstable "${NEXT_UNSTABLE}" - symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}"-unstable.d "${NEXT_UNSTABLE}".d -fi \ No newline at end of file diff --git a/environments/analysis3/environment.yml b/environments/analysis3/environment.yml deleted file mode 100644 index 3f9d303..0000000 --- a/environments/analysis3/environment.yml +++ /dev/null @@ -1,298 +0,0 @@ -name: analysis3 -channels: -- coecms -- conda-forge -- accessnri -variables: - CARTOPY_USER_BACKGROUNDS: /g/data/hh5/public/apps/cartopy-data/backgrounds -dependencies: -- python>=3.10 -- pip -- sqlalchemy<2.0 ### cosima cookbook does not import with sqlalchemy 2 -- libnetcdf=*=mpi_openmpi* -- coecms-nci -- cdo -- hdf5 - #x basemap # Deprecated, see https://matplotlib.org/basemap/users/intro.html#deprecation-notice - #x basemap-data-hires -- cartopy -- dask -- distributed -- ipython -- iris -- netcdf4 -- numpy -- pandas -- pygrib -- pyhdf -- pytest -- scipy -- seaborn -- xarray -- jupyter -- notebook -- metpy -- windspharm -- python-cdo -- arm_pyart -- bokeh -- datashader -- f90nml -- wrf-python -- sympy -- sh -- ffmpeg -- pydap -- twine -- python-graphviz -- joblib -- tqdm -- compliance-checker -- dataset -- xerces-c -- plotly -- zarr -- pytables -- nose -- ncdu -- cftime -- cfunits -- memory_profiler -- fs -- python-magic -- click_default_group -- regionmask -- rasterio -- gdal -- libgdal -- matplotlib -#- matplotlib>=3.5.2|<=3.4.3 # pinned to solve https://github.com/matplotlib/matplotlib/issues/21917 -- dreqpy -- pint ###<0.24 ### https://github.com/Ouranosinc/xclim/issues/1771 -- pycodestyle -- nc-time-axis -- jq -- pvlib-python -- parcels -- siphon -- scikit-learn -- filelock -- gsw -- nb_conda_kernels -- poppler -- fiona -- hupper -- kealib -- mo_pack -- mule -- cf-units -- python-stratify -- nbdime -- eofs -- eccodes -- python-eccodes -- h5py -- line_profiler -- bottleneck -- jupyterlab -- dask-jobqueue -- jupyterlab_server -- docrep -- gifsicle -- nodejs -- cfgrib -- h5netcdf -- intake -- gridfill -- xgcm -- xrft -- doit -- pylint -- dusqlpg -- matplotlib-venn -- nchash -- umtools -- cosima-cookbook -- clef -- nccmp -- coecms-util -- yamanifest -- cmocean -- accessnri::payu<1.1 -- splitvar -- marineHeatWaves -- holoviews -- geoviews -- pytest-cov -- panel -- xrft -- git -- openmpi==4.1.6 -- mpi4py -- palettable -- xlrd -- nccompress -- mdssdiff -- mnctools -- addmeta -- satpy -- statsmodels -- dask-labextension -- nbstripout -- jupyter-server-proxy -- sphinx -- doc8 -- pymunge -- climtas -- asyncssh -- ants -- xhistogram -- cartopy_userconfig=1.0.gadi -- geopy -- ninja -- hvplot -- cython -- esmf=*=mpi_openmpi* -- esmpy -- xesmf -- sharppy -- docrep -- earthpy -- descartes -- era5grib -- conda-tree -- xmitgcm -- cupy -- rioxarray -- rasterstats -- windrose -- pyam -- pyarrow -- python-gist -- celluloid -- jags -- xarray-spatial -- noise # unlisted dependency of xarray-spatial -- nctoolkit -- geoplot -- dask-ml -- mamba -- intake-esm -- pykrige -- nci_intake_catalogue -- cmdline_provenance -- mplleaflet -- branca -- fiona -- folium -- black -- aiohttp -- aiohttp-cors -- autopep8 -- spyder-kernels -- dask-xgboost -- git-subtree -- xarray-spatial -- watermark -- cvxpy -- rechunker -- xmovie -- pymannkendall -- param -- tensorflow -- shap -- gh -- pyqt -- vtk -- mayavi -- gcm_filters -- timezonefinder -- lftp -- xarraymannkendall -- watchdog -- numpy_groupies -- anytree -- fortran-language-server -- hdbscan -- jupyter-book -- iris-grib -- s3fs -- kerchunk -- ujson -- intake-thredds -- cafaz -- cdms2 -- pygam -- pysal -- xmhw -- argopy -- dataclasses-json -- ilamb -- posix_ipc -- spectrum -- arch-py -- pyextremes -- ipdb -- palettable -- webcolors -- mpl-scatter-density -- accessnri::benchcab -- lmfit -- xclim -- jupyter-resource-usage -- acs_replica_intake -- matchpy -- mscorefonts -- opencv -- conda-build -- conda-verify -- pygmt -- gmt -- greenlet ### Dependency of... something? -- objgraph ### Unlisted dependency of greenlet -- xarrayutils -- wradlib -- pyinterp -- wavespectra -- xmip -- xcdat -- neutralocean -- ipympl -- libtiff -- xskillscore -- geocat-comp -- ecgtools -- gcc_linux-64 -- ipywidgets -- flox -- erddapy -- tintx -- climpred -- odc-geo -- cmor -- bargeparse -- accessnri::amami -- accessnri::access-nri-intake -- ipynbname -- shellcheck -- pysteps -- ucx-py -- ucx==1.15.0 -- geocat-viz -- icclim -- py-cordex -- pdbufr -- matplotlib-scalebar -- python-kaleido -- numba -- crick ### Unlisted dependency for dask-expr -- regional-mom6 -- u8darts-all -- xwrf -- ecmwf-api-client -- pygamma_n -- numexpr>2.9 -- mopper -- scores -- pip: - - railroad-diagrams ### Unlisted dependency of pip and pyparsing diff --git a/environments/analysis3/testconfig.yml b/environments/analysis3/testconfig.yml deleted file mode 100644 index 18a30a0..0000000 --- a/environments/analysis3/testconfig.yml +++ /dev/null @@ -1,95 +0,0 @@ -# Modules test will not try to import -skip: -# Cause errors and don't throw exceptions cleanly -- psutil.tests -- pandas.io.clipboard -- matplotlib.backends -- pbr.tests -- plotly.plotly -- bokeh.server.django -# Too many, or not necessary -- iris.tests -- cartopy.tests -- plotly.validators -- xarray.tests -- pyresample.test -- pyferret.eofanal -- ants.tests -- alembic.testing -- sqlalchemy.testing -- httpx -- sanic -- tests # random tests from black? - # - cupy # Disable when testing locally - # - cupyx # ditto - # - nci_intake_catalogue # ditto - # - wrf # Prints garbage at v1.3.2.5 -- matplotlib.tests # No test data -- prometheus_client.twisted # No twisted -- pyface -- qt -- traits -- traitsui -- vtk -- pyparsing -- tensorflow_estimator -- acs_replica_intake # can't load catalogue file -- access_nri_intake # can't load catalogue file -- pip._vendor.chardet # encounters a null byte, doesn't seem to be an error -- pykrige # AttributeError: module 'scipy.linalg' has no attribute 'pinv2' -- rk4417 -# for both hdbscan and cosima_cookbook see https://accessdev.nci.org.au/jenkins/job/conda/job/analysis3-unstable/1351/consoleFull -- hdbscan # test errors but can import -- cosima_cookbook # test error - #- plotnine # can't import fails on "import matplotlib._contour as _contour" which seems to work otherwise - #- nctoolkit # calls plotnine -- xgboost.spark ### Don't think we support spark as a parallel backend -- send2trash.mac ### Gadi is not a mac -- send2trash.win ### Nor is it windows -- attrdict ### Deprecated but still erroneously bought in by wavespectra - not used by anything else -- skimage.future.graph ### Thanks for raising a module error to tell me that you've moved this -- numba.core.rvsdg_frontend ### Not supported in Python3.10 -- xgboost.testing ### Has an 'pytest.importorskip' call when imported, causing the whole thing to report skipped -- scipy._lib.array_api_compat.torch ### Don't support pytorch -- urllib3.contrib.emscripten ### Used for web browser integration and also experimental: https://urllib3.readthedocs.io/en/latest/reference/contrib/emscripten.html -- fugue_duckdb ### Optional backend not used by any package in analysis3 -- fugue_ibis ### Optional backend not used by any package in analysis3 -- fugue_polars ### Optional backend not used by any package in analysis3 -- fugue_ray ### Optional backend not used by any package in analysis3 -- fugue_spark ### Optional backend not used by any package in analysis3 -- torch.onnx ### Prevent testing of _internal APIs -- torch.testing ### Prevent testing of _internal APIs -- torch.utils.benchmark ### Uses non-existent part of deprecated pkg_resources -- torch._inductor -- keras.src.backend ### Don't test Keras backends -- functorch ### This file has moved to under torch/_functorch. It is not public API. - -# Preload these modules before testing to avoid weird python issues -preload: -- pysal -- tables -- skimage.data -- sklearn -- sklearn.covariance -- sklearn.manifold -- stratify -- xarray -- setuptools.command.build_ext ### Strange issue with numba.pycc - -# Allow loading, but pass exceptions. When exceptions no longer triggered -# can remove -exception: -- sqlalchemy.testing.suite -- zmq.green -- prometheus_client.twisted -- sqlalchemy.testing.suite -- zmq.backend.cffi -- sklearn.mixture -- matplotlib.tests -- metpy.io # While Issue #21 unresolved -- metpy.plots # While Issue #21 unresolved -- httpx -- sanic -- pyparsing.diagrams -- pysal.explore.esda -- pysal.explore.segregation -- pysal.model.access diff --git a/environments/payu-dev/build_inner.sh b/environments/payu-dev/build_inner.sh new file mode 100644 index 0000000..5935347 --- /dev/null +++ b/environments/payu-dev/build_inner.sh @@ -0,0 +1,14 @@ +### Custom install inner + +# Fix shebang headers in payu entry points (issue with pip installed packages: https://github.com/ACCESS-NRI/MED-condaenv/issues/78) +for file in ${CONDA_INSTALLATION_PATH}/envs/${FULLENV}/bin/payu-*; do + # Using payu-* to modify payu-run, payu-collate, payu-sync files + echo "Adding python header to $file" + # Substitute the first line of file (e.g. 1s), if it starts with #! (regex ^#!/.*$), + # with the python executable in the conda environment + sed -i "1s|^#!/.*$|#!${CONDA_INSTALLATION_PATH}/envs/${FULLENV}/bin/python|" "$file" +done + +# Patch payu shebang header with outer python executable that launches a container when run. +# This means when payu submits qsub commands (e.g. payu run), it uses this python executable and launches a container on PBS job +sed -i "1s|^#!/.*$|#!${CONDA_SCRIPT_PATH}/${FULLENV}.d/bin/python|" "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}/bin/payu" \ No newline at end of file diff --git a/environments/payu-dev/config.sh b/environments/payu-dev/config.sh new file mode 100644 index 0000000..05bfbfc --- /dev/null +++ b/environments/payu-dev/config.sh @@ -0,0 +1,18 @@ +### config.sh MUST provide the following: +### $FULLENV +### +### Arrays used by the build system (optional, can be empty) +### rpms_to_remove +### replace_from_apps +### outside_commands_to_include +### outside_files_to_copy + +### Version settings +export ENVIRONMENT=payu-dev +export FULLENV="${ENVIRONMENT}" + +declare -a rpms_to_remove=() +declare -a replace_from_apps=() +declare -a outside_commands_to_include=( "pbs_tmrsh" ) +declare -a outside_files_to_copy=() +declare -a replace_with_external=() \ No newline at end of file diff --git a/environments/payu-dev/environment.yml b/environments/payu-dev/environment.yml new file mode 100644 index 0000000..83bac70 --- /dev/null +++ b/environments/payu-dev/environment.yml @@ -0,0 +1,19 @@ +name: payu-dev +channels: + - accessnri + - coecms + - conda-forge +dependencies: + - python==3.10 + # Use latest changes from payu's master branch + - pip: + - git+https://github.com/payu-org/payu.git@master + - f90nml + - gh + - git + - nco + - pytest + - openssh>=8.3 + - xarray + - mule + - um2nc \ No newline at end of file diff --git a/environments/payu-dev/testconfig.yml b/environments/payu-dev/testconfig.yml new file mode 100644 index 0000000..2f36155 --- /dev/null +++ b/environments/payu-dev/testconfig.yml @@ -0,0 +1,98 @@ +# Modules test will not try to import +skip: +# # Cause errors and don't throw exceptions cleanly +# - psutil.tests + - pandas.io.clipboard + - matplotlib.backends +# - pbr.tests +# - plotly.plotly +# - bokeh.server.django +# # Too many, or not necessary + - iris.tests + - cartopy.tests +# - plotly.validators + - xarray.tests +# - pyresample.test +# - pyferret.eofanal +# - ants.tests +# - alembic.testing +# - sqlalchemy.testing +# - httpx +# - sanic +# - tests # random tests from black? +# # - cupy # Disable when testing locally +# # - cupyx # ditto +# # - nci_intake_catalogue # ditto +# # - wrf # Prints garbage at v1.3.2.5 + - matplotlib.tests # No test data +# - prometheus_client.twisted # No twisted +# - pyface +# - qt +# - traits +# - traitsui +# - vtk + - pyparsing +# - tensorflow_estimator +# - acs_replica_intake # can't load catalogue file +# - access_nri_intake # can't load catalogue file +# - pip._vendor.chardet # encounters a null byte, doesn't seem to be an error +# - pykrige # AttributeError: module 'scipy.linalg' has no attribute 'pinv2' -- rk4417 +# # for both hdbscan and cosima_cookbook see https://accessdev.nci.org.au/jenkins/job/conda/job/analysis3-unstable/1351/consoleFull +# - hdbscan # test errors but can import +# - cosima_cookbook # test error +# #- plotnine # can't import fails on "import matplotlib._contour as _contour" which seems to work otherwise +# #- nctoolkit # calls plotnine +# - xgboost.spark ### Don't think we support spark as a parallel backend +# - send2trash.mac ### Gadi is not a mac +# - send2trash.win ### Nor is it windows +# - attrdict ### Deprecated but still erroneously bought in by wavespectra - not used by anything else +# - skimage.future.graph ### Thanks for raising a module error to tell me that you've moved this +# - numba.core.rvsdg_frontend ### Not supported in Python3.10 +# - xgboost.testing ### Has an 'pytest.importorskip' call when imported, causing the whole thing to report skipped + - scipy._lib.array_api_compat.torch ### Don't support pytorch + - urllib3.contrib.emscripten ### Used for web browser integration and also experimental: https://urllib3.readthedocs.io/en/latest/reference/contrib/emscripten.html +# - fugue_duckdb ### Optional backend not used by any package in analysis3 +# - fugue_ibis ### Optional backend not used by any package in analysis3 +# - fugue_polars ### Optional backend not used by any package in analysis3 +# - fugue_ray ### Optional backend not used by any package in analysis3 +# - fugue_spark ### Optional backend not used by any package in analysis3 +# - torch.onnx ### Prevent testing of _internal APIs +# - torch.testing ### Prevent testing of _internal APIs +# - torch.utils.benchmark ### Uses non-existent part of deprecated pkg_resources +# - torch._inductor +# - keras.src.backend ### Don't test Keras backends +# - functorch ### This file has moved to under torch/_functorch. It is not public API. + +# Preload these modules before testing to avoid weird python issues +preload: +# - pysal +# - tables +# - skimage.data +# - sklearn +# - sklearn.covariance +# - sklearn.manifold +# - stratify +- xarray +- setuptools.command.build_ext ### Strange issue with numba.pycc + +# Allow loading, but pass exceptions. When exceptions no longer triggered +# can remove +exception: +# - sqlalchemy.testing.suite +# - zmq.green +# - prometheus_client.twisted +# - sqlalchemy.testing.suite +# - zmq.backend.cffi +# - sklearn.mixture +# - matplotlib.tests +# - metpy.io # While Issue #21 unresolved +# - metpy.plots # While Issue #21 unresolved +# - httpx +# - sanic +# - pyparsing.diagrams +# - pysal.explore.esda +# - pysal.explore.segregation +# - pysal.model.access +- fontTools.ufoLib # +- pandas.core._numba.kernels # +- scipy._lib.array_api_compat.cupy # \ No newline at end of file diff --git a/environments/payu/build_inner.sh b/environments/payu/build_inner.sh new file mode 100644 index 0000000..b24820e --- /dev/null +++ b/environments/payu/build_inner.sh @@ -0,0 +1,16 @@ +### Custom install inner to build jupyter lab extensions + +# set +u +# eval "$( ${MAMBA} shell hook --shell bash)" +# micromamba activate "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}" +# set -u + +# jupyter lab build + +# Patch payu shebang header with outer python executable that launches a container when run. +# This means when payu submits qsub commands (e.g. payu run), it uses this python executable and launches a container on PBS job +sed -i "1s|^#!/.*$|#!${CONDA_SCRIPT_PATH}/${FULLENV}.d/bin/python|" "${CONDA_INSTALLATION_PATH}/envs/${FULLENV}/bin/payu" +# Above command follows pattern of "sed -i '1s|pattern|replacement|' filename" +# which modifies the environment's payu executable in place (-i), +# and replaces the first line (1s), if it starts with #! (regex ^#!/.*$), +# with the payu launcher script \ No newline at end of file diff --git a/environments/payu/config.sh b/environments/payu/config.sh new file mode 100644 index 0000000..3b94c1b --- /dev/null +++ b/environments/payu/config.sh @@ -0,0 +1,22 @@ +### config.sh MUST provide the following: +### $FULLENV +### +### Arrays used by the build system (optional, can be empty) +### rpms_to_remove +### replace_from_apps +### outside_commands_to_include +### outside_files_to_copy + +### Optional config for custom deploy script +export VERSION_TO_MODIFY=1.1.5 +export STABLE_VERSION=1.1.5 + +### Version settings +export ENVIRONMENT=payu +export FULLENV="${ENVIRONMENT}-${VERSION_TO_MODIFY}" + +declare -a rpms_to_remove=() +declare -a replace_from_apps=() +declare -a outside_commands_to_include=( "pbs_tmrsh" ) +declare -a outside_files_to_copy=() +declare -a replace_with_external=() diff --git a/environments/payu/deploy.sh b/environments/payu/deploy.sh new file mode 100644 index 0000000..51b32be --- /dev/null +++ b/environments/payu/deploy.sh @@ -0,0 +1,10 @@ +### Update stable if necessary +CURRENT_STABLE=$( get_aliased_module "${MODULE_NAME}"/payu "${CONDA_MODULE_PATH}" ) +NEXT_STABLE="${ENVIRONMENT}-${STABLE_VERSION}" + +if ! [[ "${CURRENT_STABLE}" == "${MODULE_NAME}/${NEXT_STABLE}" ]]; then + echo "Updating stable environment to ${NEXT_STABLE}" + write_modulerc_stable "${NEXT_STABLE}" "${ENVIRONMENT}" "${CONDA_MODULE_PATH}" "${MODULE_NAME}" + symlink_atomic_update "${CONDA_INSTALLATION_PATH}"/envs/"${ENVIRONMENT}" "${NEXT_STABLE}" + symlink_atomic_update "${CONDA_SCRIPT_PATH}"/"${ENVIRONMENT}".d "${NEXT_STABLE}".d +fi \ No newline at end of file diff --git a/environments/payu/environment.yml b/environments/payu/environment.yml new file mode 100644 index 0000000..9022912 --- /dev/null +++ b/environments/payu/environment.yml @@ -0,0 +1,18 @@ +name: payu +channels: + - accessnri + - coecms + - conda-forge +dependencies: + - python==3.10 + - accessnri::payu==1.1.5 + - yamanifest + - f90nml + - gh + - git + - nco + - pytest + - openssh>=8.3 + - xarray + - mule + - um2nc \ No newline at end of file diff --git a/environments/payu/testconfig.yml b/environments/payu/testconfig.yml new file mode 100644 index 0000000..2f36155 --- /dev/null +++ b/environments/payu/testconfig.yml @@ -0,0 +1,98 @@ +# Modules test will not try to import +skip: +# # Cause errors and don't throw exceptions cleanly +# - psutil.tests + - pandas.io.clipboard + - matplotlib.backends +# - pbr.tests +# - plotly.plotly +# - bokeh.server.django +# # Too many, or not necessary + - iris.tests + - cartopy.tests +# - plotly.validators + - xarray.tests +# - pyresample.test +# - pyferret.eofanal +# - ants.tests +# - alembic.testing +# - sqlalchemy.testing +# - httpx +# - sanic +# - tests # random tests from black? +# # - cupy # Disable when testing locally +# # - cupyx # ditto +# # - nci_intake_catalogue # ditto +# # - wrf # Prints garbage at v1.3.2.5 + - matplotlib.tests # No test data +# - prometheus_client.twisted # No twisted +# - pyface +# - qt +# - traits +# - traitsui +# - vtk + - pyparsing +# - tensorflow_estimator +# - acs_replica_intake # can't load catalogue file +# - access_nri_intake # can't load catalogue file +# - pip._vendor.chardet # encounters a null byte, doesn't seem to be an error +# - pykrige # AttributeError: module 'scipy.linalg' has no attribute 'pinv2' -- rk4417 +# # for both hdbscan and cosima_cookbook see https://accessdev.nci.org.au/jenkins/job/conda/job/analysis3-unstable/1351/consoleFull +# - hdbscan # test errors but can import +# - cosima_cookbook # test error +# #- plotnine # can't import fails on "import matplotlib._contour as _contour" which seems to work otherwise +# #- nctoolkit # calls plotnine +# - xgboost.spark ### Don't think we support spark as a parallel backend +# - send2trash.mac ### Gadi is not a mac +# - send2trash.win ### Nor is it windows +# - attrdict ### Deprecated but still erroneously bought in by wavespectra - not used by anything else +# - skimage.future.graph ### Thanks for raising a module error to tell me that you've moved this +# - numba.core.rvsdg_frontend ### Not supported in Python3.10 +# - xgboost.testing ### Has an 'pytest.importorskip' call when imported, causing the whole thing to report skipped + - scipy._lib.array_api_compat.torch ### Don't support pytorch + - urllib3.contrib.emscripten ### Used for web browser integration and also experimental: https://urllib3.readthedocs.io/en/latest/reference/contrib/emscripten.html +# - fugue_duckdb ### Optional backend not used by any package in analysis3 +# - fugue_ibis ### Optional backend not used by any package in analysis3 +# - fugue_polars ### Optional backend not used by any package in analysis3 +# - fugue_ray ### Optional backend not used by any package in analysis3 +# - fugue_spark ### Optional backend not used by any package in analysis3 +# - torch.onnx ### Prevent testing of _internal APIs +# - torch.testing ### Prevent testing of _internal APIs +# - torch.utils.benchmark ### Uses non-existent part of deprecated pkg_resources +# - torch._inductor +# - keras.src.backend ### Don't test Keras backends +# - functorch ### This file has moved to under torch/_functorch. It is not public API. + +# Preload these modules before testing to avoid weird python issues +preload: +# - pysal +# - tables +# - skimage.data +# - sklearn +# - sklearn.covariance +# - sklearn.manifold +# - stratify +- xarray +- setuptools.command.build_ext ### Strange issue with numba.pycc + +# Allow loading, but pass exceptions. When exceptions no longer triggered +# can remove +exception: +# - sqlalchemy.testing.suite +# - zmq.green +# - prometheus_client.twisted +# - sqlalchemy.testing.suite +# - zmq.backend.cffi +# - sklearn.mixture +# - matplotlib.tests +# - metpy.io # While Issue #21 unresolved +# - metpy.plots # While Issue #21 unresolved +# - httpx +# - sanic +# - pyparsing.diagrams +# - pysal.explore.esda +# - pysal.explore.segregation +# - pysal.model.access +- fontTools.ufoLib # +- pandas.core._numba.kernels # +- scipy._lib.array_api_compat.cupy # \ No newline at end of file diff --git a/scripts/functions.sh b/scripts/functions.sh index 2cfea96..9d59d4e 100644 --- a/scripts/functions.sh +++ b/scripts/functions.sh @@ -83,10 +83,26 @@ function write_modulerc() { cat< "${module_path}"/.modulerc #%Module1.0 -module-version ${module_name}/${stable} analysis ${env_name} default +module-version ${module_name}/${stable} ${env_name} default module-version ${module_name}/${unstable} ${env_name}-unstable -module-version ${module_name}/analysis27-18.10 analysis27 +EOF + + set_apps_perms "${module_path}/.modulerc" + +} + +function write_modulerc_stable() { + stable="${1}" + env_name="${2}" + module_path="${3}" + module_name="${4}" + + cat< "${module_path}"/.modulerc +#%Module1.0 + +module-version ${module_name}/${stable} ${env_name} default + EOF set_apps_perms "${module_path}/.modulerc" diff --git a/scripts/initialise.sh b/scripts/initialise.sh index 0cd157e..b61349c 100755 --- a/scripts/initialise.sh +++ b/scripts/initialise.sh @@ -17,9 +17,10 @@ function inner() { mkdir -p "${CONDA_INSTALLATION_PATH}" pushd "${CONDA_INSTALLATION_PATH}" - #curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba + + curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba ### Modified micromamba for compatibility with nb_conda_kernels - curl -Ls https://dsroberts.github.io/mamba/latest | tar -xvj bin/micromamba bin/activate + # curl -Ls https://dsroberts.github.io/mamba/latest | tar -xvj bin/micromamba bin/activate popd mkdir -p "${CONDA_SCRIPT_PATH}"/overrides diff --git a/scripts/install_config.sh b/scripts/install_config.sh index 48a56fe..84e5fa6 100644 --- a/scripts/install_config.sh +++ b/scripts/install_config.sh @@ -1,14 +1,21 @@ ### Settings to control installation path e.g. for test installs -export CONDA_BASE="${CONDA_BASE:-/g/data/hh5/public}" -export ADMIN_DIR="${ADMIN_DIR:-/g/data/hh5/admin/conda_concept/admin}" +if [[ ! "${CONDA_BASE}" ]]; then + echo "Error! CONDA_BASE must be defined" + exit 1 +fi +if [[ ! "${ADMIN_DIR}" ]]; then + echo "Error! ADMIN_DIR must be defined" + exit 1 +fi + export CONDA_TEMP_PATH="${PBS_JOBFS:-${CONDA_TEMP_PATH}}" export SCRIPT_DIR="${SCRIPT_DIR:-$PWD}" -export SCRIPT_SUBDIR="apps/cms_conda_scripts" +export SCRIPT_SUBDIR="apps/conda_scripts" export MODULE_SUBDIR="modules" export APPS_SUBDIR="apps" -export CONDA_INSTALL_BASENAME="cms_conda" -export MODULE_NAME="conda_concept" +export CONDA_INSTALL_BASENAME="base_conda" #TODO: Replace with conda? +export MODULE_NAME="conda_container" # TODO: Replace with conda? ### Derived locations - extra '.' for arcane rsync magic export CONDA_SCRIPT_PATH="${CONDA_BASE}"/./"${SCRIPT_SUBDIR}" @@ -17,8 +24,14 @@ export JOB_LOG_DIR="${ADMIN_DIR}"/logs export BUILD_STAGE_DIR="${ADMIN_DIR}"/staging ### Groups -export APPS_USERS_GROUP=hh5 -export APPS_OWNERS_GROUP=hh5_w +if [[ ! "${APPS_USERS_GROUP}" ]]; then + echo "Error! APPS_USERS_GROUP must be defined" + exit 1 +fi +if [[ ! "${APPS_OWNERS_GROUP}" ]]; then + echo "Error! APPS_OWNERS_GROUP must be defined" + exit 1 +fi ### Other settings export TEST_OUT_FILE=test_results.xml diff --git a/scripts/test_environment.py b/scripts/test_environment.py index 1e2630e..cd254f9 100644 --- a/scripts/test_environment.py +++ b/scripts/test_environment.py @@ -152,8 +152,8 @@ def test_import(): print("Error loading {}".format(info.name), file=sys.stderr) raise -def test_cdo(): - assert subprocess.call(['cdo','--version']) == 0 +# def test_cdo(): +# assert subprocess.call(['cdo','--version']) == 0 if __name__ == '__main__': test_walk_packages()