From 6dad53291e4785cba1f4818b2b1bcbe03021523a Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Tue, 16 Jul 2024 12:03:30 +1000 Subject: [PATCH 1/8] Add initial `spack.yaml` and `config` (#5) * Added initial spack.yaml and config * Deleted versions.schema.json, added $schema property * spack.yaml: Changed `spack.config.source_cache` to a string rather than array * Changed spack-packages to 2024.05.21 and intel compiler to intel@19.0.5.281 * Added newline * Reformatting spack.yaml * spack.yaml: Updated gcom4 to 2024.05.22 (HEAD of dev branch rather than master) * spack.yaml: Added extra comments around the use of branches * spack.yaml: Explicitly include cice4, um7, mom5 * spack.yaml: Added associated projections for cice4, um7, mom5, git attributes for mom5 and oasis3-mct * spack.yaml: Removed git attributes on mom5, oasis3-mct * spack.yaml: Updating the version of the overall model to 2024.05.1 * Changed to '@git.TAG=VERSION' syntax in spack.yaml, updated spack-packages to 2024.05.28 * Modified overall version of package to `2024.05.0` * spack.yaml: Updated GCOM4 to 2024.05.28 * spack.yaml: Changed require versions back to branch * spack.yaml: Modified the other requirements to use the @git.TAG=VERSION syntax * spack.yaml: Constrained netcdf-*, hdf5 versions netcdf-c @4.7.4 (same as ACCESS-OM2 MOM5) netcdf-fortran @4.5.1 hdf5 @1.10.5 * spack.yaml: Updated netcdf-fortran to @4.6.1 * spack.yaml: Updating dependencies to be in line with https://github.com/ACCESS-NRI/spack-packages/pull/121 netcdf-c uses gcc compiler target updated to x86_64_v4 multiple updates to dependency versions * Updated spack-packages/config, put spack.yaml versions in line with reproducible pre-industrial output * spack.yaml: Update netcdf-fortran to 4.5.2 * spack.yaml: put um7 on access-esm1.5 branch, tagged new mom5 commit * spack.yaml: Update modulefile version in line with package version * spack.yaml: Update netcdf-c to 4.7.4 * spack.yaml: Updated comments * spack.yaml: Updated um7 to 2024.07.03 * versions.json: Reverted to 2024.07.01 spack-config * versions.json: spack-packages version to 2024.07.03, spack-config back to 2024.07.02 * versions.json: Updated to spack 0.22 * spack.yaml: hide_implicits: false and autoload: run * spack-config 2024.07.05 using exclude_implicits: true * versions.json: spack-packages 2024.07.05 * versions.json: Updated spack-packages 2024.07.07, rebuilding with https://github.com/ACCESS-NRI/spack/commit/42e310074a69cb4ee052a73e830e07ad26a15440 * versions.json: Update spack-packages 2024.07.08 * versions.json: Update spack-packages 2024.07.10 (updated um7 spd) --- config/versions.json | 6 ++-- spack.yaml | 82 ++++++++++++++++++++++++++++---------------- 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/config/versions.json b/config/versions.json index 8837fb1..4af2f44 100644 --- a/config/versions.json +++ b/config/versions.json @@ -1,6 +1,6 @@ { "$schema": "https://github.com/ACCESS-NRI/schema/blob/main/au.org.access-nri/model/deployment/config/versions/2-0-0.json", - "spack": "0.21", - "spack-packages": "SOME_SPECIFIC_TAG", - "spack-config": "SOME_SPECIFIC_TAG" + "spack": "0.22", + "spack-packages": "2024.07.10", + "spack-config": "2024.07.05" } diff --git a/spack.yaml b/spack.yaml index 80077c5..6082f0c 100644 --- a/spack.yaml +++ b/spack.yaml @@ -4,22 +4,47 @@ # configuration settings. spack: specs: - # TODO: Replace the MODEL and VERSION. - # The root SBD for the model and overall version of the deployment: - # - MODEL@git.VERSION + - access-esm1p5@git.2024.05.0 packages: - # TODO: Specify versions and variants of dependencies where required - # Specification of dependency versions and variants goes here. - # CI/CD requires that the first element of the require is only a version: - # openmpi: - # require: - # - '@4.0.2' + # Direct ACCESS-NRI dependencies + # Note: some packages have branch-specific logic and hence can't use + # the usual '@git.DATE' calver (https://calver.org) format, instead + # requiring a `@git.DATE=BRANCH` version. + mom5: + require: + - '@git.access-esm1.5_2024.06.20=access-esm1.5' + cice4: + require: + - '@git.2024.05.21=access-esm1.5' + um7: + require: + - '@git.2024.07.03=access-esm1.5' + # Lower-level ACCESS-NRI dependencies + gcom4: + require: + - '@git.2024.05.28=access-esm1.5' + oasis3-mct: + require: + - '@git.access-esm1.5_2024.05.24=access-esm1.5' + # Other dependencies + openmpi: + require: + - '@4.0.2' + netcdf-c: + require: + - '@4.7.4' + netcdf-fortran: + require: + - '@4.5.2' + hdf5: + require: + - '@1.10.11' - # Specifications that apply to all packages + # Preferences for all packages all: - # TODO: Specify compiler/targets for all packages - # compiler: [intel@19.0.5.281] - # target: [x86_64] + require: + - '%intel@19.0.3.199' + - 'target=x86_64_v4' view: true concretizer: unify: true @@ -33,28 +58,27 @@ spack: tcl: hash_length: 0 include: - # Explicitly, which packages are accessible as modules - # TODO: Add packages that will be included as modules - # - MODEL + - access-esm1p5 + - cice4 + - um7 + - mom5 exclude_implicits: true all: - autoload: direct + autoload: run conflict: - '{name}' environment: set: 'SPACK_{name}_ROOT': '{prefix}' projections: - # TODO: Add explicit projections for modules that will be found with `module load`. - # Naming scheme for the above included modules. - # These projection versions must be the same as the - # `spack.packages.*.require[0]` version but without the `@git.`. - # Ex. `require` version `@git.2024.04.21` -> projection `2024.04.21`. all: '{name}/{version}' - # MODEL: '{name}/VERSION' - # config: - # overridden spack configurations, if needed - # mirrors: - # overridden spack package tarball directories, if needed - # repos: - # overridden repo sources, if needed + access-esm1p5: '{name}/2024.05.0' + cice4: '{name}/2024.05.21' + um7: '{name}/2024.07.03' + mom5: '{name}/access-esm1.5_2024.06.20' + config: + install_tree: + root: $spack/../restricted/ukmo/release + source_cache: $spack/../restricted/ukmo/source_cache + build_stage: + - $TMPDIR/restricted/spack-stage From f523aa0622c141fc82514f8b3be10bafa8422371 Mon Sep 17 00:00:00 2001 From: dougiesquire Date: Wed, 21 Aug 2024 16:13:10 +1000 Subject: [PATCH 2/8] Update to MOM5 ESM1.6-dev branch --- config/versions.json | 2 +- spack.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/versions.json b/config/versions.json index 4af2f44..0688bac 100644 --- a/config/versions.json +++ b/config/versions.json @@ -1,6 +1,6 @@ { "$schema": "https://github.com/ACCESS-NRI/schema/blob/main/au.org.access-nri/model/deployment/config/versions/2-0-0.json", "spack": "0.22", - "spack-packages": "2024.07.10", + "spack-packages": "2024.08.21", "spack-config": "2024.07.05" } diff --git a/spack.yaml b/spack.yaml index 6082f0c..80fd965 100644 --- a/spack.yaml +++ b/spack.yaml @@ -4,7 +4,7 @@ # configuration settings. spack: specs: - - access-esm1p5@git.2024.05.0 + - access-esm1p6@git.2024.08.0 packages: # Direct ACCESS-NRI dependencies # Note: some packages have branch-specific logic and hence can't use @@ -12,7 +12,7 @@ spack: # requiring a `@git.DATE=BRANCH` version. mom5: require: - - '@git.access-esm1.5_2024.06.20=access-esm1.5' + - '@git.dougiesquire/issue388-accessom-gtracer' cice4: require: - '@git.2024.05.21=access-esm1.5' @@ -58,7 +58,7 @@ spack: tcl: hash_length: 0 include: - - access-esm1p5 + - access-esm1p6 - cice4 - um7 - mom5 @@ -72,10 +72,10 @@ spack: 'SPACK_{name}_ROOT': '{prefix}' projections: all: '{name}/{version}' - access-esm1p5: '{name}/2024.05.0' + access-esm1p6: '{name}/2024.08.0' cice4: '{name}/2024.05.21' um7: '{name}/2024.07.03' - mom5: '{name}/access-esm1.5_2024.06.20' + mom5: '{name}/dougiesquire/issue388-accessom-gtracer' config: install_tree: root: $spack/../restricted/ukmo/release From fcbf8a852770e37ad0bdbbe3802d89ae9f4c3248 Mon Sep 17 00:00:00 2001 From: dougiesquire Date: Wed, 21 Aug 2024 16:13:36 +1000 Subject: [PATCH 3/8] Add ACCESS-ESM1.6-specific content to README --- README.md | 93 +++++++++++++++++-------------------------------------- 1 file changed, 29 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 5ba44a3..8011413 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,47 @@ -# model-deployment-template +# ACCESS-ESM1.6 -A template repository for the deployment of `spack`-based models. +## About the model +The ACCESS Earth System Model (ACCESS-ESM) is a fully-coupled global climate model that includes atmoshpere, land, ocean, sea ice, ocean biogeochemistry and land biogeochemistry components, linked together by a coupler. ACCESS-ESM1.6 is an update to ACCESS-ESM1.5, developed for CMIP7 fast tracks. -> [!NOTE] -> Feel free to replace this README with information on the model once the TODOs have been ticked off. +## Support -## Things TODO to get your model deployed +Any questions about ACCESS-NRI releases of ACCESS-ESM1.6 should be done through the [ACCESS-Hive Forum](https://forum.access-hive.org.au/). See the [ACCESS Help and Support topic](https://forum.access-hive.org.au/t/access-help-and-support/908) for details on how to do this. -### Settings +### Build -#### Repository Settings +ACCESS-NRI is using [spack](https://spack.io), a build from source package manager designed for use with high performance computing. This repository contains a [spack environment](https://spack.readthedocs.io/en/latest/environments.html) definition file ([`spack.yaml`](https://github.com/ACCESS-NRI/ACCESS-ESM1.6/blob/main/spack.yaml)) that defines all the essential components of the ACCESS-ESM1.6 model, including exact versions. -Branch protections should be set up on `main` and the special `backport/*.*` branches, which are used for backporting of fixes to major releases (the `YEAR.MONTH` portion of the `YEAR.MONTH.MINOR` version) of models. +Spack automatically builds all the components and their dependencies, producing model component executables. Spack already contains support for compiling thousands of common software packages. Spack packages for the components in ACCESS-ESM1.6 are defined in the [spack packages repository](https://github.com/ACCESS-NRI/spack_packages/). -#### Repository Secrets/Variables +ACCESS-ESM1.6 is built and deployed automatically to `gadi` on NCI (see below). However it is possible to use spack to compile the model using the `spack.yaml` environment file in this repository. To do so follow the [instructions on the ACCESS Forum for configuring spack on `gadi`](https://forum.access-hive.org.au/t/how-to-build-access-om2-on-gadi/1545). -There are a few secrets and variables that must be set at the repository level. +Then clone this repository and run the following commands on `gadi`: -##### Repository Secrets +```bash +spack env create access-esm1.6 spack.yaml +spack env activate access-esm1.6 +spack install +``` -* `BUILD_DB_CONNECTION_STR`: A postgresql connection url to the release provenance database -* `GH_COMMIT_CHECK_TOKEN`: GitHub Token that allows workflows to run based on workflow-authored commits (in the case where a user uses `!bump` commands in PRs that bumps the version of the model) +to create a spack environment called `access-esm1.6` and build all the ACCESS-ESM1.6 components, the locations of which can be found using `spack find --paths`. -##### Repository Variables +### Deployment -* `BUILD_DB_PACKAGES`: List of `spack` packages that are model components that will be uploaded to the release provenance database -* `NAME`: which corresponds to the model name - which is usually the repository name -* `CONFIG_VERSIONS_SCHEMA_VERSION`: Version of the [`config/versions.json` schema](https://github.com/ACCESS-NRI/schema/tree/main/au.org.access-nri/model/deployment/config/versions) used in this repository -* `SPACK_YAML_SCHEMA_VERSION`: Version of the [ACCESS-NRI-style `spack.yaml` schema](https://github.com/ACCESS-NRI/schema/tree/main/au.org.access-nri/model/spack/environment/deployment) used in this repository +ACCESS-ESM1.6 is deployed automatically when a new version of the [`spack.yaml`](https://github.com/ACCESS-NRI/ACCESS-ESM1.6/blob/main/spack.yaml) file is committed to `main` or a dedicated `backport/VERSION` branch. All the ACCESS-ESM1.6 components are built using `spack` on `gadi` and installed under the [`vk83`](https://my.nci.org.au/mancini/project/vk83) project in `/g/data/vk83`. It is necessary to be a member of [`vk83`](https://my.nci.org.au/mancini/project/vk83) project to use ACCESS-NRI deployments of ACCESS-ESM1.6. -#### Environment Secrets/Variables +The deployment process also creates a GitHub release with the same tag. All releases are available under the [Releases page](https://github.com/ACCESS-NRI/ACCESS-ESM1.6/releases). Each release has a changelog and meta-data with detailed information about the build and deployment, including: -GitHub Environments are sets of variables and secrets that are used specifically to deploy software, and hence have more security requirements for their use. +- paths on `gadi` to all executables built in the deployment process (`spack.location`) +- a `spack.lock` file, which is a complete build provenance document, listing all the components that were built and their dependencies, versions, compiler version, build flags and build architecture +- the environment `spack.yaml` file used for deployment -Currently, we have two Environments per deployment target - one for `Release` and one for `Prerelease`. Our current list of deployment targets and Environments can be found in this [deployment configuration file in `build-cd`](https://github.com/ACCESS-NRI/build-cd/blob/main/config/deployment-environment.json). +Additionally the deployment creates environment modulefiles, the [standard method for deploying software on `gadi`](https://opus.nci.org.au/display/Help/Environment+Modules). To view available ACCESS-ESM1.6 versions: -In order to deploy to a given deployment target: +```bash +module use /g/data/vk83/apps/spack/0.22/release/modules/linux-rocky8-x86_64 +module avail access-esm1.6 +``` -* Environments with the name of the deployment target must be created _in this repository_ and have the associated secrets/variables set ([see below](#environment-secrets)) -* There must be a `Prerelease` Environment associated with the `Release` Environment. For example, if we are deploying to `SUPERCOMPUTER`, we require Environments with the names `SUPERCOMPUTER`, `SUPERCOMPUTER Prerelease`. +For users of ACCESS-ESM1.6 model configurations released by ACCESS-NRI the exact location of the ACCESS-ESM1.6 model executables is not required. Model configurations will be updated with new model components when necessary. -When setting the environment up, remember to require sign off by a member of ACCESS-NRI when deploying as a `Release`. - -Regarding the secrets and variables that must be created: - -##### Environment Secrets - -* `HOST`: The deployment location SSH Host -* `HOST_DATA`: The deployment location SSH Host for data transfer (may be the same as `HOST`) -* `SSH_KEY`: A SSH Key that allows access to the above `HOST`/`HOST_DATA` -* `USER`: A Username to login to the above `HOST`/`HOST_DATA` - -##### Environment Variables - -* `DEPLOYMENT_TARGET`: Name of the deployment target for logging purposes -* `SPACK_INSTALLS_ROOT_LOCATION`: Path to the directory that contains all versions of a deployment of `spack`. For example, if `/some/apps/spack` is the `SPACK_INSTALLS_ROOT_LOCATION`, that directory will contain directories like `0.20`, `0.21`, `0.22`, which in turn contain an install of `spack`, `spack-packages` and `spack-config` -* `SPACK_YAML_LOCATION`: Path to a directory that will contain the `spack.yaml` from this repository during deployment -* (Optional) `SPACK_INSTALL_PARALLEL_JOBS`: Explicit number of parallel jobs for the installation of the given model. Must be either of the form `--jobs N` or unset (for the default `--jobs 16`). - -### File Modifications - -#### In `.github/workflows` - -* Reminder that these workflows use `vars.NAME` (as well as inherit the above environment secrets) and hence these must be set. -* If the name of the root SBD for the model (in [`spack-packages`](https://github.com/ACCESS-NRI/spack-packages/tree/main/packages)) is different from the model name (for example, `ACCESS-ESM1.5`s root SBD is `access-esm1p5`), you must uncomment and set the `jobs.[pr-ci|pr-comment].with.root-sbd` line to the appropriate SBD name. - -#### In `config/versions.json` - -* `.spack` must be given a version. For example, it will clone the associated `releases/VERSION` branch of `ACCESS-NRI/spack` if you give it `VERSION`. -* `.spack-packages` should also have a CalVer-compliant tag as the version. See the [associated repo](https://github.com/ACCESS-NRI/spack-packages/tags) for a list of available tags. -* `.spack-config` should also have a CalVer-compliant tag as the version. See the [associated repo](https://github.com/ACCESS-NRI/spack-config/tags) for a list of available tags. - -#### In `spack.yaml` - -There are a few TODOs for the `spack.yaml`: - -* `spack.specs`: Set the root SBD as the only element of `spack.specs`. This must also have an `@git.YEAR.MONTH.MINOR` version as it is the version of the entire deployment (and indeed will be a tag in this repository). -* `spack.packages.*`: In this section, you can specify the versions and variants of dependencies. Note that the first element of the `spack.packages.*.require` must be only a version. Variants and other configuration can be done on subsequent lines. -* `spack.packages.all`: Can set configuration for all packages. For example, the compiler used, or the target architecture. -* `spack.modules.default.tcl.include`: List of package names that will be explicitly included and available to `module load`. -* `spack.modules.default.tcl.projections`: For included modules, you must set the name of the module to be the same as the `spack.packages.*.require[0]` version, without the `@git.`. +For information on contributing your own fixes to the ACCESS-ESM1.6 `spack.yaml`, see the [CONTRIBUTING.md](./CONTRIBUTING.md) file. From c8d6f2c4657578847c8ced85443cbb0fecff2b1b Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Wed, 21 Aug 2024 16:48:31 +1000 Subject: [PATCH 4/8] ci.yml: Added root-sbd name of access-esm1p6 as repo name has illegal '.' --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff48aed..3a4609c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: uses: access-nri/build-cd/.github/workflows/ci.yml@main with: model: ${{ vars.NAME }} - # root-sbd: if different from vars.NAME + root-sbd: access-esm1p6 permissions: pull-requests: write contents: write @@ -36,7 +36,7 @@ jobs: uses: access-nri/build-cd/.github/workflows/ci-comment.yml@main with: model: ${{ vars.NAME }} - # root-sbd: if different from vars.NAME + root-sbd: access-esm1p6 permissions: pull-requests: write contents: write From 7842695e2d173903acf514bf975a8859882405f6 Mon Sep 17 00:00:00 2001 From: dougiesquire Date: Wed, 21 Aug 2024 18:01:33 +1000 Subject: [PATCH 5/8] Add mom5 version to spec --- spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack.yaml b/spack.yaml index 80fd965..571ed73 100644 --- a/spack.yaml +++ b/spack.yaml @@ -12,7 +12,7 @@ spack: # requiring a `@git.DATE=BRANCH` version. mom5: require: - - '@git.dougiesquire/issue388-accessom-gtracer' + - '@git.dougiesquire/issue388-accessom-gtracer=access-esm1.6' cice4: require: - '@git.2024.05.21=access-esm1.5' From d0a58982aa17b352f60f02f8892fd4919328520e Mon Sep 17 00:00:00 2001 From: Dougie Squire <42455466+dougiesquire@users.noreply.github.com> Date: Fri, 23 Aug 2024 15:59:52 +1000 Subject: [PATCH 6/8] Switch to tagged version of MOM5 --- spack.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spack.yaml b/spack.yaml index 571ed73..ae6185a 100644 --- a/spack.yaml +++ b/spack.yaml @@ -12,7 +12,7 @@ spack: # requiring a `@git.DATE=BRANCH` version. mom5: require: - - '@git.dougiesquire/issue388-accessom-gtracer=access-esm1.6' + - '@git.2024.08.23=access-esm1.6' cice4: require: - '@git.2024.05.21=access-esm1.5' @@ -75,7 +75,7 @@ spack: access-esm1p6: '{name}/2024.08.0' cice4: '{name}/2024.05.21' um7: '{name}/2024.07.03' - mom5: '{name}/dougiesquire/issue388-accessom-gtracer' + mom5: '{name}/2024.08.23' config: install_tree: root: $spack/../restricted/ukmo/release From ae3f9e9610ce58da4db9fbb8d4a937737481d82f Mon Sep 17 00:00:00 2001 From: Harshula Jayasuriya Date: Mon, 26 Aug 2024 15:38:59 +1000 Subject: [PATCH 7/8] Build from MOM5 development branch with new dependencies --- config/versions.json | 2 +- spack.yaml | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/config/versions.json b/config/versions.json index 0688bac..5a8ba5d 100644 --- a/config/versions.json +++ b/config/versions.json @@ -1,6 +1,6 @@ { "$schema": "https://github.com/ACCESS-NRI/schema/blob/main/au.org.access-nri/model/deployment/config/versions/2-0-0.json", "spack": "0.22", - "spack-packages": "2024.08.21", + "spack-packages": "development", "spack-config": "2024.07.05" } diff --git a/spack.yaml b/spack.yaml index ae6185a..c8deba8 100644 --- a/spack.yaml +++ b/spack.yaml @@ -12,7 +12,7 @@ spack: # requiring a `@git.DATE=BRANCH` version. mom5: require: - - '@git.2024.08.23=access-esm1.6' + - '@git.development=access-esm1.6' cice4: require: - '@git.2024.05.21=access-esm1.5' @@ -40,6 +40,15 @@ spack: require: - '@1.10.11' + access-fms: + require: + - '@development' + access-generic-tracers: + require: + - '@development' + access-mocsy: + require: + - '@mom5' # Preferences for all packages all: require: From 6a2c6a0d6f876436b24777d49940cf004efaa421 Mon Sep 17 00:00:00 2001 From: dougiesquire Date: Wed, 28 Aug 2024 09:08:29 +1000 Subject: [PATCH 8/8] Fix mom5 projection version --- spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack.yaml b/spack.yaml index c8deba8..412bfd3 100644 --- a/spack.yaml +++ b/spack.yaml @@ -84,7 +84,7 @@ spack: access-esm1p6: '{name}/2024.08.0' cice4: '{name}/2024.05.21' um7: '{name}/2024.07.03' - mom5: '{name}/2024.08.23' + mom5: '{name}/development' config: install_tree: root: $spack/../restricted/ukmo/release