First release

.github/ci-config.yml

@@ -0,0 +1,9 @@
+dependencies: |
+  ecmwf/ecbuild
+  MathisRosenhauer/libaec@master
+  ecmwf/eccodes
+  ecmwf/eckit
+  ecmwf/odc
+dependency_branch: develop
+parallelism_factor: 8
+self_build: false # Only for python packages

.github/workflows/changelog-release-update.yml

@@ -0,0 +1,36 @@
+# .github/workflows/update-changelog.yaml
+name: "Update Changelog"
+
+on:
+  release:
+    types: [released]
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push the
+      # updated CHANGELOG back to the repository.
+      # https://github.blog/changelog/2023-02-02-github-actions-updating-the-default-github_token-permissions-to-read-only/
+      contents: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.release.target_commitish }}
+
+      - name: Update Changelog
+        uses: stefanzweifel/changelog-updater-action@v1
+        with:
+          latest-version: ${{ github.event.release.tag_name }}
+          heading-text: ${{ github.event.release.name }}
+          release-notes: ${{ github.event.release.body }}
+
+      - name: Commit updated CHANGELOG
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          branch: ${{ github.event.release.target_commitish }}
+          commit_message: Update CHANGELOG
+          file_pattern: CHANGELOG.md

.github/workflows/ci.yml

@@ -0,0 +1,43 @@
+name: ci
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - 'main'
+      - 'develop'
+    tags-ignore:
+      - '**'
+    paths:
+      - "src/**"
+      - "tests/**"
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manuallyp instals
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      anemoi-training: ecmwf/anemoi-training@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+    secrets: inherit
+
+   # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      anemoi-training: ecmwf/anemoi-training@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit

.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

.github/workflows/python-publish.yml

@@ -35,7 +35,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -48,27 +48,6 @@ jobs:
       run: pytest
 
   deploy:
-
-    if: ${{ github.event_name == 'release' }}
-    runs-on: ubuntu-latest
     needs: [checks, quality]
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.x
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install build wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
-      run: |
-        python -m build
-        twine upload dist/*
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit

.github/workflows/readtthedocs-pr-update.yml

@@ -0,0 +1,22 @@
+name: Read the Docs PR Preview
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    # Execute this action only on PRs that touch
+    # documentation files.
+    paths:
+      - "docs/**"
+
+permissions:
+  pull-requests: write
+
+jobs:
+  documentation-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/actions/preview@v1
+        with:
+          project-slug: "anemoi-training"

.gitignore

@@ -185,3 +185,4 @@ _build/
 ~*
 *.sync
 _version.py
+outputs/

.pre-commit-config.yaml

@@ -9,6 +9,12 @@ repos:
     language: python
     entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
     additional_dependencies: [jupyter]
+- repo: https://github.com/compilerla/conventional-pre-commit
+  rev: v3.4.0
+  hooks:
+    - id: conventional-pre-commit
+      stages: [commit-msg]
+      args: []
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v4.6.0
   hooks:
@@ -20,8 +26,14 @@ repos:
   - id: no-commit-to-branch # Prevent committing to main / master
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0  # Use the ref you want to point at
+    hooks:
+    -   id: python-use-type-annotations # Check for missing type annotations
+    -   id: python-check-blanket-noqa # Check for # noqa: all
+    -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.4.2
+  rev: 24.8.0
   hooks:
   - id: black
     args: [--line-length=120]
@@ -34,7 +46,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.6
+  rev: v0.5.6
   hooks:
   - id: ruff
     # Next line if for documenation cod snippets
@@ -59,12 +71,19 @@ repos:
   hooks:
   - id: docconvert
     args: ["numpy"]
-- repo: https://github.com/b8raoult/optional-dependencies-all
-  rev: "0.0.6"
-  hooks:
-  - id: optional-dependencies-all
-    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "2.1.3"
+  rev: "2.2.1"
   hooks:
   - id: pyproject-fmt
+-   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
+    rev: v0.60.0
+    hooks:
+    -   id: docsig
+        args:
+        - --ignore-no-params # Allow docstrings without parameters
+        - --check-dunders    # Check dunder methods
+        - --check-overridden # Check overridden methods
+        - --check-protected  # Check protected methods
+        - --check-class      # Check class docstrings
+        - --disable=E113     # Disable empty docstrings
+        - --summary          # Print a summary

CHANGELOG.md

@@ -1,19 +1,67 @@
 # Changelog
 
- All notable changes to this project will be documented in this file.
+All notable changes to this project will be documented in this file.
 
- The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
- and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
- Please add your functional changes to the appropriate section in the PR.
- Keep it human-readable, your future self will thank you!
+Please add your functional changes to the appropriate section in the PR.
+Keep it human-readable, your future self will thank you!
 
- ## [Unreleased]
+## [Unreleased]
 
- ### Added
- - MLflow token authentication
+### Added
+
+#### Subcommands
+ - Subcommand for training `anemoi-training train`
+ - Subcommand for config generation of configs
+ - Subcommand for mlflow: login and sync
+ - Subcommand for checkpoint handling
+
+#### Functionality
+ - Searchpaths for Hydra configs, to enable configs in CWD, `ANEMOI_CONFIG_PATH` env, and `.config/anemoi/training` in addition to package defaults
+ - MlFlow token authentication
+ - Configurable pressure level scaling
+
+#### Continuous Integration / Deployment
+ - Downstream CI to test all dependencies with changes
+ - Changelog Status check
+ - Readthedocs PR builder
+ - Changelog Release Updater Workflow
+
+#### Miscellaneous
+ - Extended ruff Ruleset
+ - Added Docsig pre-commit hook
+ - `__future__` annotations for typehints
+ - Added Typehints where missing
  - Added Changelog
 
- ### Changed
+### Changed
+
+#### Move to Anemoi Ecosystem
+ - Fixed PyPI packaging
+ - Use of Anemoi models
+ - Use of Anemoi graphs
+ - Adjusted tests to work with new Anemoi ecosystem
+ - Adjusted configs to reasonable common defaults
+
+#### Functionality
+ - Changed hardware-specific keys from configs to `???` to trigger "missing"
+ - `__len__` of NativeGridDataset
+ - Configurable dropout in attention layer
+
+#### Docs
+ - Fixed docstrings
+
+#### Miscellaneous
+ - Moved callbacks into folder to fascilitate future refactor
+ - Adjusted PyPI release infrastructure to common ECMWF workflow
+ - Bumped versions in Pre-commit hooks
+
+### Removed
+ - Dependency on mlflow-export-import
+ - Specific user configs
+
+<!-- Add Git Diffs for Links above -->
 
- ### Removed
+[unreleased]: https://github.com/ecmwf/anemoi-training/compare/x.x.x...HEAD

docs/conf.py

@@ -33,12 +33,9 @@
 author = "ECMWF"
 
 year = datetime.datetime.now().year
-if year == 2024:
-    years = "2024"
-else:
-    years = "2024-%s" % (year,)
+years = "2024" if year == 2024 else f"2024-{year}"
 
-copyright = "%s, ECMWF" % (years,)
+copyright = f"{years}, ECMWF"
 
 
 try: