diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a19ade0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +CHANGELOG.md merge=union diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..0211a4e --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# CODEOWNERS file + +# Protect workflow files +/.github/ @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb +/.pre-commit-config.yaml @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb +/pyproject.toml @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml new file mode 100644 index 0000000..cd11ba2 --- /dev/null +++ b/.github/ci-hpc-config.yml @@ -0,0 +1,8 @@ +build: + python: '3.10' + modules: + - ninja + parallel: 64 + + pytest_cmd: | + python -m pytest -vv -m 'not notebook and not no_cache_init' --cov=. --cov-report=xml diff --git a/.github/workflows/changelog-pr-update.yml b/.github/workflows/changelog-pr-update.yml new file mode 100644 index 0000000..73cb1eb --- /dev/null +++ b/.github/workflows/changelog-pr-update.yml @@ -0,0 +1,18 @@ +name: Check Changelog Update on PR +on: + pull_request: + types: [assigned, opened, synchronize, reopened, labeled, unlabeled] + branches: + - main + - develop + paths-ignore: + - .pre-commit-config.yaml + - .readthedocs.yaml +jobs: + Check-Changelog: + name: Check Changelog Action + runs-on: ubuntu-20.04 + steps: + - uses: tarides/changelog-check-action@v2 + with: + changelog: CHANGELOG.md diff --git a/.github/workflows/changelog-release-update.yml b/.github/workflows/changelog-release-update.yml new file mode 100644 index 0000000..17d9525 --- /dev/null +++ b/.github/workflows/changelog-release-update.yml @@ -0,0 +1,35 @@ +# .github/workflows/update-changelog.yaml +name: "Update Changelog" + +on: + release: + types: [released] + workflow_dispatch: ~ + +permissions: + pull-requests: write + contents: write + +jobs: + update: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.event.release.target_commitish }} + + - name: Update Changelog + uses: stefanzweifel/changelog-updater-action@v1 + with: + latest-version: ${{ github.event.release.tag_name }} + heading-text: ${{ github.event.release.name }} + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v6 + with: + branch: docs/changelog-update-${{ github.event.release.tag_name }} + title: '[Changelog] Update to ${{ github.event.release.tag_name }}' + add-paths: | + CHANGELOG.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..66bb35f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,43 @@ +name: ci + +on: + # Trigger the workflow on push to master or develop, except tag creation + push: + branches: + - 'main' + - 'develop' + tags-ignore: + - '**' + paths: + - "src/**" + - "tests/**" + + # Trigger the workflow on pull request + pull_request: ~ + + # Trigger the workflow manually installs + workflow_dispatch: ~ + + # Trigger after public PR approved for CI + pull_request_target: + types: [labeled] + +jobs: + # Run CI including downstream packages on self-hosted runners + downstream-ci: + name: downstream-ci + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main + with: + anemoi-transform: ecmwf/anemoi-transform@${{ github.event.pull_request.head.sha || github.sha }} + codecov_upload: true + secrets: inherit + + # Build downstream packages on HPC + downstream-ci-hpc: + name: downstream-ci-hpc + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main + with: + anemoi-transform: ecmwf/anemoi-transform@${{ github.event.pull_request.head.sha || github.sha }} + secrets: inherit diff --git a/.github/workflows/label-public-pr.yml b/.github/workflows/label-public-pr.yml new file mode 100644 index 0000000..59b2bfa --- /dev/null +++ b/.github/workflows/label-public-pr.yml @@ -0,0 +1,10 @@ +# Manage labels of pull requests that originate from forks +name: label-public-pr + +on: + pull_request_target: + types: [opened, synchronize] + +jobs: + label: + uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2 diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..2cb554a --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,27 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + quality: + uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2 + with: + skip-hooks: "no-commit-to-branch" + + checks: + strategy: + matrix: + python-version: ["3.9", "3.10"] + uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2 + with: + python-version: ${{ matrix.python-version }} + + deploy: + needs: [checks, quality] + uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2 + secrets: inherit diff --git a/.github/workflows/python-pull-request.yml b/.github/workflows/python-pull-request.yml new file mode 100644 index 0000000..2185264 --- /dev/null +++ b/.github/workflows/python-pull-request.yml @@ -0,0 +1,23 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Code Quality checks for PRs + +on: + push: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + quality: + uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2 + with: + skip-hooks: "no-commit-to-branch" + + checks: + strategy: + matrix: + python-version: ["3.9", "3.10"] + uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2 + with: + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/readthedocs-pr-update.yml b/.github/workflows/readthedocs-pr-update.yml new file mode 100644 index 0000000..dfdd4f5 --- /dev/null +++ b/.github/workflows/readthedocs-pr-update.yml @@ -0,0 +1,22 @@ +name: Read the Docs PR Preview +on: + pull_request_target: + types: + - opened + - synchronize + - reopened + # Execute this action only on PRs that touch + # documentation files. + paths: + - "docs/**" + +permissions: + pull-requests: write + +jobs: + documentation-links: + runs-on: ubuntu-latest + steps: + - uses: readthedocs/actions/preview@v1 + with: + project-slug: "anemoi-transform" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d4a8551 --- /dev/null +++ b/.gitignore @@ -0,0 +1,189 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +*.grib +*.onnx +*.ckpt +*.swp +*.npy +*.download +? +?.* +foo +bar +*.grib +*.nc +*.npz +*.json +*.zarr/ +~$images.pptx +test.py +cutout.png +*.out + +_build/ +? +?.* +~* +*.sync +_version.py +*.code-workspace diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..cbb03cb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,87 @@ +repos: +# Empty notebookds +- repo: local + hooks: + - id: clear-notebooks-output + name: clear-notebooks-output + files: tools/.*\.ipynb$ + stages: [commit] + language: python + entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace + additional_dependencies: [jupyter] +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml # Check YAML files for syntax errors only + args: [--unsafe, --allow-multiple-documents] + - id: debug-statements # Check for debugger imports and py37+ breakpoint() + - id: end-of-file-fixer # Ensure files end in a newline + - id: trailing-whitespace # Trailing whitespace checker + - id: no-commit-to-branch # Prevent committing to main / master + - id: check-added-large-files # Check for large files added to git + - id: check-merge-conflict # Check for files that contain merge conflict +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 # Use the ref you want to point at + hooks: + - id: python-use-type-annotations # Check for missing type annotations + - id: python-check-blanket-noqa # Check for # noqa: all + - id: python-no-log-warn # Check for log.warn +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.8.0 + hooks: + - id: black + args: [--line-length=120] +- repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + args: + - -l 120 + - --force-single-line-imports + - --profile black +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.4 + hooks: + - id: ruff + # Next line if for documenation cod snippets + exclude: '.*/[^_].*_\.py$' + args: + - --line-length=120 + - --fix + - --exit-non-zero-on-fix + - --preview +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.0 + hooks: + - id: sphinx-lint +# For now, we use it. But it does not support a lot of sphinx features +- repo: https://github.com/dzhu/rstfmt + rev: v0.0.14 + hooks: + - id: rstfmt + exclude: 'cli/.*' # Because we use argparse +- repo: https://github.com/b8raoult/pre-commit-docconvert + rev: "0.1.5" + hooks: + - id: docconvert + args: ["numpy"] +- repo: https://github.com/tox-dev/pyproject-fmt + rev: "2.2.3" + hooks: + - id: pyproject-fmt +- repo: https://github.com/jshwi/docsig # Check docstrings against function sig + rev: v0.60.1 + hooks: + - id: docsig + args: + - --ignore-no-params # Allow docstrings without parameters + - --check-dunders # Check dunder methods + - --check-overridden # Check overridden methods + - --check-protected # Check protected methods + - --check-class # Check class docstrings + - --disable=E113 # Disable empty docstrings + - --summary # Print a summary +ci: + autoupdate_schedule: monthly +ci: + autoupdate_schedule: monthly diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..06c8ab4 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..063481c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Please add your functional changes to the appropriate section in the PR. +Keep it human-readable, your future self will thank you! diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2aaccba --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +# anemoi-transform + +**DISCLAIMER** +This project is **BETA** and will be **Experimental** for the foreseeable future. +Interfaces and functionality are likely to change, and the project itself may be scrapped. +**DO NOT** use this software in any project/software that is operational. + +Miscellanous data transformation functions for training data-driven weather forecasts. + +## Documentation + +The documentation can be found at https://anemoi-transform.readthedocs.io/. + +## Install + +Install via `pip` with: + +``` +$ pip install anemoi-transform +``` + +## License + +``` +Copyright 2022, European Centre for Medium Range Weather Forecasts. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +In applying this licence, ECMWF does not waive the privileges and immunities +granted to it by virtue of its status as an intergovernmental organisation +nor does it submit to any jurisdiction. +``` diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..6c0762a --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,22 @@ +#!/usr/bin/env make -f + +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/logo.png b/docs/_static/logo.png new file mode 100644 index 0000000..f78572e Binary files /dev/null and b/docs/_static/logo.png differ diff --git a/docs/_static/style.css b/docs/_static/style.css new file mode 100644 index 0000000..9a2b3af --- /dev/null +++ b/docs/_static/style.css @@ -0,0 +1,48 @@ +.wy-side-nav-search { + background-color: #f7f7f7; +} + +/*There is a clash between xarray notebook styles and readthedoc*/ + +.rst-content dl.xr-attrs dt { + all: revert; + font-size: 95%; + white-space: nowrap; +} + +.rst-content dl.xr-attrs dd { + font-size: 95%; +} + +.xr-wrap { + font-size: 85%; +} + +.wy-table-responsive table td, .wy-table-responsive table th { + white-space: inherit; +} + +/* +.wy-table-responsive table td, +.wy-table-responsive table th { + white-space: normal !important; + vertical-align: top !important; +} + +.wy-table-responsive { + margin-bottom: 24px; + max-width: 100%; + overflow: visible; +} */ + +/* Hide notebooks warnings */ +.nboutput .stderr { + display: none; +} + +/* +Set logo size +*/ +.wy-side-nav-search .wy-dropdown > a img.logo, .wy-side-nav-search > a img.logo { + width: 200px; +} diff --git a/docs/_templates/.gitkeep b/docs/_templates/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..02b57ba --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,126 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import datetime +import os +import sys + +read_the_docs_build = os.environ.get("READTHEDOCS", None) == "True" + +sys.path.insert(0, os.path.join(os.path.abspath(".."), "src")) + +source_suffix = ".rst" +master_doc = "index" +pygments_style = "sphinx" +html_theme_options = {"logo_only": True} +html_logo = "_static/logo.png" + + +# -- Project information ----------------------------------------------------- + +project = "Anemoi Transform" + +author = "ECMWF" + +year = datetime.datetime.now().year +if year == 2024: + years = "2024" +else: + years = "2024-%s" % (year,) + +copyright = "%s, ECMWF" % (years,) + +try: + from anemoi.transform._version import __version__ + + release = __version__ +except ImportError: + release = "0.0.0" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.todo", + "sphinx_rtd_theme", + "nbsphinx", + "sphinx.ext.graphviz", + "sphinx.ext.intersphinx", + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinxarg.ext", +] + +# Add any paths that contain templates here, relative to this directory. +# templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "'**.ipynb_checkpoints'"] + +intersphinx_mapping = { + "python": ("https://python.readthedocs.io/en/latest", None), + "anemoi-utils": ( + "https://anemoi-utils.readthedocs.io/en/latest/", + ("../../anemoi-utils/docs/_build/html/objects.inv", None), + ), + "anemoi-datasets": ( + "https://anemoi-datasets.readthedocs.io/en/latest/", + ("../../anemoi-datasets/docs/_build/html/objects.inv", None), + ), + "anemoi-models": ( + "https://anemoi-models.readthedocs.io/en/latest/", + ("../../anemoi-models/docs/_build/html/objects.inv", None), + ), + "anemoi-training": ( + "https://anemoi-training.readthedocs.io/en/latest/", + ("../../anemoi-training/docs/_build/html/objects.inv", None), + ), + "anemoi-inference": ( + "https://anemoi-inference.readthedocs.io/en/latest/", + ("../../anemoi-inference/docs/_build/html/objects.inv", None), + ), + "anemoi-graphs": ( + "https://anemoi-graphs.readthedocs.io/en/latest/", + ("../../anemoi-graphs/docs/_build/html/objects.inv", None), + ), + "anemoi-registry": ( + "https://anemoi-registry.readthedocs.io/en/latest/", + ("../../anemoi-registry/docs/_build/html/objects.inv", None), + ), + "anemoi-transform": ( + "https://anemoi-transform.readthedocs.io/en/latest/", + ("../../anemoi-transform/docs/_build/html/objects.inv", None), + ), +} + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] +html_css_files = ["style.css"] + + +todo_include_todos = not read_the_docs_build + +autodoc_member_order = "bysource" # Keep file order diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..a32e88c --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,62 @@ +.. _anemoi-transform: + +.. _index-page: + +############################################## + Welcome to `anemoi-transform` documentation! +############################################## + +.. warning:: + + This documentation is work in progress. + +*Anemoi* is a framework for developing machine learning weather +forecasting models. It comprises of components or packages for preparing +training datasets, conducting ML model training and a registry for +datasets and trained models. *Anemoi* provides tools for operational +inference, including interfacing to verification software. As a +framework it seeks to handle many of the complexities that +meteorological organisations will share, allowing them to easily train +models from existing recipes but with their own data. + +This package provides a series of utility functions for used by the rest +of the *Anemoi* packages. + +- :doc:`installing` + +.. toctree:: + :maxdepth: 1 + :hidden: + + installing + +********* + Modules +********* + +.. toctree:: + :maxdepth: 1 + :glob: + + modules/* + +***************** + Anemoi packages +***************** + +- :ref:`anemoi-utils ` +- :ref:`anemoi-transform ` +- :ref:`anemoi-datasets ` +- :ref:`anemoi-models ` +- :ref:`anemoi-graphs ` +- :ref:`anemoi-training ` +- :ref:`anemoi-inference ` +- :ref:`anemoi-registry ` + +********* + License +********* + +*Anemoi* is available under the open source `Apache License`__. + +.. __: http://www.apache.org/licenses/LICENSE-2.0.html diff --git a/docs/installing.rst b/docs/installing.rst new file mode 100644 index 0000000..1055752 --- /dev/null +++ b/docs/installing.rst @@ -0,0 +1,31 @@ +############ + Installing +############ + +To install the package, you can use the following command: + +.. code:: bash + + pip install anemoi-transform[...options...] + +The options are: + +- ``dev``: install the development dependencies +- ``all``: install all the dependencies + +************** + Contributing +************** + +.. code:: bash + + git clone git@github.com:ecmwf/anemoi-transform.git + cd anemoi-transform + pip install .[dev] + pip install -r docs/requirements.txt + +You may also have to install pandoc on MacOS: + +.. code:: bash + + brew install pandoc diff --git a/docs/modules/placeholder.rst b/docs/modules/placeholder.rst new file mode 100644 index 0000000..68d1836 --- /dev/null +++ b/docs/modules/placeholder.rst @@ -0,0 +1,8 @@ +############# + placeholder +############# + +.. automodule:: anemoi.transform.placeholder + :members: + :no-undoc-members: + :show-inheritance: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4dcaca3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# (C) Copyright 2024 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/ + +[build-system] +requires = [ "setuptools>=60", "setuptools-scm>=8" ] + +[project] +name = "anemoi-transform" + +description = "A package to hold various data transformation functions to support training of ML models on ECMWF data." +keywords = [ "ai", "tools" ] + +license = { file = "LICENSE" } +authors = [ + { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" }, +] + +requires-python = ">=3.9" + +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +dynamic = [ "version" ] +dependencies = [ + "earthkit-data", + "earthkit-meteo", +] + +optional-dependencies.all = [ "anemoi-transform" ] +optional-dependencies.dev = [ "anemoi-transform[all,docs,tests]" ] + +optional-dependencies.docs = [ + "nbsphinx", + "pandoc", + "requests", + "sphinx", + "sphinx-argparse<0.5", + "sphinx-rtd-theme", + "termcolor", +] + +optional-dependencies.tests = [ "pytest" ] + +urls.Documentation = "https://anemoi-transform.readthedocs.io/" +urls.Homepage = "https://github.com/ecmwf/anemoi-transform/" +urls.Issues = "https://github.com/ecmwf/anemoi-transform/issues" +urls.Repository = "https://github.com/ecmwf/anemoi-transform/" + +scripts.anemoi-transform = "anemoi.transform.__main__:main" + +[tool.setuptools_scm] +version_file = "src/anemoi/transform/_version.py" diff --git a/src/anemoi/transform/__init__.py b/src/anemoi/transform/__init__.py new file mode 100644 index 0000000..7a799a1 --- /dev/null +++ b/src/anemoi/transform/__init__.py @@ -0,0 +1,9 @@ +# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +from ._version import __version__ diff --git a/src/anemoi/transform/__main__.py b/src/anemoi/transform/__main__.py new file mode 100644 index 0000000..be940c2 --- /dev/null +++ b/src/anemoi/transform/__main__.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# (C) Copyright 2024 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. +# + +from anemoi.utils.cli import cli_main +from anemoi.utils.cli import make_parser + +from . import __version__ +from .commands import COMMANDS + + +# For read-the-docs +def create_parser(): + return make_parser(__doc__, COMMANDS) + + +def main(): + cli_main(__version__, __doc__, COMMANDS) + + +if __name__ == "__main__": + main() diff --git a/src/anemoi/transform/commands/__init__.py b/src/anemoi/transform/commands/__init__.py new file mode 100644 index 0000000..cebb539 --- /dev/null +++ b/src/anemoi/transform/commands/__init__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +# (C) Copyright 2024 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. +# + +import os + +from anemoi.utils.cli import Command +from anemoi.utils.cli import Failed +from anemoi.utils.cli import register_commands + +__all__ = ["Command"] + +COMMANDS = register_commands( + os.path.dirname(__file__), + __name__, + lambda x: x.command(), + lambda name, error: Failed(name, error), +) diff --git a/src/anemoi/transform/commands/hello.py b/src/anemoi/transform/commands/hello.py new file mode 100644 index 0000000..793d126 --- /dev/null +++ b/src/anemoi/transform/commands/hello.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# (C) Copyright 2024 ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. +# + + +from . import Command + + +class Hello(Command): + + def add_arguments(self, command_parser): + command_parser.add_argument("--what", help="Say hello to someone") + + def run(self, args): + if args.what: + print(f"Hello {args.what}!") + else: + print("Hello!") + + +command = Hello diff --git a/src/anemoi/transform/placeholder.py b/src/anemoi/transform/placeholder.py new file mode 100644 index 0000000..1b4d478 --- /dev/null +++ b/src/anemoi/transform/placeholder.py @@ -0,0 +1,9 @@ +# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +# Remove this file when we have a few functions diff --git a/tests/test_transform.py b/tests/test_transform.py new file mode 100644 index 0000000..edacad9 --- /dev/null +++ b/tests/test_transform.py @@ -0,0 +1,17 @@ +# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +def test_transform(): + pass + + +if __name__ == "__main__": + for name, obj in list(globals().items()): + if name.startswith("test_") and callable(obj): + print(f"Running {name}...") + obj()