From c54f56385a1f9a72688d78c7875df23d27557d1a Mon Sep 17 00:00:00 2001 From: Francis Charette-Migneault Date: Wed, 8 Nov 2023 18:23:32 -0500 Subject: [PATCH] add devtool utilities --- CHANGES.md | 9 ++++ LICENSE | 21 ++++++++ Makefile | 37 ++++++++++++- README.md | 31 ++++++++--- STACpopulator/__init__.py | 2 +- pyproject.toml | 111 +++++++++++++++++++++++++++++++++++++- requirements.txt | 6 --- 7 files changed, 200 insertions(+), 17 deletions(-) create mode 100644 CHANGES.md create mode 100644 LICENSE delete mode 100644 requirements.txt diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..ffb9e64 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,9 @@ +# Changes + +## [Unreleased](https://github.com/crim-ca/stac-populator) (latest) + + + +## [0.0.1](https://github.com/crim-ca/stac-populator/tree/0.0.1) (2023-08-22) + +* Initial release with refactored implementation of `CMIP6_UofT`. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..17ee2a6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2013-2014 Computer Research Institute of Montreal (CRIM) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile index 439f93e..4a21439 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,14 @@ -IMP_DIR = STACpopulator/implementations -STAC_HOST = http://localhost:8880/stac +MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) +# Include custom config if it is available +-include Makefile.config +APP_ROOT := $(abspath $(lastword $(MAKEFILE_NAME))/..) +APP_NAME := $(shell basename $(APP_ROOT)) +APP_VERSION ?= 0.0.1 + + +IMP_DIR := STACpopulator/implementations +STAC_HOST := http://localhost:8880/stac + testcmip6: python $(IMP_DIR)/CMIP6_UofT/add_CMIP6.py $(STAC_HOST) https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html @@ -18,3 +27,27 @@ del_docker_volume: stophost docker volume rm stac-populator_stac-db resethost: del_docker_volume starthost + + +## -- Versioning targets -------------------------------------------------------------------------------------------- ## + +# Bumpversion 'dry' config +# if 'dry' is specified as target, any bumpversion call using 'BUMP_XARGS' will not apply changes +BUMP_TOOL := bump-my-version +BUMP_XARGS ?= --verbose --allow-dirty +ifeq ($(filter dry, $(MAKECMDGOALS)), dry) + BUMP_XARGS := $(BUMP_XARGS) --dry-run +endif +.PHONY: dry +dry: pyproject.toml ## run 'bump' target without applying changes (dry-run) [make VERSION= bump dry] + @-echo > /dev/null + +.PHONY: bump +bump: ## bump version using VERSION specified as user input [make VERSION= bump] + @-echo "Updating package version ..." + @[ "${VERSION}" ] || ( echo ">> 'VERSION' is not set"; exit 1 ) + @-bash -c '$(CONDA_CMD) $(BUMP_TOOL) $(BUMP_XARGS) --new-version "${VERSION}" patch;' + +.PHONY: version +version: ## display current version + @-echo "$(APP_NAME) version: $(APP_VERSION)" diff --git a/README.md b/README.md index 808926c..6555bb8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,38 @@ # STAC Catalog Populator +![Latest Version](https://img.shields.io/badge/latest%20version-0.0.1-blue?logo=github) +![Commits Since Latest](https://img.shields.io/github/commits-since/crim-ca/stac-populator/0.0.1.svg?logo=github) -This repository contains a framework [STACpopulator](STACpopulator) that can be used to implement concrete populators (see [implementations](implementations)) for populating the STAC catalog on a DACCS node. +This repository contains a framework [STACpopulator](STACpopulator) +that can be used to implement concrete populators (see [implementations](STACpopulator/implementations)) +for populating the STAC Catalog, Collections and Items from various dataset/catalog sources, and pushed using +STAC API on a server node. ## Framework -The framwork is centered around a Python Abstract Base Class: `STACpopulatorBase` that implements all the logic for populating a STAC catalog. This class implements an abstract method called `process_STAC_item` that should be defined in implementations of the class and contain all the logic for constructing the STAC representation for an item in the collection that is to be processed. +The framework is centered around a Python Abstract Base Class: `STACpopulatorBase` that implements all the logic +for populating a STAC catalog. This class provides abstract methods that should be overridden by implementations that +contain all the logic for constructing the STAC representation for an item in the collection that is to be processed. ## Implementations -Currently, one implementation of `STACpopulatorBase` is provided in [add_CMIP6.py](implementations/add_CMIP6.py). +Provided implementations of `STACpopulatorBase`: + +- [CMIP6_UofT][CMIP6_UofT] + +[CMIP6_UofT]: STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py ## Testing -The provided `docker-compose` file can be used to launch a test STAC server. The `add_CMIP6.py` script can be run as: +The provided [`docker-compose`](docker-compose.yml) configuration file can be used to launch a test STAC server. +For example, the [CMIP6_UofT][CMIP6_UofT] script can be run as: +```shell +python STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py \ + "http://localhost:8880/stac/" \ + "https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html" \ + "STACpopulator/implementations/CMIP6_UofT/collection_config.yml" ``` -python implementations/CMIP6-UofT/add_CMIP6.py http://localhost:8880/stac/ https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/catalog/birdhouse/testdata/xclim/cmip6/catalog.html implementations/CMIP6-UofT/CMIP6.yml -``` -Note: in the script above, I am currently using a sample THREDDS catalog URL and not one relevant to the global scale CMIP6 data. + +*Note*: +In the script above, a sample THREDDS catalog URL is employed and not one relevant to the global scale CMIP6 data. diff --git a/STACpopulator/__init__.py b/STACpopulator/__init__.py index f217a91..f102a9c 100644 --- a/STACpopulator/__init__.py +++ b/STACpopulator/__init__.py @@ -1 +1 @@ -from .populator_base import STACpopulatorBase +__version__ = "0.0.1" diff --git a/pyproject.toml b/pyproject.toml index dc08b7b..3e50ac0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "STACpopulator" version = "0.0.1" +description = "Utility for populating the STAC Catalog, Collections and Items from various dataset/catalog sources." requires-python = ">=3.10" dependencies = [ "colorlog", @@ -13,10 +14,118 @@ dependencies = [ "pystac", "xncml", "pydantic", - "pyessv" + "pyessv", + "requests", + "lxml", +] +urls = [ + "https://github.com/crim-ca/stac-populator" +] +readme = "README.md" +license = { file = "LICENSE" } +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT", + "Natural Language :: English", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Topic :: Database :: Database Engines/Servers", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Information Analysis", + "Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator", + "Topic :: Utilities", +] +authors = [ + { name = "Francis Charette-Migneault", email = "francis.charette-migneault@crim.ca" }, + { name = "Deepak Chandan", email = "dchandan@cs.toronto.edu" }, + { name = "David Huard", email = "huard.david@ouranos.ca" }, +] +maintainers = [ + { name = "Francis Charette-Migneault", email = "francis.charette-migneault@crim.ca" }, + { name = "Deepak Chandan", email = "dchandan@cs.toronto.edu" }, + { name = "David Huard", email = "huard.david@ouranos.ca" }, +] +keywords = [ + "STAC", + "SpatioTemporal Asset Catalog", + "Data Ingestion", + "THREDDS", + "CMIP6" ] [tool.setuptools] py-modules = ["STACpopulator"] +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-cov", + "coverage", + "bump-my-version", +] + +[tool.pytest.ini_options] +norecursedirs = [ + ".*", + "build", + "dist", + "{arch}", + "*.egg", + "venv", + "requirements*", + "lib", +] +python_files = "test*.py" +addopts = [ + "--cov", + "--cov-report=term", + "--cov-report=html", +] + +[tool.coverage.html] +directory = "reports/coverage/html" + +[tool.coverage.xml] +output = "reports/coverage.xml" + +[tool.bumpversion] +current_version = "0.0.1" +commit = true +commit_args = "--no-verify" +tag = true +tag_name = "{new_version}" +allow_dirty = true +parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)(\\.dev\\d+)?" +serialize = [ + "{major}.{minor}.{patch}.dev{distance_to_latest_tag}", + "{major}.{minor}.{patch}" +] +message = "Version updated from {current_version} to {new_version}" + +[[tool.bumpversion.files]] +filename = "STACpopulator/__init__.py" + +[[tool.bumpversion.files]] +filename = "README.md" + +[[tool.bumpversion.files]] +filename = "Makefile" +search = "APP_VERSION ?= {current_version}" +replace = "APP_VERSION ?= {new_version}" + +[[tool.bumpversion.files]] +filename = "CHANGES.md" +search = "## [Unreleased](https://github.com/crim-ca/stac-populator) (latest)" +replace = """ +## [Unreleased](https://github.com/crim-ca/stac-populator) (latest)" + + +## [{new_version}](https://github.com/crim-ca/stac-populator/tree/{new_version}) ({now:%Y-%m-%d}) +""" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index dc03813..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -colorlog -requests -pystac -pyyaml -siphon -lxml