From 1855b73edde345ac8ff0fc65089bb6da6a0d5388 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 10 Nov 2023 10:08:47 -0500 Subject: [PATCH 1/6] update xncml version for python 3.12 support --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ac83d51..efeff4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "pyyaml", "siphon", "pystac", - "xncml", + "xncml>=0.3.1", # python 3.12 support "pydantic", "pyessv", "requests", From 1b0ba5c9c3c7e474a21cc991da4662fd12643807 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 10 Nov 2023 11:59:58 -0500 Subject: [PATCH 2/6] add dockerfile and related configs --- .dockerignore | 28 ++++++++++++++++ .github/workflows/release.yml | 3 ++ .gitignore | 24 ++++++++++---- Makefile | 9 ++++-- README.md | 26 ++++++++++++++- docker/Dockerfile | 32 +++++++++++++++++++ .../docker-compose.yml | 0 pyproject.toml | 7 +++- 8 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 .dockerignore create mode 100644 docker/Dockerfile rename docker-compose.yml => docker/docker-compose.yml (100%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cf551b9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,28 @@ +## IDE +.idea/ +.vscode/ + +## SCM +.git* + +## Configurations +.* +*.rc + +## Environment +.conda/ +.env* +*.env +.venv/ +jupyter/ + +## Tests +.coverage +.pytest_cache +reports + +## Caches +**/__pycache__/ +STACpopulator.egg-info/ +build +*.pyc diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 47c1e7b..f937cb3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,6 +15,8 @@ jobs: release: name: release runs-on: ubuntu-latest + # FIXME: until working + ## if: ${{ success() && (contains(github.ref, 'refs/tags') || github.ref == 'refs/heads/master') }} steps: - name: Checkout uses: actions/checkout@v2 @@ -47,5 +49,6 @@ jobs: uses: docker/build-push-action@v3 with: context: . + file: docker/Dockerfile push: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.extract_branch.outputs.branch }} diff --git a/.gitignore b/.gitignore index c6ac5f8..9d5674a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,21 @@ +## IDE +.idea/ +.vscode/ + +## Environment +.conda/ +.env* +*.env +.venv/ +jupyter/ + +## Tests .coverage .pytest_cache -build reports -*.pyc + +## Caches +**/__pycache__/ STACpopulator.egg-info/ -.vscode/ -.venv/ -jupyter/ -.idea -.vscode +build +*.pyc diff --git a/Makefile b/Makefile index 19ca1c0..bd5daeb 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,8 @@ APP_ROOT := $(abspath $(lastword $(MAKEFILE_NAME))/..) APP_NAME := STACpopulator APP_VERSION ?= 0.1.0 +DOCKER_COMPOSE_FILES := -f "$(APP_ROOT)/docker/docker-compose.yml" +DOCKER_TAG := ghcr.io/crim-ca/stac-populator:$(APP_VERSION) IMP_DIR := $(APP_NAME)/implementations STAC_HOST ?= http://localhost:8880/stac @@ -26,13 +28,16 @@ del-cmip6: @echo "" docker-start: - docker compose up + docker compose $(DOCKER_COMPOSE_FILES) up starthost: docker-start docker-stop: - docker compose down + docker compose $(DOCKER_COMPOSE_FILES) down stophost: docker-stop +docker-build: + docker build "$(APP_ROOT)" -f "$(APP_ROOT)/docker/Dockerfile" -t "$(DOCKER_TAG)" + del_docker_volume: stophost docker volume rm stac-populator_stac-db diff --git a/README.md b/README.md index cb56e4c..4efb3ee 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,33 @@ Provided implementations of `STACpopulatorBase`: [CMIP6_UofT]: STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py +## Installation and Execution + +Either with Python directly (in an environment of your choosing): + +```shell +pip install . +# OR +make install +``` + +With development packages: + +```shell +pip install .[dev] +# OR +make install-dev +``` + +You can also employ the pre-built Docker: + +```shell +docker run -ti ghcr.io/crim-ca/stac-populator:0.1.0 [command] +``` + ## Testing -The provided [`docker-compose`](docker-compose.yml) configuration file can be used to launch a test STAC server. +The provided [`docker-compose`](docker/docker-compose.yml) configuration file can be used to launch a test STAC server. For example, the [CMIP6_UofT][CMIP6_UofT] script can be run as: ```shell diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..07c269c --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.10-slim +LABEL description.short="STAC Populator" +LABEL description.long="Utility to populate STAC Catalog, Collections and Items from various dataset/catalog sources." +LABEL maintainer="Francis Charette-Migneault " +LABEL vendor="CRIM" +LABEL version="0.1.0" + +# setup paths +ENV APP_DIR=/opt/local/src/stac-populator +WORKDIR ${APP_DIR} + +# obtain source files +COPY STACpopulator/ ${APP_DIR}/STACpopulator/ +COPY README.md LICENSE pyproject.toml ${APP_DIR}/ + +# install runtime/package dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + netbase \ + git \ + && mkdir -p /home/stac/.esdoc/ \ + && git clone "https://github.com/ES-DOC/pyessv-archive" /home/stac/.esdoc/pyessv-archive/ \ + && pip install --no-cache-dir ${APP_DIR} \ + && apt-get remove -y \ + git \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd -r stac && useradd -r -g stac stac +USER stac + +# FIXME: use common CLI +CMD ["bash"] diff --git a/docker-compose.yml b/docker/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to docker/docker-compose.yml diff --git a/pyproject.toml b/pyproject.toml index efeff4e..ca64cba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ exclude = [ [project] name = "STACpopulator" version = "0.1.0" -description = "Utility for populating the STAC Catalog, Collections and Items from various dataset/catalog sources." +description = "Utility to populate STAC Catalog, Collections and Items from various dataset/catalog sources." requires-python = ">=3.10" dependencies = [ "colorlog", @@ -134,6 +134,11 @@ filename = "Makefile" search = "APP_VERSION ?= {current_version}" replace = "APP_VERSION ?= {new_version}" +[[tool.bumpversion.files]] +filename = "docker/Dockerfile" +search = "LABEL version=\"{current_version}\"" +replace = "LABEL version=\"{new_version}\"" + [[tool.bumpversion.files]] filename = "CHANGES.md" search = "## [Unreleased](https://github.com/crim-ca/stac-populator) (latest)" From d108ef5af3eba8d60b10ab5de9901d060b060882 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 10 Nov 2023 12:12:45 -0500 Subject: [PATCH 3/6] limit docker build CI release to new versions or latest now that it is confirmed to work --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f937cb3..ec87644 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,8 +15,7 @@ jobs: release: name: release runs-on: ubuntu-latest - # FIXME: until working - ## if: ${{ success() && (contains(github.ref, 'refs/tags') || github.ref == 'refs/heads/master') }} + if: ${{ success() && (contains(github.ref, 'refs/tags') || github.ref == 'refs/heads/master') }} steps: - name: Checkout uses: actions/checkout@v2 From 0d99de281d08182b5c0df94e67f3971113c588e7 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 10 Nov 2023 12:47:36 -0500 Subject: [PATCH 4/6] revert MutableMapping to dict but align caller types with it --- STACpopulator/api_requests.py | 6 +++--- STACpopulator/populator_base.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/STACpopulator/api_requests.py b/STACpopulator/api_requests.py index a67473b..e7a5380 100644 --- a/STACpopulator/api_requests.py +++ b/STACpopulator/api_requests.py @@ -1,6 +1,6 @@ import logging import os -from typing import Any, MutableMapping, Optional +from typing import Any, Optional import requests from colorlog import ColoredFormatter @@ -35,7 +35,7 @@ def stac_collection_exists(stac_host: str, collection_id: str) -> bool: return r.status_code == 200 -def post_stac_collection(stac_host: str, json_data: MutableMapping[str, Any], update: Optional[bool] = True) -> None: +def post_stac_collection(stac_host: str, json_data: dict[str, Any], update: Optional[bool] = True) -> None: """Post/create a collection on the STAC host :param stac_host: address of the STAC host @@ -65,7 +65,7 @@ def post_stac_item( stac_host: str, collection_id: str, item_name: str, - json_data: MutableMapping[str, dict], + json_data: dict[str, dict], update: Optional[bool] = True, ) -> None: """Post a STAC item to the host server. diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py index 83eb2b0..55db015 100644 --- a/STACpopulator/populator_base.py +++ b/STACpopulator/populator_base.py @@ -2,7 +2,7 @@ import logging from abc import ABC, abstractmethod from datetime import datetime -from typing import Any, MutableMapping, Optional +from typing import Any, Optional import pystac from colorlog import ColoredFormatter @@ -84,7 +84,7 @@ def item_geometry_model(self): raise NotImplementedError @abstractmethod - def create_stac_item(self, item_name: str, item_data: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + def create_stac_item(self, item_name: str, item_data: dict[str, Any]) -> dict[str, Any]: raise NotImplementedError def validate_host(self, stac_host: str) -> str: @@ -99,7 +99,7 @@ def validate_host(self, stac_host: str) -> str: # STAC collections are supposed to include 'summaries' with # an aggregation of all supported 'properties' by its child items @functools.cache - def create_stac_collection(self) -> MutableMapping[str, Any]: + def create_stac_collection(self) -> dict[str, Any]: """ Create a basic STAC collection. @@ -125,7 +125,7 @@ def create_stac_collection(self) -> MutableMapping[str, Any]: self.publish_stac_collection(collection_data) return collection_data - def publish_stac_collection(self, collection_data: MutableMapping[str, Any]) -> None: + def publish_stac_collection(self, collection_data: dict[str, Any]) -> None: post_stac_collection(self.stac_host, collection_data, self.update) def ingest(self) -> None: From 3cd9e24f49827e99cd306f11c2f989a0d9a6b2da Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 10 Nov 2023 12:48:20 -0500 Subject: [PATCH 5/6] fix wrong request content/bytes to text/str type --- STACpopulator/input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/STACpopulator/input.py b/STACpopulator/input.py index a5d2774..f72bc6e 100644 --- a/STACpopulator/input.py +++ b/STACpopulator/input.py @@ -125,7 +125,7 @@ def extract_metadata(self, ds: siphon.catalog.Dataset) -> MutableMapping[str, An url = ds.access_urls["NCML"] r = requests.get(url) # Convert NcML to CF-compliant dictionary - attrs = xncml.Dataset.from_text(r.content).to_cf_dict() + attrs = xncml.Dataset.from_text(r.text).to_cf_dict() attrs["attributes"] = numpy_to_python_datatypes(attrs["attributes"]) attrs["access_urls"] = ds.access_urls return attrs From e7fd123c40ebd404ade128cd5c3fab7e6a167a34 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 10 Nov 2023 14:06:40 -0500 Subject: [PATCH 6/6] Version updated from 0.1.0 to 0.2.0 --- CHANGES.md | 5 +++++ Makefile | 2 +- README.md | 6 +++--- STACpopulator/__init__.py | 2 +- docker/Dockerfile | 2 +- pyproject.toml | 4 ++-- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 3a3eece..0b6b22d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,11 @@ ## [Unreleased](https://github.com/crim-ca/stac-populator) (latest) + + +## [0.2.0](https://github.com/crim-ca/stac-populator/tree/0.2.0) (2023-11-10) + + * Add `LICENSE` file. * Add `bump-my-version` with `make version` and `make VERSION=<...> bump` utilities to self-update release versions. * Add more metadata to `pyproject.toml`. diff --git a/Makefile b/Makefile index bd5daeb..5f80f6f 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) -include Makefile.config APP_ROOT := $(abspath $(lastword $(MAKEFILE_NAME))/..) APP_NAME := STACpopulator -APP_VERSION ?= 0.1.0 +APP_VERSION ?= 0.2.0 DOCKER_COMPOSE_FILES := -f "$(APP_ROOT)/docker/docker-compose.yml" DOCKER_TAG := ghcr.io/crim-ca/stac-populator:$(APP_VERSION) diff --git a/README.md b/README.md index 4efb3ee..6d5893b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # STAC Catalog Populator -![Latest Version](https://img.shields.io/badge/latest%20version-0.1.0-blue?logo=github) -![Commits Since Latest](https://img.shields.io/github/commits-since/crim-ca/stac-populator/0.1.0.svg?logo=github) +![Latest Version](https://img.shields.io/badge/latest%20version-0.2.0-blue?logo=github) +![Commits Since Latest](https://img.shields.io/github/commits-since/crim-ca/stac-populator/0.2.0.svg?logo=github) This repository contains a framework [STACpopulator](STACpopulator) that can be used to implement concrete populators (see [implementations](STACpopulator/implementations)) @@ -43,7 +43,7 @@ make install-dev You can also employ the pre-built Docker: ```shell -docker run -ti ghcr.io/crim-ca/stac-populator:0.1.0 [command] +docker run -ti ghcr.io/crim-ca/stac-populator:0.2.0 [command] ``` ## Testing diff --git a/STACpopulator/__init__.py b/STACpopulator/__init__.py index 3dc1f76..d3ec452 100644 --- a/STACpopulator/__init__.py +++ b/STACpopulator/__init__.py @@ -1 +1 @@ -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/docker/Dockerfile b/docker/Dockerfile index 07c269c..235d708 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,7 +3,7 @@ LABEL description.short="STAC Populator" LABEL description.long="Utility to populate STAC Catalog, Collections and Items from various dataset/catalog sources." LABEL maintainer="Francis Charette-Migneault " LABEL vendor="CRIM" -LABEL version="0.1.0" +LABEL version="0.2.0" # setup paths ENV APP_DIR=/opt/local/src/stac-populator diff --git a/pyproject.toml b/pyproject.toml index ca64cba..ac0db2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ exclude = [ [project] name = "STACpopulator" -version = "0.1.0" +version = "0.2.0" description = "Utility to populate STAC Catalog, Collections and Items from various dataset/catalog sources." requires-python = ">=3.10" dependencies = [ @@ -110,7 +110,7 @@ directory = "reports/coverage/html" output = "reports/coverage.xml" [tool.bumpversion] -current_version = "0.1.0" +current_version = "0.2.0" commit = true commit_args = "--no-verify" tag = true