Merge remote-tracking branch 'origin/master' into stac-dir-populator

crim-ca · Nov 10, 2023 · fa78f85 · fa78f85
2 parents c011c61 + e7fd123
commit fa78f85
Show file tree

Hide file tree

Showing 13 changed files with 137 additions and 26 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,28 @@
+## IDE
+.idea/
+.vscode/
+
+## SCM
+.git*
+
+## Configurations
+.*
+*.rc
+
+## Environment
+.conda/
+.env*
+*.env
+.venv/
+jupyter/
+
+## Tests
+.coverage
+.pytest_cache
+reports
+
+## Caches
+**/__pycache__/
+STACpopulator.egg-info/
+build
+*.pyc
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -15,6 +15,7 @@ jobs:
   release:
     name: release
     runs-on: ubuntu-latest
+    if: ${{ success() && (contains(github.ref, 'refs/tags') || github.ref == 'refs/heads/master') }}
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -47,5 +48,6 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           context: .
+          file: docker/Dockerfile
           push: true
           tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.extract_branch.outputs.branch }}
diff --git a/.gitignore b/.gitignore
@@ -1,11 +1,21 @@
+## IDE
+.idea/
+.vscode/
+
+## Environment
+.conda/
+.env*
+*.env
+.venv/
+jupyter/
+
+## Tests
 .coverage
 .pytest_cache
-build
 reports
-*.pyc
+
+## Caches
+**/__pycache__/
 STACpopulator.egg-info/
-.vscode/
-.venv/
-jupyter/
-.idea
-.vscode
+build
+*.pyc
diff --git a/CHANGES.md b/CHANGES.md
@@ -2,6 +2,11 @@
 
 ## [Unreleased](https://github.com/crim-ca/stac-populator) (latest)
 
+<!-- insert list items of new changes here -->
+
+## [0.2.0](https://github.com/crim-ca/stac-populator/tree/0.2.0) (2023-11-10)
+
+
 * Add `LICENSE` file.
 * Add `bump-my-version` with `make version` and `make VERSION=<...> bump` utilities to self-update release versions.
 * Add more metadata to `pyproject.toml`.

diff --git a/Makefile b/Makefile
@@ -3,8 +3,10 @@ MAKEFILE_NAME := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
 -include Makefile.config
 APP_ROOT    := $(abspath $(lastword $(MAKEFILE_NAME))/..)
 APP_NAME    := STACpopulator
-APP_VERSION ?= 0.1.0
+APP_VERSION ?= 0.2.0
 
+DOCKER_COMPOSE_FILES := -f "$(APP_ROOT)/docker/docker-compose.yml"
+DOCKER_TAG := ghcr.io/crim-ca/stac-populator:$(APP_VERSION)
 
 IMP_DIR := $(APP_NAME)/implementations
 STAC_HOST ?= http://localhost:8880/stac
@@ -26,13 +28,16 @@ del-cmip6:
 	@echo ""
 
 docker-start:
-	docker compose up
+	docker compose $(DOCKER_COMPOSE_FILES) up
 starthost: docker-start
 
 docker-stop:
-	docker compose down
+	docker compose $(DOCKER_COMPOSE_FILES) down
 stophost: docker-stop
 
+docker-build:
+	docker build "$(APP_ROOT)" -f "$(APP_ROOT)/docker/Dockerfile" -t "$(DOCKER_TAG)"
+
 del_docker_volume: stophost
 	docker volume rm stac-populator_stac-db
 

diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 # STAC Catalog Populator
 
-![Latest Version](https://img.shields.io/badge/latest%20version-0.1.0-blue?logo=github)
-![Commits Since Latest](https://img.shields.io/github/commits-since/crim-ca/stac-populator/0.1.0.svg?logo=github)
+![Latest Version](https://img.shields.io/badge/latest%20version-0.2.0-blue?logo=github)
+![Commits Since Latest](https://img.shields.io/github/commits-since/crim-ca/stac-populator/0.2.0.svg?logo=github)
 
 This repository contains a framework [STACpopulator](STACpopulator)
 that can be used to implement concrete populators (see [implementations](STACpopulator/implementations))
@@ -22,9 +22,33 @@ Provided implementations of `STACpopulatorBase`:
 
 [CMIP6_UofT]: STACpopulator/implementations/CMIP6_UofT/add_CMIP6.py
 
+## Installation and Execution
+
+Either with Python directly (in an environment of your choosing):
+
+```shell
+pip install .
+# OR
+make install
+```
+
+With development packages:
+
+```shell
+pip install .[dev]
+# OR
+make install-dev
+```
+
+You can also employ the pre-built Docker:
+
+```shell
+docker run -ti ghcr.io/crim-ca/stac-populator:0.2.0 [command]
+```
+
 ## Testing
 
-The provided [`docker-compose`](docker-compose.yml) configuration file can be used to launch a test STAC server.
+The provided [`docker-compose`](docker/docker-compose.yml) configuration file can be used to launch a test STAC server.
 For example, the [CMIP6_UofT][CMIP6_UofT] script can be run as:
 
 ```shell

diff --git a/STACpopulator/__init__.py b/STACpopulator/__init__.py
@@ -1 +1 @@
-__version__ = "0.1.0"
+__version__ = "0.2.0"
diff --git a/STACpopulator/api_requests.py b/STACpopulator/api_requests.py
@@ -1,6 +1,6 @@
 import logging
 import os
-from typing import Any, MutableMapping, Optional
+from typing import Any, Optional
 
 import requests
 from colorlog import ColoredFormatter
@@ -35,7 +35,7 @@ def stac_collection_exists(stac_host: str, collection_id: str) -> bool:
     return r.status_code == 200
 
 
-def post_stac_collection(stac_host: str, json_data: MutableMapping[str, Any], update: Optional[bool] = True) -> None:
+def post_stac_collection(stac_host: str, json_data: dict[str, Any], update: Optional[bool] = True) -> None:
     """Post/create a collection on the STAC host
 
     :param stac_host: address of the STAC host
@@ -65,7 +65,7 @@ def post_stac_item(
     stac_host: str,
     collection_id: str,
     item_name: str,
-    json_data: MutableMapping[str, dict],
+    json_data: dict[str, dict],
     update: Optional[bool] = True,
 ) -> None:
     """Post a STAC item to the host server.

diff --git a/STACpopulator/input.py b/STACpopulator/input.py
@@ -127,7 +127,7 @@ def extract_metadata(self, ds: siphon.catalog.Dataset) -> MutableMapping[str, An
         url = ds.access_urls["NCML"]
         r = requests.get(url)
         # Convert NcML to CF-compliant dictionary
-        attrs = xncml.Dataset.from_text(r.content).to_cf_dict()
+        attrs = xncml.Dataset.from_text(r.text).to_cf_dict()
         attrs["attributes"] = numpy_to_python_datatypes(attrs["attributes"])
         attrs["access_urls"] = ds.access_urls
         return attrs

diff --git a/STACpopulator/populator_base.py b/STACpopulator/populator_base.py
@@ -2,7 +2,7 @@
 import logging
 from abc import ABC, abstractmethod
 from datetime import datetime
-from typing import Any, MutableMapping, Optional
+from typing import Any, Optional
 
 import pystac
 from colorlog import ColoredFormatter
@@ -84,7 +84,7 @@ def item_geometry_model(self):
         raise NotImplementedError
 
     @abstractmethod
-    def create_stac_item(self, item_name: str, item_data: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+    def create_stac_item(self, item_name: str, item_data: dict[str, Any]) -> dict[str, Any]:
         raise NotImplementedError
 
     def validate_host(self, stac_host: str) -> str:
@@ -99,7 +99,7 @@ def validate_host(self, stac_host: str) -> str:
     #   STAC collections are supposed to include 'summaries' with
     #   an aggregation of all supported 'properties' by its child items
     @functools.cache
-    def create_stac_collection(self) -> MutableMapping[str, Any]:
+    def create_stac_collection(self) -> dict[str, Any]:
         """
         Create a basic STAC collection.
 
@@ -125,7 +125,7 @@ def create_stac_collection(self) -> MutableMapping[str, Any]:
         self.publish_stac_collection(collection_data)
         return collection_data
 
-    def publish_stac_collection(self, collection_data: MutableMapping[str, Any]) -> None:
+    def publish_stac_collection(self, collection_data: dict[str, Any]) -> None:
         post_stac_collection(self.stac_host, collection_data, self.update)
 
     def ingest(self) -> None:

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -0,0 +1,32 @@
+FROM python:3.10-slim
+LABEL description.short="STAC Populator"
+LABEL description.long="Utility to populate STAC Catalog, Collections and Items from various dataset/catalog sources."
+LABEL maintainer="Francis Charette-Migneault <[email protected]>"
+LABEL vendor="CRIM"
+LABEL version="0.2.0"
+
+# setup paths
+ENV APP_DIR=/opt/local/src/stac-populator
+WORKDIR ${APP_DIR}
+
+# obtain source files
+COPY STACpopulator/ ${APP_DIR}/STACpopulator/
+COPY README.md LICENSE pyproject.toml ${APP_DIR}/
+
+# install runtime/package dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        netbase \
+        git \
+    && mkdir -p /home/stac/.esdoc/ \
+    && git clone "https://github.com/ES-DOC/pyessv-archive" /home/stac/.esdoc/pyessv-archive/ \
+    && pip install --no-cache-dir ${APP_DIR} \
+    && apt-get remove -y \
+        git \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -r stac && useradd -r -g stac stac
+USER stac
+
+# FIXME: use common CLI
+CMD ["bash"]
diff --git a/docker-compose.yml → docker/docker-compose.yml b/docker-compose.yml → docker/docker-compose.yml
diff --git a/pyproject.toml b/pyproject.toml
@@ -16,15 +16,15 @@ exclude = [
 
 [project]
 name = "STACpopulator"
-version = "0.1.0"
-description = "Utility for populating the STAC Catalog, Collections and Items from various dataset/catalog sources."
+version = "0.2.0"
+description = "Utility to populate STAC Catalog, Collections and Items from various dataset/catalog sources."
 requires-python = ">=3.10"
 dependencies = [
     "colorlog",
     "pyyaml",
     "siphon",
     "pystac",
-    "xncml",
+    "xncml>=0.3.1",  # python 3.12 support
     "pydantic",
     "pyessv",
     "requests",
@@ -113,7 +113,7 @@ directory = "reports/coverage/html"
 output = "reports/coverage.xml"
 
 [tool.bumpversion]
-current_version = "0.1.0"
+current_version = "0.2.0"
 commit = true
 commit_args = "--no-verify"
 tag = true
@@ -137,6 +137,11 @@ filename = "Makefile"
 search = "APP_VERSION ?= {current_version}"
 replace = "APP_VERSION ?= {new_version}"
 
+[[tool.bumpversion.files]]
+filename = "docker/Dockerfile"
+search = "LABEL version=\"{current_version}\""
+replace = "LABEL version=\"{new_version}\""
+
 [[tool.bumpversion.files]]
 filename = "CHANGES.md"
 search = "## [Unreleased](https://github.com/crim-ca/stac-populator) (latest)"