Add STAC catalog

CHANGES.md

@@ -15,7 +15,30 @@
 [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest)
 ------------------------------------------------------------------------------------------------------------------
 
-[//]: # (list changes here, using '-' for each new entry, remove this when items are added)
+## Changes
+- Adds [STAC](https://github.com/crim-ca/stac-app) to the stack (optional) when ``./components/stac`` 
+  is added to ``EXTRA_CONF_DIRS``. For more details, refer to 
+  [STAC Component](https://github.com/bird-house/birdhouse-deploy/blob/master/birdhouse/components/README.rst#STAC)
+  Following happens when enabled:
+
+  * Service ``stac`` (API) gets added with endpoints ``/twitcher/ows/proxy/stac`` and ``/stac``.
+
+  * STAC catalog can be explored via the ``stac-browser`` component, available under ``/stac-browser``.
+
+  * Image [crim-ca/stac-app](https://github.com/crim-ca/stac-app) is a STAC implementation based on 
+  [stac-utils/stac-fastapi](https://github.com/stac-utils/stac-fastapi).
+
+  * Image [crim-ca/stac-browser](https://github.com/crim-ca/stac-browser) is a fork of 
+  [radiantearth/stac-browser](https://github.com/radiantearth/stac-browser) in order to have the capacity to build 
+  the Docker container. The image reference will change when the 
+  [stac-browser PR related to Dockerfile](https://github.com/bird-house/birdhouse-deploy/issues/346) will have been 
+  merged.
+
+  * Adds `Magpie` permissions and service for `stac` endpoints.
+
+- Adds [stac-populator](https://github.com/crim-ca/stac-populator) to populate STAC catalog with sample collection 
+  items via [CEDA STAC Generator](https://github.com/cedadev/stac-generator), employed in sample 
+  [CMIP Dataset Ingestion Workflows](https://github.com/cedadev/stac-generator-example/tree/master/conf).
 
 [1.26.7](https://github.com/bird-house/birdhouse-deploy/tree/1.26.7) (2023-06-19)
 ------------------------------------------------------------------------------------------------------------------

birdhouse/components/README.rst

@@ -378,3 +378,40 @@ define your custom values in ``env.local`` directly.
 
 .. |cowbird-default| replace:: cowbird/default.env
 .. _cowbird-default: ./cowbird/default.env
+
+
+STAC
+====
+
+`STAC`_ is the common name of the REST API that implements the STAC specification, common representation of geospatial 
+information.
+
+.. _STAC: https://stacspec.org/en
+
+Usage
+-----
+
+The STAC API can be browsed via the ``stac-browser`` component. By default, the browser will point to the STAC API 
+exposed by the current stack instance. Once this component is enabled, STAC API will be accessible at 
+``https://<PAVICS_FQDN_PUBLIC>/stac`` endpoint and the STAC browser will be available at 
+``https://<PAVICS_FQDN_PUBLIC>/stac-browser`` endpoint. In order to make the STAC browser the default entrypoint, 
+define the following in the ``env.local`` file::
+
+  export PROXY_ROOT_LOCATION="return 302 https://\$host/stac-browser;"
+
+Here is a sample search query using a CLI::
+
+.. code-block:: shell
+
+    pip install pystac-client
+    stac-client search $PAVIS_FQDN/stac -q "variable_id=txgt_32" "scenario=ssp585"
+
+Calls to the STAC API pass through Twitcher in order to validate authorization. Unauthenticated users will have 
+read-only access by default to STAC API resources while members of the `stac-admin` group can create and modify 
+resources. STAC Browser is not protected by any authorization mechanism.
+
+How to Enable the Component
+---------------------------
+
+- Edit ``env.local`` (a copy of `env.local.example`_)
+- Add ``"./optional-components/stac"`` to ``EXTRA_CONF_DIRS``.

birdhouse/components/stac/.gitignore

@@ -0,0 +1 @@
+config/proxy/conf.extra-service.d/stac.conf

birdhouse/components/stac/config/canarie-api/_docker-compose-extra.yml

@@ -0,0 +1,5 @@
+version: "3.4"
+services:
+  proxy:
+    volumes:
+    - ./components/stac/config/canarie-api/canarie_api_monitoring.py:${CANARIE_MONITORING_EXTRA_CONF_DIR}/stac_canarie_api_monitoring.py:ro

birdhouse/components/stac/config/canarie-api/canarie_api_monitoring.py.template

@@ -0,0 +1,35 @@
+SERVICES['STAC'] = {
+    'info': {
+        'name': 'STAC',
+        'synopsis': 'STAC is the common name of the REST API that implements the STAC specification, common representation of geospatial information.',
+        'version': "",
+        'institution': 'CRIM',
+        'releaseTime': "2023-06-16T00:00:00Z",
+        'researchSubject': 'Any',
+        'supportEmail': '${SUPPORT_EMAIL}',
+        'category': 'Data Manipulation',
+        'tags': ['Catalog', 'Data', 'OGC']
+    },
+    'stats': {
+        'method': '.*',
+        'route': "/stac/.*"
+    },
+    'redirect': {
+        'doc': 'https://stac-utils.github.io/stac-fastapi/',
+        'releasenotes': 'https://github.com/crim-ca/sac-app/blob/master/CHANGES.rst',
+        'support': 'https://github.com/crim-ca/stac-app/issues',
+        'source': 'https://github.com/crim-ca/stac-app',
+        'tryme': 'https://${PAVICS_FQDN_PUBLIC}/stac/',
+        'licence': 'https://github.com/crim-ca/stac-app/blob/master/LICENSE',
+        'provenance': 'https://github.com/crim-ca/stac-app'
+    },
+    "monitoring": {
+        "STAC": {
+            'request': {
+                'url': 'http://stac:8000/stac'
+            }
+        }
+    }
+}
+
+# vi: tabstop=8 expandtab shiftwidth=4 softtabstop=4 syntax=python

birdhouse/components/stac/config/proxy/conf.extra-service.d/stac.conf.template

@@ -0,0 +1,29 @@
+
+    location /stac {
+        # We need the first `/stac` for service resolution.
+        # We need the second `/stac` for API redirect in STAC (see `root-path` and `ROUTER_PREFIX`).
+        # See https://github.com/stac-utils/stac-fastapi/issues/427
+        # See https://github.com/crim-ca/stac-app/blob/main/stac_app.py#L60
+        proxy_pass https://${PAVICS_FQDN_PUBLIC}${TWITCHER_PROTECTED_PATH}/stac/stac;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host $host:$server_port;
+        proxy_buffering off;
+        include /etc/nginx/conf.d/cors.include;
+    }
+
+    location /stac-browser/ {
+        # STAC API is protected behind Twitcher so we might not need to protect the browser as well.
+        # In case we encounter a valid use case in which we need to protect the browser, we might 
+        # consider using Twitcher's verify capability to protect the route in an efficient manner.
+        proxy_pass http://stac-browser:8080/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-Host localhost;
+        proxy_set_header X-Forwarded-Server localhost;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header Origin localhost;
+        proxy_hide_header Access-Control-Allow-Origin;
+        proxy_redirect off;
+    }

birdhouse/components/stac/config/proxy/docker-compose-extra.yml

@@ -0,0 +1,5 @@
+version: "3.4"
+services:
+  proxy:
+    volumes:
+    - ./components/stac/config/proxy/conf.extra-service.d:/etc/nginx/conf.extra-service.d/stac:ro

birdhouse/components/stac/default.env

@@ -0,0 +1,16 @@
+export STAC_POSTGRES_USER=${POSTGRES_PAVICS_USERNAME}
+export STAC_POSTGRES_PASSWORD=${POSTGRES_PAVICS_PASSWORD}
+export STAC_PGUSER=${POSTGRES_PAVICS_USERNAME}
+export STAC_PGPASSWORD=${POSTGRES_PAVICS_PASSWORD}
+
+# add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
+# single quotes are important in below list to keep variable names intact until 'pavics-compose' parses them
+EXTRA_VARS='
+  $STAC_POSTGRES_USER
+  $STAC_POSTGRES_PASSWORD
+  $STAC_PGUSER
+  $STAC_PGPASSWORD
+'
+# extend the original 'VARS' from 'birdhouse/pavics-compose.sh' to employ them for template substitution
+# adding them to 'VARS', they will also be validated in case of override of 'default.env' using 'env.local'
+VARS="$VARS $EXTRA_VARS"

birdhouse/components/stac/docker-compose-extra.yml

@@ -0,0 +1,63 @@
+version: "3.4"
+
+x-logging:
+  &default-logging
+  driver: "json-file"
+  options:
+    max-size: "50m"
+    max-file: "10"
+
+services:
+  stac:
+    container_name: stac
+    image: ghcr.io/crim-ca/stac-app:main
+    depends_on:
+      - stac-db
+    environment:
+      - POSTGRES_USER=${STAC_POSTGRES_USER}
+      - POSTGRES_PASS=${STAC_POSTGRES_PASSWORD}
+      - POSTGRES_DBNAME=postgis
+      - POSTGRES_HOST_READER=stac-db
+      - POSTGRES_HOST_WRITER=stac-db
+      - POSTGRES_PORT=5432
+      - ROUTER_PREFIX=/stac
+      - OPENAPI_URL=/stac/api
+      - DOCS_URL=/stac/api.html
+    logging: *default-logging
+    restart: always
+
+  stac-browser:
+    container_name: stac-browser
+    image: ghcr.io/crim-ca/stac-browser:docker_image_push
+    environment:
+      - CATALOG_URL=https://${PAVICS_FQDN_PUBLIC}/stac/
+      - ROOT_PATH=/stac-browser/
+
+  stac-db:
+    container_name: stac-db
+    image: ghcr.io/stac-utils/pgstac:v0.6.10
+    environment:
+      - POSTGRES_USER=${STAC_POSTGRES_USER}
+      - POSTGRES_PASSWORD=${STAC_POSTGRES_PASSWORD}
+      - POSTGRES_DB=postgis
+      - PGUSER=${STAC_PGUSER}
+      - PGPASSWORD=${STAC_PGPASSWORD}
+      - PGHOST=localhost
+      - PGDATABASE=postgis
+    volumes:
+      - stac-db:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # extend proxy with endpoint and config for STAC API access
+  proxy:
+    volumes:
+      - ./components/stac/conf.extra-service.d:/etc/nginx/conf.extra-service.d/stac:ro
+    links:
+      - stac
+
+volumes:
+  stac-db:

birdhouse/config/twitcher/config/proxy/conf.extra-service.d/twitcher.conf.template

@@ -4,4 +4,5 @@
         proxy_set_header X-Forwarded-Proto $real_scheme;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header X-Forwarded-Host $host:$server_port;
+        proxy_set_header Forwarded "proto=https;host=${PAVICS_FQDN}";    # Helps the STAC component to craft URLs containing the full PAVICS_FQDN
     }

birdhouse/optional-components/README.rst

@@ -292,3 +292,19 @@ See |geoserver_secured_pr|_. for more details.
 
 .. _geoserver_secured_pr: https://github.com/bird-house/birdhouse-deploy/pull/242
 .. |geoserver_secured_pr| replace:: Pull Request
+
+
+Populate STAC catalog with sample data
+--------------------------------------------------------
+
+STAC Populator contains the workflow logic to ingest sample STAC item into the STAC catalog.
+
+Once enabled in the stack, this component will run automatically on stack boot time in order to populate the catalog. 
+On stack initialization, STAC item generation workflows will run for ``STAC_ASSET_GENERATOR_TIMEOUT`` seconds in order 
+to populate the catalog with sample data. Change this timeout as needed, as there are no impact on the stack boot, 
+except time required to feed the catalog.
+
+To enable this optional-component:
+
+- Edit ``env.local`` (a copy of `env.local.example`_)
+- Add ``"./optional-components/stac-populator"`` to ``EXTRA_CONF_DIRS``.

birdhouse/optional-components/all-public-access/default.env

@@ -3,4 +3,5 @@
 # add any component that this component requires to run
 COMPONENT_DEPENDENCIES="
   ./config/magpie
+  ./optional-components/stac-public-access
 "

birdhouse/optional-components/stac-populator/default.env

@@ -0,0 +1,15 @@
+export STAC_ASSET_GENERATOR_TIMEOUT=30
+
+# add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here
+# single quotes are important in below list to keep variable names intact until 'pavics-compose' parses them
+EXTRA_VARS='
+  $STAC_ASSET_GENERATOR_TIMEOUT
+'
+# extend the original 'VARS' from 'birdhouse/pavics-compose.sh' to employ them for template substitution
+# adding them to 'VARS', they will also be validated in case of override of 'default.env' using 'env.local'
+VARS="$VARS $EXTRA_VARS"
+
+# add any component that this component requires to run
+COMPONENT_DEPENDENCIES="
+  ./components/stac
+"

birdhouse/optional-components/stac-populator/docker-compose-extra.yml

@@ -0,0 +1,21 @@
+version: "3.4"
+
+x-logging:
+  &default-logging
+  driver: "json-file"
+  options:
+    max-size: "50m"
+    max-file: "10"
+
+services:
+  # populates STAC catalog with sample collection items
+  stac-populator:
+    container_name: stac-populator
+    image: ghcr.io/crim-ca/stac-populator:master
+    environment:
+      - STAC_ASSET_GENERATOR_TIMEOUT=${STAC_ASSET_GENERATOR_TIMEOUT}
+      - STAC_HOST=http://stac:8000/stac  # STAC API internally accessed to avoid Twitcher authentication
+    command: >
+      bash -c "./wait-for-it.sh stac:8000 -t 30 && ./populate.sh"
+    depends_on:
+      - stac

birdhouse/optional-components/stac-public-access/.gitignore

@@ -0,0 +1 @@
+config/magpie/config.yml

birdhouse/optional-components/stac-public-access/config/magpie/config.yml.template

@@ -0,0 +1,19 @@
+providers:
+  # definition of STAC service for API access
+  stac:
+    url: http://stac:8000
+    title: STAC
+    public: true
+    c4i: false
+    type: api
+    sync_type: api
+
+permissions:
+  - service: stac
+    permission: read
+    group: anonymous
+    action: create
+  - service: stac
+    permission: write
+    group: stac-admin
+    action: create

birdhouse/optional-components/stac-public-access/default.env

@@ -0,0 +1,4 @@
+# add any component that this component requires to run
+COMPONENT_DEPENDENCIES="
+  ./components/stac
+"

birdhouse/optional-components/stac-public-access/docker-compose-extra.yml

@@ -0,0 +1,6 @@
+version: "3.4"
+services:
+  magpie:
+    volumes:
+    - ./optional-components/stac-public-access/config/magpie/config.yml:${MAGPIE_PERMISSIONS_CONFIG_PATH}/stac-public-access.yml:ro
+    - ./optional-components/stac-public-access/config/magpie/config.yml:${MAGPIE_PROVIDERS_CONFIG_PATH}/stac-public-access.yml:ro