diff --git a/lib/database/bootstrapper_runtime/Dockerfile b/lib/database/bootstrapper_runtime/Dockerfile index 09e0f9a..b466608 100644 --- a/lib/database/bootstrapper_runtime/Dockerfile +++ b/lib/database/bootstrapper_runtime/Dockerfile @@ -17,4 +17,4 @@ RUN rm -rf /asset/asyncio* # A command must be present avoid the following error on CDK deploy: # Error response from daemon: No command specified -CMD [ "echo", "ready to go!" ] \ No newline at end of file +CMD [ "echo", "ready to go!" ] diff --git a/lib/database/bootstrapper_runtime/handler.py b/lib/database/bootstrapper_runtime/handler.py index 317e5a0..7c937d0 100644 --- a/lib/database/bootstrapper_runtime/handler.py +++ b/lib/database/bootstrapper_runtime/handler.py @@ -4,6 +4,7 @@ """ import json +import logging import boto3 import httpx @@ -13,6 +14,8 @@ from pypgstac.db import PgstacDB from pypgstac.migrate import Migrate +logger = logging.getLogger("eoapi-bootstrap") + def send( event, @@ -36,10 +39,6 @@ def send( It isn't available for source code that's stored in Amazon S3 buckets. For code in buckets, you must write your own functions to send responses. """ - responseUrl = event["ResponseURL"] - - print(responseUrl) - responseBody = {} responseBody["Status"] = responseStatus responseBody["Reason"] = ( @@ -53,21 +52,21 @@ def send( responseBody["Data"] = responseData json_responseBody = json.dumps(responseBody) - - print("Response body:\n" + json_responseBody) - - headers = {"content-type": "", "content-length": str(len(json_responseBody))} + print("Response body:\n " + json_responseBody) try: response = httpx.put( - responseUrl, + event["ResponseURL"], data=json_responseBody, - headers=headers, + headers={"content-type": "", "content-length": str(len(json_responseBody))}, timeout=30, ) - print("Status code: " + response.status_code) + print("Status code: ", response.status_code) + logger.debug(f"OK - Status code: {response.status_code}") + except Exception as e: print("send(..) failed executing httpx.put(..): " + str(e)) + logger.debug(f"NOK - failed executing PUT requests: {e}") def get_secret(secret_name): @@ -84,9 +83,9 @@ def create_db(cursor, db_name: str) -> None: sql.SQL("SELECT 1 FROM pg_catalog.pg_database " "WHERE datname = %s"), [db_name] ) if cursor.fetchone(): - print(f"database {db_name} exists, not creating DB") + print(f" database {db_name} exists, not creating DB") else: - print(f"database {db_name} not found, creating...") + print(f" database {db_name} not found, creating...") cursor.execute( sql.SQL("CREATE DATABASE {db_name}").format(db_name=sql.Identifier(db_name)) ) @@ -114,8 +113,8 @@ def create_user(cursor, username: str, password: str) -> None: ) -def create_permissions(cursor, db_name: str, username: str) -> None: - """Add permissions.""" +def update_user_permissions(cursor, db_name: str, username: str) -> None: + """Update eoAPI user permissions.""" cursor.execute( sql.SQL( "GRANT CONNECT ON DATABASE {db_name} TO {username};" @@ -140,6 +139,33 @@ def register_extensions(cursor) -> None: cursor.execute(sql.SQL("CREATE EXTENSION IF NOT EXISTS postgis;")) +############################################################################### +# PgSTAC Customization +############################################################################### +def customization(cursor, params) -> None: + """ + CUSTOMIZED YOUR PGSTAC DATABASE + + ref: https://github.com/stac-utils/pgstac/blob/main/docs/src/pgstac.md + + """ + if str(params.get("context", "FALSE")).upper() == "TRUE": + # Add CONTEXT=ON + pgstac_settings = """ + INSERT INTO pgstac_settings (name, value) + VALUES ('context', 'on') + ON CONFLICT ON CONSTRAINT pgstac_settings_pkey DO UPDATE SET value = excluded.value;""" + cursor.execute(sql.SQL(pgstac_settings)) + + if str(params.get("mosaic_index", "TRUE")).upper() == "TRUE": + # Create index of searches with `mosaic`` type + cursor.execute( + sql.SQL( + "CREATE INDEX IF NOT EXISTS searches_mosaic ON searches ((true)) WHERE metadata->>'type'='mosaic';" + ) + ) + + def handler(event, context): """Lambda Handler.""" print(f"Handling {event}") @@ -149,88 +175,90 @@ def handler(event, context): try: params = event["ResourceProperties"] - connection_params = get_secret(params["conn_secret_arn"]) - user_params = get_secret(params["new_user_secret_arn"]) - - print("Connecting to admin DB...") - admin_db_conninfo = make_conninfo( - dbname=connection_params.get("dbname", "postgres"), - user=connection_params["username"], - password=connection_params["password"], - host=connection_params["host"], - port=connection_params["port"], + + # Admin (AWS RDS) user/password/dbname parameters + admin_params = get_secret(params["conn_secret_arn"]) + + # Custom eoAPI user/password/dbname parameters + eoapi_params = get_secret(params["new_user_secret_arn"]) + + print("Connecting to RDS...") + rds_conninfo = make_conninfo( + dbname=admin_params.get("dbname", "postgres"), + user=admin_params["username"], + password=admin_params["password"], + host=admin_params["host"], + port=admin_params["port"], ) - with psycopg.connect(admin_db_conninfo, autocommit=True) as conn: + with psycopg.connect(rds_conninfo, autocommit=True) as conn: with conn.cursor() as cur: - print("Creating database...") + print(f"Creating eoAPI '{eoapi_params['dbname']}' database...") create_db( cursor=cur, - db_name=user_params["dbname"], + db_name=eoapi_params["dbname"], ) - print("Creating user...") + print(f"Creating eoAPI '{eoapi_params['username']}' user...") create_user( cursor=cur, - username=user_params["username"], - password=user_params["password"], + username=eoapi_params["username"], + password=eoapi_params["password"], ) - # Install extensions on the user DB with - # superuser permissions, since they will - # otherwise fail to install when run as - # the non-superuser within the pgstac - # migrations. - print("Connecting to STAC DB...") - stac_db_conninfo = make_conninfo( - dbname=user_params["dbname"], - user=connection_params["username"], - password=connection_params["password"], - host=connection_params["host"], - port=connection_params["port"], + # Install postgis and pgstac on the eoapi database with + # superuser permissions + print(f"Connecting to eoAPI '{eoapi_params['dbname']}' database...") + eoapi_db_admin_conninfo = make_conninfo( + dbname=eoapi_params["dbname"], + user=admin_params["username"], + password=admin_params["password"], + host=admin_params["host"], + port=admin_params["port"], ) - with psycopg.connect(stac_db_conninfo, autocommit=True) as conn: + with psycopg.connect(eoapi_db_admin_conninfo, autocommit=True) as conn: with conn.cursor() as cur: - print("Registering PostGIS ...") + print( + f"Registering Extension in '{eoapi_params['dbname']}' database..." + ) register_extensions(cursor=cur) - stac_db_admin_dsn = ( - "postgresql://{user}:{password}@{host}:{port}/{dbname}".format( - dbname=user_params.get("dbname", "postgres"), - user=connection_params["username"], - password=connection_params["password"], - host=connection_params["host"], - port=connection_params["port"], - ) - ) - - pgdb = PgstacDB(dsn=stac_db_admin_dsn, debug=True) - print(f"Current {pgdb.version=}") + print("Starting PgSTAC Migration ") + with PgstacDB(connection=conn, debug=True) as pgdb: + print(f"Current PgSTAC Version: {pgdb.version}") - # As admin, run migrations - print("Running migrations...") - Migrate(pgdb).run_migration(params["pgstac_version"]) - - # Assign appropriate permissions to user (requires pgSTAC migrations to have run) - with psycopg.connect(admin_db_conninfo, autocommit=True) as conn: - with conn.cursor() as cur: - print("Setting permissions...") - create_permissions( - cursor=cur, - db_name=user_params["dbname"], - username=user_params["username"], - ) + print(f"Running migrations to PgSTAC {params['pgstac_version']}...") + Migrate(pgdb).run_migration(params["pgstac_version"]) - print("Adding mosaic index...") with psycopg.connect( - stac_db_admin_dsn, + eoapi_db_admin_conninfo, autocommit=True, options="-c search_path=pgstac,public -c application_name=pgstac", ) as conn: - conn.execute( - sql.SQL( - "CREATE INDEX IF NOT EXISTS searches_mosaic ON searches ((true)) WHERE metadata->>'type'='mosaic';" + print("Customize PgSTAC database...") + # Update permissions to eoAPI user to assume pgstac_* roles + with conn.cursor() as cur: + print(f"Update '{eoapi_params['username']}' permissions...") + update_user_permissions( + cursor=cur, + db_name=eoapi_params["dbname"], + username=eoapi_params["username"], ) + + customization(cursor=cur, params=params) + + # Make sure the user can access the database + eoapi_user_dsn = ( + "postgresql://{user}:{password}@{host}:{port}/{dbname}".format( + dbname=eoapi_params["dbname"], + user=eoapi_params["username"], + password=eoapi_params["password"], + host=admin_params["host"], + port=admin_params["port"], ) + ) + print("Checking eoAPI user access to the PgSTAC database...") + with PgstacDB(dsn=eoapi_user_dsn, debug=True) as pgdb: + print(f" OK - User has access to pgstac db, pgstac schema version: {pgdb.version}") except Exception as e: print(f"Unable to bootstrap database with exception={e}") diff --git a/lib/database/index.ts b/lib/database/index.ts index 2e64efe..94f7267 100644 --- a/lib/database/index.ts +++ b/lib/database/index.ts @@ -14,7 +14,12 @@ import { Construct } from "constructs"; import { CustomLambdaFunctionProps } from "../utils"; const instanceSizes: Record = require("./instance-memory.json"); -const DEFAULT_PGSTAC_VERSION = "0.7.10"; +const DEFAULT_PGSTAC_VERSION = "0.8.5"; + +let defaultPgSTACCustomOptions :{ [key: string]: any } = { + "context": "FALSE", + "mosaic_index": "TRUE" +} function hasVpc( instance: rds.DatabaseInstance | rds.IDatabaseInstance @@ -106,12 +111,7 @@ export class PgStacDatabase extends Construct { // connect to database this.db.connections.allowFrom(handler, ec2.Port.tcp(5432)); - let customResourceProperties : { [key: string]: any} = {}; - - // if customResourceProperties are provided, fill in the values. - if (props.customResourceProperties) { - Object.assign(customResourceProperties, props.customResourceProperties); - } + let customResourceProperties : { [key: string]: any} = props.customResourceProperties ? { ...defaultPgSTACCustomOptions, ...props.customResourceProperties } : defaultPgSTACCustomOptions; // update properties customResourceProperties["conn_secret_arn"] = this.db.secret!.secretArn; @@ -195,7 +195,7 @@ export interface PgStacDatabaseProps extends rds.DatabaseInstanceProps { /** * Lambda function Custom Resource properties. A custom resource property is going to be created * to trigger the boostrapping lambda function. This parameter allows the user to specify additional properties - * on top of the defaults ones. + * on top of the defaults ones. * */ readonly customResourceProperties?: { diff --git a/lib/ingestor-api/runtime/requirements.txt b/lib/ingestor-api/runtime/requirements.txt index 7aff84a..187cd77 100644 --- a/lib/ingestor-api/runtime/requirements.txt +++ b/lib/ingestor-api/runtime/requirements.txt @@ -5,7 +5,7 @@ orjson>=3.6.8 psycopg[binary,pool]>=3.0.15 pydantic_ssm_settings>=0.2.0 pydantic>=1.9.0 -pypgstac==0.7.10 +pypgstac==0.8.5 requests>=2.27.1 # Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 stac-pydantic @ git+https://github.com/alukach/stac-pydantic.git@patch-1 diff --git a/lib/tipg-api/runtime/Dockerfile b/lib/tipg-api/runtime/Dockerfile index 25f5bc2..954254b 100644 --- a/lib/tipg-api/runtime/Dockerfile +++ b/lib/tipg-api/runtime/Dockerfile @@ -4,7 +4,8 @@ FROM --platform=linux/amd64 public.ecr.aws/lambda/python:${PYTHON_VERSION} WORKDIR /tmp RUN python -m pip install pip -U -RUN python -m pip install tipg==0.3.1 "mangum>=0.14,<0.15" -t /asset --no-binary pydantic +COPY runtime/requirements.txt requirements.txt +RUN python -m pip install -r requirements.txt "mangum>=0.14,<0.15" -t /asset --no-binary pydantic # Reduce package size and remove useless files RUN cd /asset && find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[0-9]*//'); cp $f $n; done; @@ -14,4 +15,4 @@ RUN find /asset -type d -a -name 'tests' -print0 | xargs -0 rm -rf COPY runtime/src/*.py /asset/ -CMD ["echo", "hello world"] \ No newline at end of file +CMD ["echo", "hello world"] diff --git a/lib/tipg-api/runtime/requirements.txt b/lib/tipg-api/runtime/requirements.txt new file mode 100644 index 0000000..f30e289 --- /dev/null +++ b/lib/tipg-api/runtime/requirements.txt @@ -0,0 +1 @@ +tipg==0.6.3 diff --git a/lib/titiler-pgstac-api/runtime/requirements.txt b/lib/titiler-pgstac-api/runtime/requirements.txt index 20e5c18..229e4b7 100644 --- a/lib/titiler-pgstac-api/runtime/requirements.txt +++ b/lib/titiler-pgstac-api/runtime/requirements.txt @@ -1,2 +1,2 @@ -titiler.pgstac==0.5.1 -psycopg[binary, pool] \ No newline at end of file +titiler.pgstac==1.2.2 +psycopg[binary, pool]