feat(release): support transactions with validation and authentication (#378), add auth requirements install for ingest-api tests (#415), update .example.env and README (#412)

.example.env

@@ -4,12 +4,13 @@ CDK_DEFAULT_REGION=[REQUIRED IF DEPLOYING TO EXISTING VPC]
 
 STAGE=[FILL ME IN]
 
-VEDA_PROJECT_NAME=
-VEDA_PROJECT_DESCRIPTION=
-
-VEDA_DB_PGSTAC_VERSION=0.6.6
+VEDA_DB_PGSTAC_VERSION=0.7.10
 VEDA_DB_SCHEMA_VERSION=0.1.0
 VEDA_DB_SNAPSHOT_ID=[OPTIONAL BUT **REQUIRED** FOR ALL DEPLOYMENTS AFTER BASING DEPLOYMENT ON SNAPSHOT]
+VEDA_DB_PUBLICLY_ACCESSIBLE=TRUE
+VEDA_DB_USE_RDS_PROXY=[OPTIONAL]
+VEDA_DB_RDS_INSTANCE_CLASS=[OPTIONAL]
+VEDA_DB_RDS_INSTANCE_SIZE=[OPTIONAL]
 
 VEDA_DOMAIN_HOSTED_ZONE_ID=[OPTIONAL]
 VEDA_DOMAIN_HOSTED_ZONE_NAME=[OPTIONAL]
@@ -21,11 +22,16 @@ VEDA_DOMAIN_ALT_HOSTED_ZONE_NAME=[OPTIONAL SECOND DOMAIN]
 VEDA_RASTER_ENABLE_MOSAIC_SEARCH=TRUE
 VEDA_RASTER_DATA_ACCESS_ROLE_ARN=[OPTIONAL ARN OF IAM ROLE TO BE ASSUMED BY RASTER API]
 VEDA_RASTER_EXPORT_ASSUME_ROLE_CREDS_AS_ENVS=False
-
-VEDA_DB_PUBLICLY_ACCESSIBLE=TRUE
-
 VEDA_RASTER_ROOT_PATH=
+
 VEDA_STAC_ROOT_PATH=
+VEDA_STAC_ENABLE_TRANSACTIONS=FALSE
+
+VEDA_USERPOOL_ID=
+VEDA_CLIENT_ID=
+VEDA_CLIENT_SECRET=secret
+VEDA_DATA_ACCESS_ROLE_ARN=
+VEDA_COGNITO_DOMAIN=
 
 STAC_BROWSER_BUCKET=
 STAC_URL=

.github/workflows/develop.yml

@@ -65,6 +65,9 @@ jobs:
       - name: Install reqs for ingest api
         run: python -m pip install -r ingest_api/runtime/requirements_dev.txt
 
+      - name: Install veda auth for ingest api
+        run: python -m pip install common/auth
+
       - name: Ingest unit tests
         run: NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest ingest_api/runtime/tests/ -vv -s
 

.github/workflows/main.yml

@@ -65,6 +65,9 @@ jobs:
       - name: Install reqs for ingest api
         run: python -m pip install -r ingest_api/runtime/requirements_dev.txt
 
+      - name: Install veda auth for ingest api
+        run: python -m pip install common/auth
+
       - name: Ingest unit tests
         run: NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest ingest_api/runtime/tests/ -vv -s
 

.github/workflows/pr.yml

@@ -77,9 +77,15 @@ jobs:
       - name: Install reqs for ingest api
         run: python -m pip install -r ingest_api/runtime/requirements_dev.txt
 
+      - name: Install veda auth for ingest api
+        run: python -m pip install common/auth
+
       - name: Ingest unit tests
         run: NO_PYDANTIC_SSM_SETTINGS=1 python -m pytest ingest_api/runtime/tests/ -vv -s
 
+      # - name: Stac-api transactions unit tests
+      #   run: python -m pytest stac_api/runtime/tests/ -vv -s
+
       - name: Stop services
         run: docker compose stop
 

README.md

@@ -28,7 +28,7 @@ This project uses an AWS CDK [CloudFormation](https://docs.aws.amazon.com/AWSClo
 
 ### Enviroment variables
 
-An [.example.env](.example.env) template is supplied for for local deployments. If updating an existing deployment, it is essential to check the most current values for these variables by fetching these values from AWS Secrets Manager. The environment secrets are named `<app-name>-<stage>-env`, for example `veda-backend-dev-env`.
+An [.example.env](.example.env) template is supplied for local deployments. If updating an existing deployment, it is essential to check the most current values for these variables by fetching these values from AWS Secrets Manager. The environment secrets are named `<app-name>-<stage>-env`, for example `veda-backend-dev-env`.
 > **Warning** The environment variables stored as AWS secrets are manually maintained and should be reviewed before deploying updates to existing stacks.
 
 ### Fetch environment variables using AWS CLI
@@ -92,6 +92,8 @@ python3 -m pip install -e ".[dev,deploy,test]"
 #### Run the deployment
 
 ```
+# Login to ECR so that you can pull public docker images
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
 # Review what infrastructure changes your deployment will cause
 cdk diff
 # Execute deployment and standby--security changes will require approval for deployment

common/__init__.py

@@ -0,0 +1 @@
+"""common utils shared by veda stacks"""

common/auth/setup.py

@@ -0,0 +1,17 @@
+"""Setup veda_auth
+"""
+
+from setuptools import find_packages, setup
+
+inst_reqs = ["cryptography>=42.0.5", "pyjwt>=2.8.0", "fastapi", "pydantic<2"]
+
+setup(
+    name="veda_auth",
+    version="0.0.1",
+    description="",
+    python_requires=">=3.7",
+    packages=find_packages(),
+    zip_safe=False,
+    install_requires=inst_reqs,
+    include_package_data=True,
+)

common/auth/veda_auth/__init__.py

@@ -0,0 +1,5 @@
+"""
+    VEDA cognito auth
+"""
+
+from veda_auth.main import VedaAuth  # noqa: F401

common/auth/veda_auth/main.py

@@ -0,0 +1,128 @@
+"""Authentication handler for veda.stac and veda.ingest"""
+
+import base64
+import hashlib
+import hmac
+import logging
+from typing import Annotated, Any, Dict
+
+import boto3
+import jwt
+
+from fastapi import Depends, HTTPException, Security, security, status
+
+logger = logging.getLogger(__name__)
+
+
+class VedaAuth:
+    """Class for handling authentication"""
+
+    def __init__(self, settings) -> None:
+        """
+        Args:
+            settings: pydantic settings object containing cognito details
+        Returns:
+            None
+
+        """
+        self.oauth2_scheme = security.OAuth2AuthorizationCodeBearer(
+            authorizationUrl=settings.cognito_authorization_url,
+            tokenUrl=settings.cognito_token_url,
+            refreshUrl=settings.cognito_token_url,
+        )
+
+        self.jwks_client = jwt.PyJWKClient(settings.jwks_url)  # Caches JWKS
+
+        def validated_token(
+            token_str: Annotated[str, Security(self.oauth2_scheme)],
+            required_scopes: security.SecurityScopes,
+        ) -> Dict:
+            # Parse & validate token
+            logger.info(f"\nToken String {token_str}")
+            try:
+                token = jwt.decode(
+                    token_str,
+                    self.jwks_client.get_signing_key_from_jwt(token_str).key,
+                    algorithms=["RS256"],
+                )
+            except jwt.exceptions.InvalidTokenError as e:
+                raise HTTPException(
+                    status_code=status.HTTP_401_UNAUTHORIZED,
+                    detail="Could not validate credentials",
+                    headers={"WWW-Authenticate": "Bearer"},
+                ) from e
+
+            # Validate scopes (if required)
+            for scope in required_scopes.scopes:
+                if scope not in token["scope"]:
+                    raise HTTPException(
+                        status_code=status.HTTP_401_UNAUTHORIZED,
+                        detail="Not enough permissions",
+                        headers={
+                            "WWW-Authenticate": f'Bearer scope="{required_scopes.scope_str}"'
+                        },
+                    )
+
+            return token
+
+        self.validated_token = validated_token
+
+        def get_username(
+            token: Annotated[Dict[Any, Any], Depends(self.validated_token)]
+        ) -> str:
+            result = token["username"] if "username" in token else str(token.get("sub"))
+            return result
+
+        self.get_username = get_username
+
+    def _get_secret_hash(
+        self, username: str, client_id: str, client_secret: str
+    ) -> str:
+        # A keyed-hash message authentication code (HMAC) calculated using
+        # the secret key of a user pool client and username plus the client
+        # ID in the message.
+        message = username + client_id
+        dig = hmac.new(
+            bytearray(client_secret, "utf-8"),
+            msg=message.encode("UTF-8"),
+            digestmod=hashlib.sha256,
+        ).digest()
+        return base64.b64encode(dig).decode()
+
+    def authenticate_and_get_token(
+        self,
+        username: str,
+        password: str,
+        user_pool_id: str,
+        app_client_id: str,
+        app_client_secret: str,
+    ) -> Dict:
+        """Authenticates the credentials and returns token"""
+        client = boto3.client("cognito-idp")
+        if app_client_secret:
+            auth_params = {
+                "USERNAME": username,
+                "PASSWORD": password,
+                "SECRET_HASH": self._get_secret_hash(
+                    username, app_client_id, app_client_secret
+                ),
+            }
+        else:
+            auth_params = {
+                "USERNAME": username,
+                "PASSWORD": password,
+            }
+        try:
+            resp = client.admin_initiate_auth(
+                UserPoolId=user_pool_id,
+                ClientId=app_client_id,
+                AuthFlow="ADMIN_USER_PASSWORD_AUTH",
+                AuthParameters=auth_params,
+            )
+        except client.exceptions.NotAuthorizedException:
+            return {
+                "message": "Login failed, please make sure the credentials are correct."
+            }
+        except Exception as e:
+            return {"message": f"Login failed with exception {e}"}
+        return resp["AuthenticationResult"]

docker-compose.yml

@@ -128,6 +128,7 @@ services:
       - PGPASSWORD=password
       - PGDATABASE=postgis
       - DYNAMODB_ENDPOINT=http://localhost:8085
+      - VEDA_DB_PGSTAC_VERSION=0.7.10
     ports:
       - "8083:8083"
     command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && python /asset/local.py"

ingest_api/infrastructure/config.py

@@ -67,6 +67,10 @@ class IngestorConfig(BaseSettings):
 
     ingest_root_path: str = Field("", description="Root path for ingest API")
     custom_host: Optional[str] = Field(description="Custom host name")
+    db_pgstac_version: str = Field(
+        ...,
+        description="Version of PgStac database, i.e. 0.5",
+    )
 
     class Config:
         case_sensitive = False

ingest_api/infrastructure/construct.py

@@ -66,6 +66,7 @@ def __init__(
             "db_secret": db_secret,
             "db_vpc": db_vpc,
             "db_security_group": db_security_group,
+            "pgstac_version": config.db_pgstac_version,
         }
 
         if config.raster_data_access_role_arn:
@@ -98,21 +99,15 @@ def __init__(
             custom_host=config.custom_host,
         )
 
-        # CfnOutput(self, "ingest-api", value=self.api.url)
         stack_name = Stack.of(self).stack_name
         CfnOutput(
             self,
             "stac-ingestor-api-url",
             export_name=f"{stack_name}-stac-ingestor-api-url",
             value=self.api.url,
+            key="ingestapiurl",
         )
 
-        register_ssm_parameter(
-            self,
-            name="jwks_url",
-            value=self.jwks_url,
-            description="JWKS URL for Cognito user pool",
-        )
         register_ssm_parameter(
             self,
             name="dynamodb_table",
@@ -130,6 +125,7 @@ def build_api_lambda(
         db_vpc: ec2.IVpc,
         db_security_group: ec2.ISecurityGroup,
         data_access_role: Union[iam.IRole, None] = None,
+        pgstac_version: str,
         code_dir: str = "./",
     ) -> apigateway.LambdaRestApi:
         stack_name = Stack.of(self).stack_name
@@ -156,6 +152,7 @@ def build_api_lambda(
                 path=os.path.abspath(code_dir),
                 file="ingest_api/runtime/Dockerfile",
                 platform="linux/amd64",
+                build_args={"PGSTAC_VERSION": pgstac_version},
             ),
             runtime=aws_lambda.Runtime.PYTHON_3_9,
             timeout=Duration.seconds(30),
@@ -297,6 +294,7 @@ def __init__(
             db_vpc=db_vpc,
             db_security_group=db_security_group,
             db_vpc_subnets=db_vpc_subnets,
+            pgstac_version=config.db_pgstac_version,
         )
 
     def build_ingestor(
@@ -308,6 +306,7 @@ def build_ingestor(
         db_vpc: ec2.IVpc,
         db_security_group: ec2.ISecurityGroup,
         db_vpc_subnets: ec2.SubnetSelection,
+        pgstac_version: str,
         code_dir: str = "./",
     ) -> aws_lambda.Function:
         handler = aws_lambda.Function(
@@ -317,6 +316,7 @@ def build_ingestor(
                 path=os.path.abspath(code_dir),
                 file="ingest_api/runtime/Dockerfile",
                 platform="linux/amd64",
+                build_args={"PGSTAC_VERSION": pgstac_version},
             ),
             handler="ingestor.handler",
             runtime=aws_lambda.Runtime.PYTHON_3_9,

ingest_api/runtime/Dockerfile

@@ -1,9 +1,16 @@
 FROM public.ecr.aws/sam/build-python3.9:latest
 
+ARG PGSTAC_VERSION
+RUN echo "Using PGSTAC Version ${PGSTAC_VERSION}"
+
 WORKDIR /tmp
 
+COPY common/auth /tmp/common/auth
+RUN pip install /tmp/common/auth -t /asset
+RUN rm -rf /tmp/common
+
 COPY ingest_api/runtime/requirements.txt /tmp/ingestor/requirements.txt
-RUN pip install -r /tmp/ingestor/requirements.txt -t /asset --no-binary pydantic uvicorn
+RUN pip install -r /tmp/ingestor/requirements.txt pypgstac==${PGSTAC_VERSION} -t /asset --no-binary pydantic uvicorn
 RUN rm -rf /tmp/ingestor
 # TODO this is temporary until we use a real packaging system like setup.py or poetry
 COPY ingest_api/runtime/src /asset/src

ingest_api/runtime/requirements.txt

@@ -1,15 +1,13 @@
 # Waiting for https://github.com/stac-utils/stac-pydantic/pull/116 and 117
 cryptography>=42.0.5
 ddbcereal==2.1.1
-fastapi<=0.108.0
+fastapi>=0.109.1
 fsspec==2023.3.0
 mangum>=0.15.0
 orjson>=3.6.8
 psycopg[binary,pool]>=3.0.15
 pydantic_ssm_settings>=0.2.0
 pydantic>=1.10.12
-pyjwt>=2.8.0
-pypgstac==0.7.4
 python-multipart==0.0.7
 requests>=2.27.1
 s3fs==2023.3.0