From d1f5cfeeda284f5957098a124d1dbbc04d1f8c33 Mon Sep 17 00:00:00 2001 From: emileten Date: Thu, 12 Oct 2023 17:42:48 +0900 Subject: [PATCH 01/12] import eoapi-cdk constructs and reuse them --- infrastructure/aws/cdk/app.py | 370 ++++++++---------------- infrastructure/aws/cdk/config.py | 5 +- infrastructure/aws/requirements-cdk.txt | 8 +- 3 files changed, 133 insertions(+), 250 deletions(-) diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index b327440..45a435f 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -1,19 +1,14 @@ """ CDK Stack definition code for EOAPI """ -import json import os from typing import Any -from aws_cdk import App, CfnOutput, CustomResource, Duration, RemovalPolicy, Stack, Tags -from aws_cdk import aws_apigatewayv2_alpha as apigw +from aws_cdk import App, CfnOutput, Duration, RemovalPolicy, Stack, Tags from aws_cdk import aws_ec2 as ec2 -from aws_cdk import aws_iam as iam from aws_cdk import aws_lambda from aws_cdk import aws_logs as logs from aws_cdk import aws_rds as rds -from aws_cdk import aws_secretsmanager as secretsmanager -from aws_cdk.aws_apigatewayv2_integrations_alpha import HttpLambdaIntegration from config import ( eoAPISettings, eoDBSettings, @@ -22,103 +17,16 @@ eoVectorSettings, ) from constructs import Construct +from eoapi_cdk import ( + PgStacApiLambda, + PgStacDatabase, + TiPgApiLambda, + TitilerPgstacApiLambda, +) eoapi_settings = eoAPISettings() -class BootstrappedDb(Construct): - """ - Given an RDS database, connect to DB and create a database, user, and - password - """ - - def __init__( - self, - scope: Construct, - id: str, - db: rds.DatabaseInstance, - new_dbname: str, - new_username: str, - secrets_prefix: str, - pgstac_version: str, - enable_context: bool = False, - enable_mosaic_index: bool = False, - context_dir: str = "../../", - ) -> None: - """Update RDS database.""" - super().__init__(scope, id) - - # TODO: Utilize a singleton function. - handler = aws_lambda.Function( - self, - "DatabaseBootstrapper", - handler="handler.handler", - runtime=aws_lambda.Runtime.PYTHON_3_10, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.db", - build_args={"PYTHON_VERSION": "3.10", "PGSTAC_VERSION": pgstac_version}, - platform="linux/amd64", - ), - timeout=Duration.minutes(5), - vpc=db.vpc, - allow_public_subnet=True, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - - self.secret = secretsmanager.Secret( - self, - id, - secret_name=os.path.join( - secrets_prefix, id.replace(" ", "_"), self.node.addr - ), - generate_secret_string=secretsmanager.SecretStringGenerator( - secret_string_template=json.dumps( - { - "dbname": new_dbname, - "engine": "postgres", - "port": 5432, - "host": db.instance_endpoint.hostname, - "username": new_username, - }, - ), - generate_string_key="password", - exclude_punctuation=True, - ), - description=f"Deployed by {Stack.of(self).stack_name}", - ) - - self.resource = CustomResource( - scope=scope, - id="BootstrappedDbResource", - service_token=handler.function_arn, - properties={ - # By setting pgstac_version in the properties assures - # that Create/Update events will be passed to the service token - "pgstac_version": pgstac_version, - "context": enable_context, - "mosaic_index": enable_mosaic_index, - "conn_secret_arn": db.secret.secret_arn, - "new_user_secret_arn": self.secret.secret_arn, - }, - # We do not need to run the custom resource on STAC Delete - # Custom Resource are not physical resources so it's OK to `Retain` it - removal_policy=RemovalPolicy.RETAIN, - ) - - # Allow lambda to... - # read new user secret - self.secret.grant_read(handler) - # read database secret - db.secret.grant_read(handler) - # connect to database - db.connections.allow_from(handler, port_range=ec2.Port.tcp(5432)) - - def is_required_by(self, construct: Construct): - """Register required services.""" - return construct.node.add_dependency(self.resource) - - class eoAPIconstruct(Stack): """Earth Observation API CDK application""" @@ -181,62 +89,76 @@ def __init__( # noqa: C901 vpc.add_gateway_endpoint(key, service=service) eodb_settings = eoDBSettings() - db = rds.DatabaseInstance( + + pgstac_db = PgStacDatabase( self, - f"{id}-postgres-db", + "pgstac-db", vpc=vpc, - engine=rds.DatabaseInstanceEngine.POSTGRES, + engine=rds.DatabaseInstanceEngine.postgres( + version=rds.PostgresEngineVersion.VER_14 + ), + vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), + allocated_storage=eodb_settings.allocated_storage, instance_type=ec2.InstanceType.of( ec2.InstanceClass.BURSTABLE3, ec2.InstanceSize(eodb_settings.instance_size), ), database_name="postgres", - # should set the subnet to `PRIVATE_ISOLATED` but then we need either a bastion host to connect to the db - # or an API to ingest/delete data in the DB - vpc_subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PUBLIC), backup_retention=Duration.days(7), deletion_protection=eoapi_settings.stage.lower() == "production", removal_policy=RemovalPolicy.SNAPSHOT if eoapi_settings.stage.lower() == "production" else RemovalPolicy.DESTROY, - ) - - setup_db = BootstrappedDb( - self, - "STAC DB for eoapi", - db=db, - new_dbname=eodb_settings.dbname, - new_username=eodb_settings.user, + custom_resource_properties={ + "pgstac_version": eodb_settings.pgstac_version, + "context": eodb_settings.context, + "mosaic_index": eodb_settings.mosaic_index, + }, + bootstrapper_lambda_function_options={ + "handler": "handler.handler", + "runtime": aws_lambda.Runtime.PYTHON_3_10, + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.db", + build_args={ + "PYTHON_VERSION": "3.10", + "PGSTAC_VERSION": eodb_settings.pgstac_version, + }, + platform="linux/amd64", + ), + "timeout": Duration.minutes(5), + "allow_public_subnet": True, + "log_retention": logs.RetentionDays.ONE_WEEK, + }, + pgstac_db_name=eodb_settings.dbname, + pgstac_username=eodb_settings.user, secrets_prefix=os.path.join(stage, name), - pgstac_version=eodb_settings.pgstac_version, - enable_context=eodb_settings.context, - enable_mosaic_index=eodb_settings.mosaic_index, - context_dir=context_dir, ) CfnOutput( self, f"{id}-database-secret-arn", - value=db.secret.secret_arn, + value=pgstac_db.pgstac_secret.secret_arn, description="Arn of the SecretsManager instance holding the connection info for Postgres DB", ) # eoapi.raster if "raster" in eoapi_settings.functions: + db_secrets = { - "POSTGRES_HOST": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( + "POSTGRES_DBNAME": pgstac_db.pgstac_secret.secret_value_from_json( "dbname" ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( + "POSTGRES_USER": pgstac_db.pgstac_secret.secret_value_from_json( "username" ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( + "POSTGRES_PASS": pgstac_db.pgstac_secret.secret_value_from_json( "password" ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( + "POSTGRES_PORT": pgstac_db.pgstac_secret.secret_value_from_json( "port" ).to_string(), } @@ -245,76 +167,56 @@ def __init__( # noqa: C901 env = eoraster_settings.env or {} if "DB_MAX_CONN_SIZE" not in env: env["DB_MAX_CONN_SIZE"] = "1" + env.update(db_secrets) - eoraster_function = aws_lambda.Function( + eoraster = TitilerPgstacApiLambda( self, f"{id}-raster-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.raster", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, vpc=vpc, - vpc_subnets=ec2.SubnetSelection( + subnet_selection=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), - allow_public_subnet=True, - handler="handler.handler", - memory_size=eoraster_settings.memory, - timeout=Duration.seconds(eoraster_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eoraster_function.add_environment(key=k, value=str(v)) - - eoraster_function.add_to_role_policy( - iam.PolicyStatement( - actions=["s3:GetObject"], - resources=[ - f"arn:aws:s3:::{bucket}/{eoraster_settings.key}" - for bucket in eoraster_settings.buckets - ], - ) - ) - - db.connections.allow_from(eoraster_function, port_range=ec2.Port.tcp(5432)) - - raster_api = apigw.HttpApi( - self, - f"{id}-raster-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-raster-integration", - eoraster_function, - ), + api_env=env, + lambda_function_options={ + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.raster", + build_args={ + "PYTHON_VERSION": "3.11", + }, + platform="linux/amd64", + ), + "allow_public_subnet": True, + "handler": "handler.handler", + "runtime": aws_lambda.Runtime.PYTHON_3_11, + "memory_size": eoraster_settings.memory, + "timeout": Duration.seconds(eoraster_settings.timeout), + "log_retention": logs.RetentionDays.ONE_WEEK, + }, + buckets=eoraster_settings.buckets, ) - CfnOutput(self, "eoAPI-raster", value=raster_api.url.strip("/")) - - setup_db.is_required_by(eoraster_function) # eoapi.stac if "stac" in eoapi_settings.functions: db_secrets = { - "POSTGRES_HOST_READER": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST_READER": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_HOST_WRITER": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST_WRITER": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( + "POSTGRES_DBNAME": pgstac_db.pgstac_secret.secret_value_from_json( "dbname" ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( + "POSTGRES_USER": pgstac_db.pgstac_secret.secret_value_from_json( "username" ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( + "POSTGRES_PASS": pgstac_db.pgstac_secret.secret_value_from_json( "password" ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( + "POSTGRES_PORT": pgstac_db.pgstac_secret.secret_value_from_json( "port" ).to_string(), } @@ -325,65 +227,54 @@ def __init__( # noqa: C901 env["DB_MAX_CONN_SIZE"] = "1" if "DB_MIN_CONN_SIZE" not in env: env["DB_MIN_CONN_SIZE"] = "1" - - eostac_function = aws_lambda.Function( - self, - f"{id}-stac-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.stac", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), - vpc=vpc, - handler="handler.handler", - memory_size=eostac_settings.memory, - timeout=Duration.seconds(eostac_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eostac_function.add_environment(key=k, value=str(v)) - + env.update(db_secrets) # If raster is deployed we had the TITILER_ENDPOINT env to add the Proxy extension if "raster" in eoapi_settings.functions: - eostac_function.add_environment( - key="TITILER_ENDPOINT", value=raster_api.url.strip("/") - ) + env["TITILER_ENDPOINT"] = eoraster.url.strip("/") - db.connections.allow_from(eostac_function, port_range=ec2.Port.tcp(5432)) - - stac_api = apigw.HttpApi( + PgStacApiLambda( self, - f"{id}-stac-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-stac-integration", - eostac_function, + id=f"{id}-stac-lambda", + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, + vpc=vpc, + subnet_selection=ec2.SubnetSelection( + subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), + api_env=env, + lambda_function_options={ + "runtime": aws_lambda.Runtime.PYTHON_3_11, + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.stac", + build_args={ + "PYTHON_VERSION": "3.11", + }, + platform="linux/amd64", + ), + "handler": "handler.handler", + "memory_size": eostac_settings.memory, + "timeout": Duration.seconds(eostac_settings.timeout), + "log_retention": logs.RetentionDays.ONE_WEEK, + }, ) - CfnOutput(self, "eoAPI-stac", value=stac_api.url.strip("/")) - - setup_db.is_required_by(eostac_function) # eoapi.vector if "vector" in eoapi_settings.functions: db_secrets = { - "POSTGRES_HOST": setup_db.secret.secret_value_from_json( + "POSTGRES_HOST": pgstac_db.pgstac_secret.secret_value_from_json( "host" ).to_string(), - "POSTGRES_DBNAME": setup_db.secret.secret_value_from_json( + "POSTGRES_DBNAME": pgstac_db.pgstac_secret.secret_value_from_json( "dbname" ).to_string(), - "POSTGRES_USER": setup_db.secret.secret_value_from_json( + "POSTGRES_USER": pgstac_db.pgstac_secret.secret_value_from_json( "username" ).to_string(), - "POSTGRES_PASS": setup_db.secret.secret_value_from_json( + "POSTGRES_PASS": pgstac_db.pgstac_secret.secret_value_from_json( "password" ).to_string(), - "POSTGRES_PORT": setup_db.secret.secret_value_from_json( + "POSTGRES_PORT": pgstac_db.pgstac_secret.secret_value_from_json( "port" ).to_string(), } @@ -396,41 +287,34 @@ def __init__( # noqa: C901 if "DB_MIN_CONN_SIZE" not in env: env["DB_MIN_CONN_SIZE"] = "1" - eovector_function = aws_lambda.Function( + env.update(db_secrets) + + TiPgApiLambda( self, f"{id}-vector-lambda", - runtime=aws_lambda.Runtime.PYTHON_3_11, - code=aws_lambda.Code.from_docker_build( - path=os.path.abspath(context_dir), - file="infrastructure/aws/dockerfiles/Dockerfile.vector", - build_args={ - "PYTHON_VERSION": "3.11", - }, - platform="linux/amd64", - ), vpc=vpc, - handler="handler.handler", - memory_size=eovector_settings.memory, - timeout=Duration.seconds(eovector_settings.timeout), - environment=env, - log_retention=logs.RetentionDays.ONE_WEEK, - ) - for k, v in db_secrets.items(): - eovector_function.add_environment(key=k, value=str(v)) - - db.connections.allow_from(eovector_function, port_range=ec2.Port.tcp(5432)) - - vector_api = apigw.HttpApi( - self, - f"{id}-vector-endpoint", - default_integration=HttpLambdaIntegration( - f"{id}-vector-integration", - eovector_function, + db=pgstac_db.db, + db_secret=pgstac_db.pgstac_secret, + subnet_selection=ec2.SubnetSelection( + subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), + api_env=env, + lambda_function_options={ + "runtime": aws_lambda.Runtime.PYTHON_3_11, + "code": aws_lambda.Code.from_docker_build( + path=os.path.abspath(context_dir), + file="infrastructure/aws/dockerfiles/Dockerfile.vector", + build_args={ + "PYTHON_VERSION": "3.11", + }, + platform="linux/amd64", + ), + "handler": "handler.handler", + "memory_size": eovector_settings.memory, + "timeout": Duration.seconds(eovector_settings.timeout), + "log_retention": logs.RetentionDays.ONE_WEEK, + }, ) - CfnOutput(self, "eoAPI-vector", value=vector_api.url.strip("/")) - - setup_db.is_required_by(eovector_function) app = App() diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index 0450bf9..80d8bba 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -42,7 +42,7 @@ class eoDBSettings(BaseSettings): instance_size: str = "SMALL" context: bool = True mosaic_index: bool = True - + allocated_storage: int = 20 model_config = { "env_prefix": "CDK_EOAPI_DB_", "env_file": ".env", @@ -90,9 +90,6 @@ class eoRasterSettings(BaseSettings): # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-arn-format.html buckets: List = ["*"] - # S3 key pattern to limit the access to specific items (e.g: "my_data/*.tif") - key: str = "*" - timeout: int = 10 memory: int = 3008 diff --git a/infrastructure/aws/requirements-cdk.txt b/infrastructure/aws/requirements-cdk.txt index c134c87..404e9f1 100644 --- a/infrastructure/aws/requirements-cdk.txt +++ b/infrastructure/aws/requirements-cdk.txt @@ -1,9 +1,11 @@ # aws cdk -aws-cdk-lib==2.94.0 -aws_cdk-aws_apigatewayv2_alpha==2.94.0a0 -aws_cdk-aws_apigatewayv2_integrations_alpha==2.94.0a0 +aws-cdk-lib==2.99.1 +aws_cdk-aws_apigatewayv2_alpha==2.99.1a0 +aws_cdk-aws_apigatewayv2_integrations_alpha==2.99.1a0 constructs>=10.0.0 # pydantic settings pydantic~=2.0 pydantic-settings~=2.0 + +../eoapi-cdk/dist/python/eoapi_cdk-5.4.0-py3-none-any.whl \ No newline at end of file From 8e40f0c49e9835f720502e07dcf3773376800f26 Mon Sep 17 00:00:00 2001 From: emileten Date: Tue, 31 Oct 2023 16:45:23 +0900 Subject: [PATCH 02/12] add ingestor, browser, bump eoapi-cdk --- infrastructure/aws/cdk/app.py | 139 ++++++++++++++++++------ infrastructure/aws/cdk/config.py | 4 +- infrastructure/aws/requirements-cdk.txt | 5 +- 3 files changed, 113 insertions(+), 35 deletions(-) diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index 45a435f..b253d84 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -4,11 +4,14 @@ import os from typing import Any +import boto3 from aws_cdk import App, CfnOutput, Duration, RemovalPolicy, Stack, Tags from aws_cdk import aws_ec2 as ec2 +from aws_cdk import aws_iam as iam from aws_cdk import aws_lambda from aws_cdk import aws_logs as logs from aws_cdk import aws_rds as rds +from aws_cdk import aws_s3 as s3 from config import ( eoAPISettings, eoDBSettings, @@ -20,6 +23,8 @@ from eoapi_cdk import ( PgStacApiLambda, PgStacDatabase, + StacBrowser, + StacIngestor, TiPgApiLambda, TitilerPgstacApiLambda, ) @@ -42,8 +47,6 @@ def __init__( # noqa: C901 """Define stack.""" super().__init__(scope, id, **kwargs) - # vpc = ec2.Vpc(self, f"{id}-vpc", nat_gateways=0) - vpc = ec2.Vpc( self, f"{id}-vpc", @@ -52,23 +55,8 @@ def __init__( # noqa: C901 name="ingress", cidr_mask=24, subnet_type=ec2.SubnetType.PUBLIC, - ), - ec2.SubnetConfiguration( - name="application", - cidr_mask=24, - subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS, - ), - ec2.SubnetConfiguration( - name="rds", - cidr_mask=28, - subnet_type=ec2.SubnetType.PRIVATE_ISOLATED, - ), + ) ], - nat_gateways=1, - ) - print( - """The eoAPI stack use AWS NatGateway for the Raster service so it can reach the internet. -This might incurs some cost (https://docs.aws.amazon.com/vpc/latest/userguide/vpc-nat-gateway.html).""" ) interface_endpoints = [ @@ -135,6 +123,9 @@ def __init__( # noqa: C901 secrets_prefix=os.path.join(stage, name), ) + # allow connections from anywhere to the DB + pgstac_db.db.connections.allow_default_port_from_any_ipv4() + CfnOutput( self, f"{id}-database-secret-arn", @@ -174,10 +165,6 @@ def __init__( # noqa: C901 f"{id}-raster-lambda", db=pgstac_db.db, db_secret=pgstac_db.pgstac_secret, - vpc=vpc, - subnet_selection=ec2.SubnetSelection( - subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS - ), api_env=env, lambda_function_options={ "code": aws_lambda.Code.from_docker_build( @@ -232,15 +219,11 @@ def __init__( # noqa: C901 if "raster" in eoapi_settings.functions: env["TITILER_ENDPOINT"] = eoraster.url.strip("/") - PgStacApiLambda( + eostac = PgStacApiLambda( self, id=f"{id}-stac-lambda", db=pgstac_db.db, db_secret=pgstac_db.pgstac_secret, - vpc=vpc, - subnet_selection=ec2.SubnetSelection( - subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS - ), api_env=env, lambda_function_options={ "runtime": aws_lambda.Runtime.PYTHON_3_11, @@ -259,6 +242,38 @@ def __init__( # noqa: C901 }, ) + if eostac_settings.stac_browser_github_tag is not None: + assert ( + eostac_settings.stac_api_custom_domain_name is not None + ), "stac_api_custom_domain_name must be set if stac_browser_github_tag is not None. The browser deployment needs a resolved STAC API url at deployment time and so needs to rely on a predefined custom domain name." + stac_browser_bucket = s3.Bucket( + self, + "stac-browser-bucket", + bucket_name=f"{id.lower()}-stac-browser", + removal_policy=RemovalPolicy.DESTROY, + auto_delete_objects=True, + website_index_document="index.html", + public_read_access=True, + block_public_access=s3.BlockPublicAccess( + block_public_acls=False, + block_public_policy=False, + ignore_public_acls=False, + restrict_public_buckets=False, + ), + object_ownership=s3.ObjectOwnership.OBJECT_WRITER, + ) + + # need to build this manually, the attribute eostac.url is not resolved yet. + + StacBrowser( + self, + "stac-browser", + github_repo_tag=eostac_settings.stac_browser_github_tag, + stac_catalog_url=eostac_settings.stac_api_custom_domain_name, + website_index_document="index.html", + bucket_arn=stac_browser_bucket.bucket_arn, + ) + # eoapi.vector if "vector" in eoapi_settings.functions: db_secrets = { @@ -292,12 +307,8 @@ def __init__( # noqa: C901 TiPgApiLambda( self, f"{id}-vector-lambda", - vpc=vpc, db=pgstac_db.db, db_secret=pgstac_db.pgstac_secret, - subnet_selection=ec2.SubnetSelection( - subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS - ), api_env=env, lambda_function_options={ "runtime": aws_lambda.Runtime.PYTHON_3_11, @@ -316,6 +327,72 @@ def __init__( # noqa: C901 }, ) + if "ingestor" in eoapi_settings.functions: + + data_access_role = self._create_data_access_role() + + stac_ingestor = StacIngestor( + self, + "stac-ingestor", + stac_url=eostac.url, + stage=eoapi_settings.stage, + data_access_role=data_access_role, + stac_db_secret=pgstac_db.pgstac_secret, + stac_db_security_group=pgstac_db.db.connections.security_groups[0], + api_env={"REQUESTER_PAYS": "True"}, + ) + + data_access_role = self._grant_assume_role_with_principal_pattern( + data_access_role, stac_ingestor.handler_role.role_name + ) + + def _create_data_access_role(self) -> iam.Role: + + """ + Creates an IAM role with full S3 read access. + """ + + data_access_role = iam.Role( + self, + "data-access-role", + assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), + ) + + data_access_role.add_managed_policy( + iam.ManagedPolicy.from_aws_managed_policy_name("AmazonS3FullAccess") + ) + + return data_access_role + + def _grant_assume_role_with_principal_pattern( + self, + role_to_assume: iam.Role, + principal_pattern: str, + account_id: str = boto3.client("sts").get_caller_identity().get("Account"), + ) -> iam.Role: + """ + Grants assume role permissions to the role of the given + account with the given name pattern. Default account + is the current account. + """ + + role_to_assume.assume_role_policy.add_statements( + iam.PolicyStatement( + effect=iam.Effect.ALLOW, + principals=[iam.AnyPrincipal()], + actions=["sts:AssumeRole"], + conditions={ + "StringLike": { + "aws:PrincipalArn": [ + f"arn:aws:iam::{account_id}:role/{principal_pattern}" + ] + } + }, + ) + ) + + return role_to_assume + app = App() diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index 80d8bba..c177bd8 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -12,6 +12,7 @@ class functionName(str, Enum): stac = "stac" raster = "raster" vector = "vector" + ingestor = "ingestor" class eoAPISettings(BaseSettings): @@ -56,7 +57,8 @@ class eoSTACSettings(BaseSettings): timeout: int = 10 memory: int = 256 - + stac_browser_github_tag: None | str = "v3.1.0" # if not none, will try to deploy this version of radiant earth stac browser + stac_api_custom_domain_name: None | str = "https://stac.eoapi.dev" model_config = { "env_prefix": "CDK_EOAPI_STAC_", "env_file": ".env", diff --git a/infrastructure/aws/requirements-cdk.txt b/infrastructure/aws/requirements-cdk.txt index 404e9f1..1a49cd7 100644 --- a/infrastructure/aws/requirements-cdk.txt +++ b/infrastructure/aws/requirements-cdk.txt @@ -3,9 +3,8 @@ aws-cdk-lib==2.99.1 aws_cdk-aws_apigatewayv2_alpha==2.99.1a0 aws_cdk-aws_apigatewayv2_integrations_alpha==2.99.1a0 constructs>=10.0.0 - +boto3==1.28.71 # pydantic settings pydantic~=2.0 pydantic-settings~=2.0 - -../eoapi-cdk/dist/python/eoapi_cdk-5.4.0-py3-none-any.whl \ No newline at end of file +eoapi-cdk==6.0.0 \ No newline at end of file From 0f138f5fc7e4285b77641b1e854e909ee6e001b9 Mon Sep 17 00:00:00 2001 From: emileten Date: Wed, 1 Nov 2023 12:55:34 +0900 Subject: [PATCH 03/12] remove default API url --- infrastructure/aws/cdk/app.py | 6 +++--- infrastructure/aws/cdk/config.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/infrastructure/aws/cdk/app.py b/infrastructure/aws/cdk/app.py index b253d84..7a1bfc9 100644 --- a/infrastructure/aws/cdk/app.py +++ b/infrastructure/aws/cdk/app.py @@ -242,10 +242,10 @@ def __init__( # noqa: C901 }, ) - if eostac_settings.stac_browser_github_tag is not None: + if eostac_settings.stac_api_custom_domain_name is not None: assert ( - eostac_settings.stac_api_custom_domain_name is not None - ), "stac_api_custom_domain_name must be set if stac_browser_github_tag is not None. The browser deployment needs a resolved STAC API url at deployment time and so needs to rely on a predefined custom domain name." + eostac_settings.stac_browser_github_tag is not None + ), "stac_browser_github_tag must be set if stac_api_custom_domain_name is not None." stac_browser_bucket = s3.Bucket( self, "stac-browser-bucket", diff --git a/infrastructure/aws/cdk/config.py b/infrastructure/aws/cdk/config.py index c177bd8..28cbba4 100644 --- a/infrastructure/aws/cdk/config.py +++ b/infrastructure/aws/cdk/config.py @@ -57,8 +57,10 @@ class eoSTACSettings(BaseSettings): timeout: int = 10 memory: int = 256 - stac_browser_github_tag: None | str = "v3.1.0" # if not none, will try to deploy this version of radiant earth stac browser - stac_api_custom_domain_name: None | str = "https://stac.eoapi.dev" + stac_browser_github_tag: None | str = "v3.1.0" + stac_api_custom_domain_name: None | str = ( + None # if not none, will try to deploy a browser with the above tag + ) model_config = { "env_prefix": "CDK_EOAPI_STAC_", "env_file": ".env", From 5eaf5057d655b94163f649968e2cbdec0ecea3a9 Mon Sep 17 00:00:00 2001 From: emileten Date: Sat, 11 Nov 2023 19:07:16 +0900 Subject: [PATCH 04/12] add stac browser to docker deployment, change tiler in aws deployment to our tiler --- docker-compose.custom.yml | 9 +++++ docker-compose.yml | 10 +++++ dockerfiles/Dockerfile.browser | 34 ++++++++++++++++ dockerfiles/browser_config.js | 39 +++++++++++++++++++ infrastructure/aws/cdk/app.py | 1 + infrastructure/aws/cdk/stac_browser_config.js | 39 +++++++++++++++++++ infrastructure/aws/requirements-cdk.txt | 2 +- 7 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 dockerfiles/Dockerfile.browser create mode 100644 dockerfiles/browser_config.js create mode 100644 infrastructure/aws/cdk/stac_browser_config.js diff --git a/docker-compose.custom.yml b/docker-compose.custom.yml index 8e257f1..e1a0b4c 100644 --- a/docker-compose.custom.yml +++ b/docker-compose.custom.yml @@ -259,6 +259,15 @@ services: volumes: - ./.pgdata:/var/lib/postgresql/data + stac-browser: + build: + context: dockerfiles + dockerfile: Dockerfile.browser + ports: + - "${MY_DOCKER_IP:-127.0.0.1}:8084:8080" + depends_on: + - stac + networks: default: name: eoapi-network diff --git a/docker-compose.yml b/docker-compose.yml index a29863d..f7764dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -132,6 +132,16 @@ services: volumes: - ./.pgdata:/var/lib/postgresql/data + # change to official image when available https://github.com/radiantearth/stac-browser/pull/386 + stac-browser: + build: + context: dockerfiles + dockerfile: Dockerfile.browser + ports: + - "${MY_DOCKER_IP:-127.0.0.1}:8084:8080" + depends_on: + - stac-fastapi + networks: default: name: eoapi-network diff --git a/dockerfiles/Dockerfile.browser b/dockerfiles/Dockerfile.browser new file mode 100644 index 0000000..f564524 --- /dev/null +++ b/dockerfiles/Dockerfile.browser @@ -0,0 +1,34 @@ +# Copyright Radiant Earth Foundation + +FROM node:lts-alpine3.18 AS build-step +ARG DYNAMIC_CONFIG=true + +WORKDIR /app + +RUN apk add --no-cache git +RUN git clone https://github.com/radiantearth/stac-browser.git . +# remove the default config.js +RUN rm config.js +RUN npm install +# replace the default config.js with our config file +COPY ./browser_config.js ./config.js +RUN \[ "${DYNAMIC_CONFIG}" == "true" \] && sed -i 's//