feat: add integration tests

.github/workflows/build.yaml

@@ -12,17 +12,30 @@ on:
         required: false
       DS_RELEASE_BOT_PRIVATE_KEY:
         required: false
-
+      AWS_DEFAULT_REGION_DEPLOY:
+        required: false
+      AWS_ACCESS_KEY_ID_DEPLOY:
+        required: false
+      AWS_SECRET_ACCESS_KEY_DEPLOY:
+        required: false
+      AWS_ACCOUNT_ID:
+        required: false
 jobs:
   build_and_package:
     name: Build and package
     runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION_DEPLOY }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_DEPLOY }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEPLOY }}
+      AWS_DEFAULT_ACCOUNT: ${{ secrets.AWS_ACCOUNT_ID }}
     steps:
       - uses: actions/checkout@v3
 
       - uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
           cache: "npm"
 
       - name: Install Dependencies
@@ -65,8 +78,70 @@ jobs:
           app_id: ${{ secrets.DS_RELEASE_BOT_ID }}
           private_key: ${{ secrets.DS_RELEASE_BOT_PRIVATE_KEY }}
 
-      - name: Maybe Release 🚀
+      - name: Check release
+        id: check_release
         if: ${{ inputs.release }}
+        run: |
+          SHOULD_RELEASE=false
+          npm run semantic-release --dry-run > check_release_output.txt
+          if grep -q "Published release" check_release_output.txt; then
+            echo "SHOULD_RELEASE=true" >> $GITHUB_OUTPUT
+          else
+            echo "SHOULD_RELEASE=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Install deployment environment
+        if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}"
+        id: install_deploy_env
+        run: |
+          # install deployment environment with eoapi-cdk from build
+          python -m venv .deployment_venv
+          source .deployment_venv/bin/activate
+          pip install dist/python/*.gz
+          cd integration_tests/cdk
+          pip install -r requirements.txt
+          npm install   
+          deactivate
+          cd -
+
+
+      - name: Deploy test stack
+        if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}"
+        id: deploy_step
+        run: |
+          source .deployment_venv/bin/activate
+
+          # synthesize the stack
+          cd integration_tests/cdk
+          npx cdk synth --debug --all --require-approval never
+
+          # deploy the stack and grab URLs for testing
+          npx cdk deploy --ci --all --require-approval never
+          echo "ingestor_url=$(aws cloudformation describe-stacks --stack-name eoapi-cdk-integration-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'stacingestor')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT
+          echo "stac_api_url=$(aws cloudformation describe-stacks --stack-name eoapi-cdk-integration-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'pgstacapi')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT
+          echo "titiler_pgstac_api_url=$(aws cloudformation describe-stacks --stack-name eoapi-cdk-integration-test-pgSTAC-infra --query "Stacks[0].Outputs[?starts_with(OutputKey, 'titilerpgstac')].OutputValue | [0]" --output text)" >> $GITHUB_OUTPUT
+          deactivate
+          cd -
+
+      - name: Tear down any infrastructure
+        if: always()
+        run: |
+          cd integration_tests/cdk
+          # run this only if we find a 'cdk.out' directory, which means there might be things to tear down
+          if [ -d "cdk.out" ]; then
+            cd -
+            source .deployment_venv/bin/activate
+            cd integration_tests/cdk
+            # see https://github.com/aws/aws-cdk/issues/24946
+            rm -f cdk.out/synth.lock
+            npx cdk destroy --ci --all --force
+          fi
+
+
+      # run if the previous step set SHOULD_RELEASE to true
+      - name: Maybe Release 🚀
+        # only run if the previous step set SHOULD_RELEASE to true
+        if: "${{ inputs.release && steps.check_release.outputs.SHOULD_RELEASE }}"
         run: |
           npm run semantic-release
         env:

.github/workflows/build_and_release.yaml

@@ -0,0 +1,17 @@
+name: Build & try to release
+
+on:
+  push:
+
+jobs:
+  package:
+    uses: ./.github/workflows/build.yaml
+    with:
+      release: true
+    secrets:
+      DS_RELEASE_BOT_ID: ${{ secrets.DS_RELEASE_BOT_ID }}
+      DS_RELEASE_BOT_PRIVATE_KEY: ${{ secrets.DS_RELEASE_BOT_PRIVATE_KEY }}
+      AWS_DEFAULT_REGION_DEPLOY: ${{ secrets.AWS_DEFAULT_REGION_DEPLOY }}
+      AWS_ACCESS_KEY_ID_DEPLOY: ${{ secrets.AWS_ACCESS_KEY_ID_DEPLOY }}
+      AWS_SECRET_ACCESS_KEY_DEPLOY: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEPLOY }}
+      AWS_ACCOUNT_ID: ${{ secrets.AWS_ACCOUNT_ID }}

.github/workflows/distribute.yaml

@@ -13,11 +13,6 @@ jobs:
     runs-on: ubuntu-latest
     needs: package
     steps:
-      - uses: actions/download-artifact@v3
-        with:
-          name: python
-          path: dist
-
       - run: pip install twine
 
       - run: twine upload dist/*

.gitignore

@@ -8,3 +8,6 @@ docs
 __pycache__
 .venv
 .tox
+tests/*.egg*
+tests/*venv*
+tests/__pycache__

README.md

@@ -54,3 +54,6 @@ Versioning is automatically handled via [Conventional Commits](https://www.conve
 _Warning_: If you rebase `main`, you must ensure that the commits referenced by tags point to commits that are within the `main` branch. If a commit references a commit that is no longer on the `main` branch, Semantic Release will fail to detect the correct version of the project. [More information](https://github.com/semantic-release/semantic-release/issues/1121#issuecomment-517945233).
 
 
+## Tests
+
+Each new release triggers an integration test against a running deployment that uses the newly releases constructs. See the corresponding [github workflow](https://github.com/developmentseed/eoapi-cdk/blob/main/.github/workflows/deploy.yaml) and the [tests definition](https://github.com/developmentseed/eoapi-cdk/blob/main/tests).

integration_tests/cdk/README.md

@@ -0,0 +1,57 @@
+This is a non-forked version of [eoapi-template](https://github.com/developmentseed/eoapi-template).
+
+# Deployment CDK code for eoapi-cdk integration tests
+
+This is a wrapper CDK code that provides the `eoapi-cdk` deployment to run integration tests on the latest releases of the `eoapi-cdk` constructs.
+
+## Requirements
+
+- python
+- docker
+- node
+- AWS credentials environment variables configured to point to an account. 
+- **Optional** a `config.yaml` file to override the default deployment settings defined in `config.py`.
+
+## Installation
+
+Install python dependencies with 
+
+```
+python -m venv .venv
+source .venv/bin/activate
+python -m pip install -r requirements.txt
+```
+
+Install the latest `eoapi-cdk` either from PyPI:
+
+```
+pip install eoapi-cdk
+```
+
+Or alternatively, compile and package from the root of this repository to get the python version of the constructs locally.
+
+Also install node dependencies with 
+
+```
+npm install
+```
+
+Verify that the `cdk` CLI is available. Since `aws-cdk` is installed as a local dependency, you can use the `npx` node package runner tool, that comes with `npm`.
+
+```
+npx cdk --version
+```
+
+## Deployment
+
+First, synthesize the app 
+
+```
+npx cdk synth --all
+```
+
+Then, deploy
+
+```
+npx cdk deploy --all --require-approval never
+```

integration_tests/cdk/app.py

@@ -0,0 +1,17 @@
+from aws_cdk import App
+
+from config import build_app_config
+from eoapi_template import pgStacInfra, vpc
+
+app = App()
+
+app_config = build_app_config()
+
+vpc_stack = vpc.VpcStack(scope=app, app_config=app_config)
+
+pgstac_infra_stack = pgStacInfra.pgStacInfraStack(
+    scope=app,
+    vpc=vpc_stack.vpc,
+    app_config=app_config,
+)
+app.synth()

integration_tests/cdk/cdk.json

@@ -0,0 +1,32 @@
+{
+  "app": "python3 app.py",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "requirements*.txt",
+      "source.bat",
+      "**/*.pyc",
+      "**/*.tmp",
+      "**/__pycache__",
+      "tests",
+      "scripts",
+      "*venv"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true,
+    "@aws-cdk/core:stackRelativeExports": true,
+    "@aws-cdk/aws-rds:lowercaseDbIdentifier": true,
+    "@aws-cdk/aws-lambda:recognizeVersionProps": true,
+    "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true,
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ]
+  }
+}

integration_tests/cdk/config.py

@@ -0,0 +1,58 @@
+from typing import Dict
+
+import pydantic
+import yaml
+from pydantic_core.core_schema import FieldValidationInfo
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class AppConfig(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env"
+    )
+    aws_default_account: str = pydantic.Field(
+        description="AWS account ID"
+    )
+    project_id: str = pydantic.Field(
+        description="Project ID", default="eoapi-cdk"
+    )
+    stage: str = pydantic.Field(description="Stage of deployment", default="test")
+    # because of its validator, `tags` should always come after `project_id` and `stage`
+    tags: Dict[str, str] | None = pydantic.Field(
+        description="""Tags to apply to resources. If none provided,
+        will default to the defaults defined in `default_tags`.
+        Note that if tags are passed to the CDK CLI via `--tags`,
+        they will override any tags defined here.""",
+        default=None,
+    )
+    db_instance_type: str = pydantic.Field(
+        description="Database instance type", default="t3.micro"
+    )
+    db_allocated_storage: int = pydantic.Field(
+        description="Allocated storage for the database", default=5
+    )
+
+    @pydantic.field_validator("tags")
+    def default_tags(cls, v, info: FieldValidationInfo):
+        return v or {"project_id": info.data["project_id"], "stage": info.data["stage"]}
+
+    def build_service_name(self, service_id: str) -> str:
+        return f"{self.project_id}-{self.stage}-{service_id}"
+
+
+def build_app_config() -> AppConfig:
+    """Builds the AppConfig object from config.yaml file if exists,
+    otherwise use defaults"""
+    try:
+        with open("config.yaml") as f:
+            print("Loading config from config.yaml")
+            app_config = yaml.safe_load(f)
+            app_config = (
+                {} if app_config is None else app_config
+            )  # if config is empty, set it to an empty dict
+            app_config = AppConfig(**app_config)
+    except FileNotFoundError:
+        # if no config at the expected path, using defaults
+        app_config = AppConfig()
+
+    return app_config