switch docker compose file to default to official runtimes

developmentseed · Oct 26, 2023 · 7f8b614 · 7f8b614
1 parent adcb499
commit 7f8b614
Show file tree

Hide file tree

Showing 7 changed files with 367 additions and 296 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -38,7 +38,7 @@ jobs:
           pre-commit run --all-files
 
       - name: Launch services
-        run: docker compose up -d stac raster vector
+        run: docker compose -f docker-compose.custom.yml --profile gunicorn up -d
 
       - name: install lib postgres
         run: |
@@ -67,8 +67,8 @@ jobs:
       # see https://github.com/developmentseed/tipg/issues/37
       - name: Restart the Vector service
         run: |
-          docker compose stop vector
-          docker compose up -d vector
+          docker compose -f docker-compose.custom.yml --profile gunicorn stop vector
+          docker compose -f docker-compose.custom.yml --profile gunicorn up -d vector
 
       - name: Sleep for 10 seconds
         run: sleep 10s
@@ -78,7 +78,7 @@ jobs:
         run: python -m pytest .github/workflows/tests/
 
       - name: Stop services
-        run: docker compose stop
+        run: docker compose -f docker-compose.custom.yml  stop
 
 
   publish-docker:

diff --git a/README.md b/README.md
@@ -60,35 +60,80 @@ Then you can start exploring your dataset with:
 
 If you've added a vector dataset to the `public` schema in the Postgres database, they will be available through the **Vector** service at [http://localhost:8083](http://localhost:8083).
 
-Alternatively, you may launch the application locally:
+## Deployment
+
+This repository has current runtimes that are consistently updated with new functionality.
+
+The services can be deployed locally via docker with `docker compose up`.
+
+Two Infrastructure as Code (IaC) repositories are available:
+- [eoapi-cdk](https://github.com/developmentseed/eoapi-cdk): A set of AWS CDK constructs to deploy eoAPI services
+- [eoapi-k8s](https://github.com/developmentseed/eoapi-k8s): IaC and Helm charts for deploying eoAPI services on AWS and GCP
+
+Finally, [eoapi-template](https://github.com/developmentseed/eoapi-template) is an AWS CDK app that shows how to configure the eoapi-cdk constructs.
+
+Alternatively, you may install the libraries locally:
+
+<details>
+
 ```bash
 python -m pip install --upgrade virtualenv
 virtualenv .venv
 source .venv/bin/activate
 
-python -m pip install "psycopg[binary,pool]" uvicorn
-python -m pip install runtime/eoapi/{SERVICE}  # SERVICE should be one of `raster, vector, stac.`
-
 export DATABASE_URL=postgresql://username:[email protected]:5439/postgis  # Connect to the database of your choice
 
-.venv/bin/uvicorn eoapi.{SERVICE}.app:app --port 8000 --reload
+python -m pip install uvicorn
+
+###############################################################################
+# Install and launch the application
+# Select one of the following
+
+###############################################################################
+# STAC
+python -m pip install "psycopg[binary,pool]" stac-fastapi-pgstac
+.venv/bin/uvicorn stac_fastapi.pgstac.app:app --port 8081 --reload
+
+###############################################################################
+# RASTER
+python -m pip install "psycopg[binary,pool]" titiler-pgstac
+.venv/bin/uvicorn titiler.pgstac.main:app --port 8082 --reload
+
+###############################################################################
+# VECTOR
+python -m pip install tipg
+.venv/bin/uvicorn tipg.main:app --port 8083 --reload
 ```
 
-Note: services might have incompatible dependencies, which you can resolve by using a virtual environment for each service
+Note: python libraries might have incompatible dependencies, which you can resolve by using a virtual environment for each ones
 
----
+</details>
 
-## Deployment
+## Custom runtimes
 
-This repository has current runtimes that are consistently updated with new functionality.
+The eoAPI repository hosts customized versions of each base service which can work in parallel or in combination with each other.
 
-The services can be deployed locally via docker with `docker-compose up`. The official runtimes can be launched with `docker compose -f docker-compose.official.yml up stac-fastapi titiler-pgstac tipg`.
+eoAPI custom runtimes can be launched with docker
 
-Two Infrastructure as Code (IaC) repositories are available:
-- [eoapi-cdk](https://github.com/developmentseed/eoapi-cdk): A set of AWS CDK constructs to deploy eoAPI services
-- [eoapi-k8s](https://github.com/developmentseed/eoapi-k8s): IaC and Helm charts for deploying eoAPI services on AWS and GCP
+```
+docker compose -f docker-compose.custom.yml --profile gunicorn up
+```
 
-Finally, [eoapi-template](https://github.com/developmentseed/eoapi-template) is an AWS CDK app that shows how to configure the eoapi-cdk constructs.
+Alternatively, you may launch the application locally:
+```bash
+python -m pip install --upgrade virtualenv
+virtualenv .venv
+source .venv/bin/activate
+
+python -m pip install "psycopg[binary,pool]" uvicorn
+python -m pip install runtime/eoapi/{SERVICE}  # SERVICE should be one of `raster, vector, stac.`
+
+export DATABASE_URL=postgresql://username:[email protected]:5439/postgis  # Connect to the database of your choice
+
+.venv/bin/uvicorn eoapi.{SERVICE}.app:app --port 8000 --reload
+```
+
+Note: services might have incompatible dependencies, which you can resolve by using a virtual environment for each service
 
 ## Contribution & Development
 
@@ -113,4 +158,4 @@ See [contributors](https://github.com/developmentseed/eoAPI/graphs/contributors)
 
 ## Changes
 
-See [CHANGES.md](https://github.com/developmentseed/eoAPI/blob/main/CHANGES.md).
+See [CHANGES.md](https://github.com/developmentseed/eoAPI/blob/main/CHANGES.md).
diff --git a/docker-compose.custom.yml b/docker-compose.custom.yml
@@ -0,0 +1,264 @@
+version: '3'
+
+services:
+  stac:
+    container_name: eoapi.stac
+    profiles:
+      - gunicorn
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile.stac
+    ports:
+      - "${MY_DOCKER_IP:-127.0.0.1}:8081:8081"
+    environment:
+      - APP_HOST=0.0.0.0
+      - APP_PORT=8081
+      - HOST=0.0.0.0
+      - PORT=8081
+      - ENVIRONMENT=local
+      # https://github.com/tiangolo/uvicorn-gunicorn-docker#web_concurrency
+      - WEB_CONCURRENCY=10
+      # https://github.com/tiangolo/uvicorn-gunicorn-docker#workers_per_core
+      # - WORKERS_PER_CORE=1
+      # https://github.com/tiangolo/uvicorn-gunicorn-docker#max_workers
+      # - MAX_WORKERS=10
+      # Postgres connection
+      - POSTGRES_USER=username
+      - POSTGRES_PASS=password
+      - POSTGRES_DBNAME=postgis
+      - POSTGRES_HOST_READER=database
+      - POSTGRES_HOST_WRITER=database
+      - POSTGRES_PORT=5432
+      - DB_MIN_CONN_SIZE=1
+      - DB_MAX_CONN_SIZE=10
+      # https://github.com/developmentseed/eoAPI/issues/16
+      # - TITILER_ENDPOINT=raster
+      - TITILER_ENDPOINT=http://127.0.0.1:8082
+    depends_on:
+      - database
+      - raster
+    command:
+      bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && /start.sh"
+    volumes:
+      - ./dockerfiles/scripts:/tmp/scripts
+
+  raster:
+    container_name: eoapi.raster
+    profiles:
+      - gunicorn
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile.raster
+    # At the time of writing, rasterio and psycopg wheels are not available for arm64 arch
+    # so we force the image to be built with linux/amd64
+    platform: linux/amd64
+    ports:
+      - "${MY_DOCKER_IP:-127.0.0.1}:8082:8082"
+    environment:
+      # Application
+      - HOST=0.0.0.0
+      - PORT=8082
+      # https://github.com/tiangolo/uvicorn-gunicorn-docker#web_concurrency
+      - WEB_CONCURRENCY=1
+      # https://github.com/tiangolo/uvicorn-gunicorn-docker#workers_per_core
+      - WORKERS_PER_CORE=1
+      # https://github.com/tiangolo/uvicorn-gunicorn-docker#max_workers
+      - MAX_WORKERS=10
+      # Postgres connection
+      - POSTGRES_USER=username
+      - POSTGRES_PASS=password
+      - POSTGRES_DBNAME=postgis
+      - POSTGRES_HOST=database
+      - POSTGRES_PORT=5432
+      - DB_MIN_CONN_SIZE=1
+      - DB_MAX_CONN_SIZE=10
+      # - DB_MAX_QUERIES=10
+      # - DB_MAX_IDLE=10
+      # GDAL Config
+      - CPL_TMPDIR=/tmp
+      - GDAL_CACHEMAX=75%
+      - GDAL_INGESTED_BYTES_AT_OPEN=32768
+      - GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR
+      - GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES
+      - GDAL_HTTP_MULTIPLEX=YES
+      - GDAL_HTTP_VERSION=2
+      - VSI_CACHE=TRUE
+      - VSI_CACHE_SIZE=536870912
+      # TiTiler Config
+      - MOSAIC_CONCURRENCY=1
+      # AWS S3 endpoint config
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      # API Config
+      - EOAPI_RASTER_ENABLE_MOSAIC_SEARCH=TRUE
+    depends_on:
+      - database
+    command:
+      bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && /start.sh"
+    volumes:
+      - ./dockerfiles/scripts:/tmp/scripts
+
+  vector:
+    container_name: eoapi.vector
+    profiles:
+      - gunicorn
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile.vector
+    ports:
+      - "${MY_DOCKER_IP:-127.0.0.1}:8083:8083"
+    environment:
+      # Application
+      - HOST=0.0.0.0
+      - PORT=8083
+      # https://github.com/tiangolo/uvicorn-gunicorn-docker#web_concurrency
+      - WEB_CONCURRENCY=10
+      # https://github.com/tiangolo/uvicorn-gunicorn-docker#workers_per_core
+      # - WORKERS_PER_CORE=1
+      # https://github.com/tiangolo/uvicorn-gunicorn-docker#max_workers
+      # - MAX_WORKERS=10
+      # Postgres connection
+      - POSTGRES_USER=username
+      - POSTGRES_PASS=password
+      - POSTGRES_DBNAME=postgis
+      - POSTGRES_HOST=database
+      - POSTGRES_PORT=5432
+      - DB_MIN_CONN_SIZE=1
+      - DB_MAX_CONN_SIZE=10
+    command:
+      bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && /start.sh"
+    depends_on:
+      - database
+    volumes:
+      - ./dockerfiles/scripts:/tmp/scripts
+
+  stac-uvicorn:
+    container_name: eoapi.stac-uvicorn
+    profiles:
+      - uvicorn
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile.stac-uvicorn
+    ports:
+      - "${MY_DOCKER_IP:-127.0.0.1}:8081:8081"
+    environment:
+      - APP_HOST=0.0.0.0
+      - APP_PORT=8081
+      - HOST=0.0.0.0
+      - PORT=8081
+      - ENVIRONMENT=local
+      # Postgres connection
+      - POSTGRES_USER=username
+      - POSTGRES_PASS=password
+      - POSTGRES_DBNAME=postgis
+      - POSTGRES_HOST_READER=database
+      - POSTGRES_HOST_WRITER=database
+      - POSTGRES_PORT=5432
+      - DB_MIN_CONN_SIZE=1
+      - DB_MAX_CONN_SIZE=10
+      # https://github.com/developmentseed/eoAPI/issues/16
+      # - TITILER_ENDPOINT=raster
+      - TITILER_ENDPOINT=http://127.0.0.1:8082
+    depends_on:
+      - database
+      - raster-uvicorn
+    command:
+      bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && uvicorn eoapi.stac.app:app --host 0.0.0.0 --port 8081"
+    volumes:
+      - ./dockerfiles/scripts:/tmp/scripts
+
+  raster-uvicorn:
+    container_name: eoapi.raster-uvicorn
+    profiles:
+      - uvicorn
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile.raster-uvicorn
+    # At the time of writing, rasterio and psycopg wheels are not available for arm64 arch
+    # so we force the image to be built with linux/amd64
+    platform: linux/amd64
+    ports:
+      - "${MY_DOCKER_IP:-127.0.0.1}:8082:8082"
+    environment:
+      # Application
+      - HOST=0.0.0.0
+      - PORT=8082
+      # Postgres connection
+      - POSTGRES_USER=username
+      - POSTGRES_PASS=password
+      - POSTGRES_DBNAME=postgis
+      - POSTGRES_HOST=database
+      - POSTGRES_PORT=5432
+      - DB_MIN_CONN_SIZE=1
+      - DB_MAX_CONN_SIZE=10
+      # GDAL Config
+      - CPL_TMPDIR=/tmp
+      - GDAL_CACHEMAX=75%
+      - GDAL_INGESTED_BYTES_AT_OPEN=32768
+      - GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR
+      - GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES
+      - GDAL_HTTP_MULTIPLEX=YES
+      - GDAL_HTTP_VERSION=2
+      - VSI_CACHE=TRUE
+      - VSI_CACHE_SIZE=536870912
+      # TiTiler Config
+      - MOSAIC_CONCURRENCY=1
+      # AWS S3 endpoint config
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      # API Config
+      - EOAPI_RASTER_ENABLE_MOSAIC_SEARCH=TRUE
+    depends_on:
+      - database
+    command:
+      bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && uvicorn eoapi.raster.app:app --host 0.0.0.0 --port 8082"
+    volumes:
+      - ./dockerfiles/scripts:/tmp/scripts
+
+  vector-uvicorn:
+    container_name: eoapi.vector-uvicorn
+    profiles:
+      - uvicorn
+    build:
+      context: .
+      dockerfile: dockerfiles/Dockerfile.vector-uvicorn
+    ports:
+      - "${MY_DOCKER_IP:-127.0.0.1}:8083:8083"
+    environment:
+      # Application
+      - HOST=0.0.0.0
+      - PORT=8083
+      # Postgres connection
+      - POSTGRES_USER=username
+      - POSTGRES_PASS=password
+      - POSTGRES_DBNAME=postgis
+      - POSTGRES_HOST=database
+      - POSTGRES_PORT=5432
+      - DB_MIN_CONN_SIZE=1
+      - DB_MAX_CONN_SIZE=10
+    command:
+      bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h database -p 5432 && uvicorn eoapi.vector.app:app --host 0.0.0.0 --port 8083"
+    depends_on:
+      - database
+    volumes:
+      - ./dockerfiles/scripts:/tmp/scripts
+
+  database:
+    container_name: eoapi.db
+    image: ghcr.io/stac-utils/pgstac:v0.8.1
+    environment:
+      - POSTGRES_USER=username
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_DB=postgis
+      - PGUSER=username
+      - PGPASSWORD=password
+      - PGDATABASE=postgis
+    ports:
+      - "${MY_DOCKER_IP:-127.0.0.1}:5439:5432"
+    command: postgres -N 500
+    volumes:
+      - ./.pgdata:/var/lib/postgresql/data
+
+networks:
+  default:
+    name: eoapi-network