From 626469d0adeadafa6da3fd5a554cfd24cf08c3fd Mon Sep 17 00:00:00 2001 From: sambles Date: Mon, 13 May 2024 11:26:49 +0100 Subject: [PATCH] Docker compose, removed debugging options from default (#1036) * Move debug compose file * disable debug options and remove src mounts in deafult compose * Updated Package Requirements: aiohttp==3.9.4 * test --------- Co-authored-by: awsbuild --- compose/debug.docker-compose.yml | 232 ++++++++++++------ docker-compose.yml | 20 +- kubernetes/worker-controller/requirements.txt | 2 +- requirements-server.txt | 4 +- requirements-worker.txt | 2 +- requirements.txt | 2 +- scripts/deploy.sh | 3 +- 7 files changed, 171 insertions(+), 94 deletions(-) diff --git a/compose/debug.docker-compose.yml b/compose/debug.docker-compose.yml index 1771447c5..d70b7eaf2 100755 --- a/compose/debug.docker-compose.yml +++ b/compose/debug.docker-compose.yml @@ -1,30 +1,51 @@ version: '3.4' +volumes: + server-db-OasisData: + celery-db-OasisData: + filestore-OasisData: x-shared-env: &shared-env OASIS_DEBUG: 1 - #OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672" - OASIS_CELERY_BROKER_URL: "redis://broker:6379" + OASIS_KEEP_RUN_DIR: 1 + OASIS_KEEP_LOCAL_DATA: 1 + OASIS_KEEP_REMOTE_DATA: 1 + OASIS_URL_SUB_PATH: 0 + OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672" + #OASIS_CELERY_BROKER_URL: "redis://broker:6379" OASIS_SERVER_DB_HOST: server-db OASIS_SERVER_DB_PASS: oasis OASIS_SERVER_DB_USER: oasis OASIS_SERVER_DB_NAME: oasis - OASIS_SERVER_DB_PORT: 3306 + OASIS_SERVER_DB_PORT: 5432 OASIS_SERVER_CHANNEL_LAYER_HOST: channel-layer - OASIS_CELERY_DB_ENGINE: db+mysql+pymysql + OASIS_SERVER_DB_ENGINE: django.db.backends.postgresql_psycopg2 + OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2 OASIS_CELERY_DB_HOST: celery-db OASIS_CELERY_DB_PASS: password OASIS_CELERY_DB_USER: celery OASIS_CELERY_DB_NAME: celery - OASIS_CELERY_DB_PORT: 3306 + OASIS_CELERY_DB_PORT: 5432 OASIS_TASK_CONTROLLER_QUEUE: task-controller + +x-oasis-env-v1: &oasis-env-v1 + OASIS_DEBUG: ${DEBUG:-0} + OASIS_RABBIT_HOST: broker + OASIS_RABBIT_PORT: 5672 + OASIS_RABBIT_USER: rabbit + OASIS_RABBIT_PASS: rabbit + OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2 + OASIS_CELERY_DB_HOST: celery-db + OASIS_CELERY_DB_PASS: password + OASIS_CELERY_DB_USER: celery + OASIS_CELERY_DB_NAME: celery + OASIS_CELERY_DB_PORT: 5432 + x-volumes: &shared-volumes - - ${OASIS_MEDIA_ROOT:-./docker-shared-fs}:/shared-fs:rw - # DEBUG Mounts - - ./src/startup_server.sh:/usr/local/bin/startup - - ./src/server/oasisapi/analyses:/var/www/oasis/src/server/oasisapi/analyses + - filestore-OasisData:/shared-fs:rw services: server: restart: always - image: coreoasis/api_server:latest + image: coreoasis/api_server:dev + command: ["./wsgi/run-wsgi.sh"] build: context: . dockerfile: Dockerfile.api_server @@ -40,33 +61,67 @@ services: STARTUP_RUN_MIGRATIONS: "true" OASIS_ADMIN_USER: admin OASIS_ADMIN_PASS: password - volumes: *shared-volumes - worker-monitor: + volumes: + - filestore-OasisData:/shared-fs:rw + - ../src/server/oasisapi:/var/www/oasis/src/server/oasisapi + server_websocket: restart: always - image: coreoasis/api_server:latest - command: [celery, -A, src.server.oasisapi, worker, --loglevel=INFO] + image: coreoasis/api_server:dev + command: ["./asgi/run-asgi.sh"] links: - server-db - celery-db - broker + ports: + - 8001:8001 environment: <<: *shared-env - volumes: *shared-volumes - task-controller: + volumes: + - filestore-OasisData:/shared-fs:rw + - ../src/server/oasisapi:/var/www/oasis/src/server/oasisapi + v1-worker-monitor: restart: always - image: coreoasis/api_server:latest - command: [celery, -A, src.server.oasisapi, worker, --loglevel=INFO, -Q, task-controller] + image: coreoasis/api_server:dev + command: [celery, -A, 'src.server.oasisapi.celery_app_v1', worker, --loglevel=INFO,] links: - server-db - celery-db - broker environment: <<: *shared-env - volumes: *shared-volumes - celery-beat: + volumes: + - filestore-OasisData:/shared-fs:rw + - ../src/server/oasisapi:/var/www/oasis/src/server/oasisapi + v2-worker-monitor: + restart: always + image: coreoasis/api_server:dev + command: [celery, -A, 'src.server.oasisapi.celery_app_v2', worker, --loglevel=INFO, -Q, celery-v2] + links: + - server-db + - celery-db + - broker + environment: + <<: *shared-env + volumes: + - filestore-OasisData:/shared-fs:rw + - ../src/server/oasisapi:/var/www/oasis/src/server/oasisapi + v2-task-controller: restart: always - image: coreoasis/api_server:latest - command: [celery, -A, src.server.oasisapi, beat, --loglevel=INFO] + image: coreoasis/api_server:dev + command: [celery, -A, 'src.server.oasisapi.celery_app_v2', worker, --loglevel=INFO, -Q, task-controller] + links: + - server-db + - celery-db + - broker + environment: + <<: *shared-env + volumes: + - filestore-OasisData:/shared-fs:rw + - ../src/server/oasisapi/analyses:/var/www/oasis/src/server/oasisapi/analyses + celery-beat_v2: + restart: always + image: coreoasis/api_server:dev + command: [celery, -A, src.server.oasisapi.celery_app_v2, beat, --loglevel=INFO] links: - server-db - celery-db @@ -74,9 +129,24 @@ services: environment: <<: *shared-env volumes: *shared-volumes - worker: + stable-worker: + restart: always + image: coreoasis/model_worker:1.28.4 + links: + - celery-db + - broker:mybroker + environment: + <<: *oasis-env-v1 + OASIS_MODEL_DATA_DIRECTORY: /home/worker/model + OASIS_MODEL_SUPPLIER_ID: OasisLMF + OASIS_MODEL_ID: PiWind + OASIS_MODEL_VERSION_ID: '1.28.4' + volumes: + - ${OASIS_MODEL_DATA_DIR:-./data/static}:/home/worker/model:rw + - filestore-OasisData:/shared-fs:rw + v1-worker: restart: always - image: coreoasis/model_worker:latest + image: coreoasis/model_worker:dev build: context: . dockerfile: Dockerfile.model_worker @@ -87,73 +157,89 @@ services: <<: *shared-env OASIS_MODEL_SUPPLIER_ID: OasisLMF OASIS_MODEL_ID: PiWind - OASIS_MODEL_VERSION_ID: 1 - OASIS_MODEL_NUM_ANALYSIS_CHUNKS: 8 + OASIS_MODEL_VERSION_ID: 'v1' + OASIS_RUN_MODE: v1 volumes: - - /home/sam/repos/models/piwind:/home/worker/model:rw - - ${OASIS_MEDIA_ROOT:-./docker-shared-fs}:/shared-fs:rw - # DEBUG Mounts - - ./src/model_execution_worker:/home/worker/src/model_execution_worker - - ./src/startup_worker.sh:/home/worker/startup.sh - + - ${OASIS_MODEL_DATA_DIR:-./data/static}:/home/worker/model:rw + - ../src/model_execution_worker:/home/worker/src/model_execution_worker + - filestore-OasisData:/shared-fs:rw + v2-worker: + restart: always + image: coreoasis/model_worker:dev + build: + context: . + dockerfile: Dockerfile.model_worker + links: + - celery-db + - broker:mybroker + environment: + <<: *shared-env + OASIS_MODEL_SUPPLIER_ID: OasisLMF + OASIS_MODEL_ID: PiWind + OASIS_MODEL_VERSION_ID: 'v2' + OASIS_RUN_MODE: v2 + volumes: + - ${OASIS_MODEL_DATA_DIR:-./data/static}:/home/worker/model:rw + - ../src/model_execution_worker:/home/worker/src/model_execution_worker + - filestore-OasisData:/shared-fs:rw server-db: restart: always - image: mysql:8.0 - command: - - --default-authentication-plugin=mysql_native_password + image: postgres environment: - - MYSQL_ROOT_PASSWORD=password - - MYSQL_USER=oasis - - MYSQL_PASSWORD=oasis - - MYSQL_DATABASE=oasis + - POSTGRES_DB=oasis + - POSTGRES_USER=oasis + - POSTGRES_PASSWORD=oasis volumes: - - ${OASIS_DOCKER_DB_DATA_DIR:-./db-data}/server:/var/lib/mysql/:rw + - server-db-OasisData:/var/lib/postgresql/data:rw ports: - - 33307:3306 + - 33307:5432 celery-db: restart: always - image: mysql - command: - - --default-authentication-plugin=mysql_native_password + image: postgres environment: - - MYSQL_ROOT_PASSWORD=password - - MYSQL_USER=celery - - MYSQL_PASSWORD=password - - MYSQL_DATABASE=celery - ports: - - 33306:3306 + - POSTGRES_DB=celery + - POSTGRES_USER=celery + - POSTGRES_PASSWORD=password volumes: - - ${OASIS_DOCKER_DB_DATA_DIR:-./db-data}/celery:/var/lib/mysql/:rw - broker: - restart: always - image: redis:5.0.7 + - celery-db-OasisData:/var/lib/postgresql/data:rw ports: - - 6379 + - 33306:5432 # broker: # restart: always -# image: rabbitmq:3.8.14-management -# environment: -# - RABBITMQ_DEFAULT_USER=rabbit -# - RABBITMQ_DEFAULT_PASS=rabbit +# image: redis:5.0.7 # ports: -# - 5672:5672 -# - 15672:15672 - flower: +# - 5672:6379 + broker: restart: always - image: iserko/docker-celery-flower - ports: - - 5555:5555 + image: rabbitmq:3.8.14-management environment: - - CELERY_BROKER_URL=amqp://rabbit:rabbit@broker:5672 - entrypoint: - - flower - - --port=5555 - - --broker_api=http://rabbit:rabbit@broker:15672/api/ - links: - - celery-db - - broker + - RABBITMQ_DEFAULT_USER=rabbit + - RABBITMQ_DEFAULT_PASS=rabbit + ports: + - 5672:5672 + - 15672:15672 channel-layer: restart: always image: redis:5.0.7 ports: - 6379:6379 + user-interface: + restart: always + image: coreoasis/oasisui_app:${VERS_UI:-latest} + container_name: oasisui_app + environment: + - API_IP=server:8000/api/ + - API_VERSION=v2 + - API_SHARE_FILEPATH=./downloads + - OASIS_ENVIRONMENT=oasis_localhost + ports: + - 8080:3838 + portainer: + restart: always + image: portainer/portainer:latest + ports: + - 8002:8002 + - 9000:9000 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./portainer_data:/data portainer/portainer diff --git a/docker-compose.yml b/docker-compose.yml index 12cb56c37..9c8e5662d 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,10 +4,10 @@ volumes: celery-db-OasisData: filestore-OasisData: x-shared-env: &shared-env - OASIS_DEBUG: 1 - OASIS_KEEP_RUN_DIR: 1 - OASIS_KEEP_LOCAL_DATA: 1 - OASIS_KEEP_REMOTE_DATA: 1 + OASIS_DEBUG: 0 + OASIS_KEEP_RUN_DIR: 0 + OASIS_KEEP_LOCAL_DATA: 0 + OASIS_KEEP_REMOTE_DATA: 0 OASIS_URL_SUB_PATH: 0 OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672" #OASIS_CELERY_BROKER_URL: "redis://broker:6379" @@ -63,7 +63,6 @@ services: OASIS_ADMIN_PASS: password volumes: - filestore-OasisData:/shared-fs:rw - - ./src/server/oasisapi:/var/www/oasis/src/server/oasisapi server_websocket: restart: always image: coreoasis/api_server:dev @@ -78,7 +77,6 @@ services: <<: *shared-env volumes: - filestore-OasisData:/shared-fs:rw - - ./src/server/oasisapi:/var/www/oasis/src/server/oasisapi v1-worker-monitor: restart: always image: coreoasis/api_server:dev @@ -91,7 +89,6 @@ services: <<: *shared-env volumes: - filestore-OasisData:/shared-fs:rw - - ./src/server/oasisapi:/var/www/oasis/src/server/oasisapi v2-worker-monitor: restart: always image: coreoasis/api_server:dev @@ -104,7 +101,6 @@ services: <<: *shared-env volumes: - filestore-OasisData:/shared-fs:rw - - ./src/server/oasisapi:/var/www/oasis/src/server/oasisapi v2-task-controller: restart: always image: coreoasis/api_server:dev @@ -117,7 +113,6 @@ services: <<: *shared-env volumes: - filestore-OasisData:/shared-fs:rw - - ./src/server/oasisapi/analyses:/var/www/oasis/src/server/oasisapi/analyses celery-beat_v2: restart: always image: coreoasis/api_server:dev @@ -161,7 +156,6 @@ services: OASIS_RUN_MODE: v1 volumes: - ${OASIS_MODEL_DATA_DIR:-./data/static}:/home/worker/model:rw - - ./src/model_execution_worker:/home/worker/src/model_execution_worker - filestore-OasisData:/shared-fs:rw v2-worker: restart: always @@ -180,7 +174,6 @@ services: OASIS_RUN_MODE: v2 volumes: - ${OASIS_MODEL_DATA_DIR:-./data/static}:/home/worker/model:rw - - ./src/model_execution_worker:/home/worker/src/model_execution_worker - filestore-OasisData:/shared-fs:rw server-db: restart: always @@ -204,11 +197,6 @@ services: - celery-db-OasisData:/var/lib/postgresql/data:rw ports: - 33306:5432 -# broker: -# restart: always -# image: redis:5.0.7 -# ports: -# - 5672:6379 broker: restart: always image: rabbitmq:3.8.14-management diff --git a/kubernetes/worker-controller/requirements.txt b/kubernetes/worker-controller/requirements.txt index b8c3e4f51..466784497 100644 --- a/kubernetes/worker-controller/requirements.txt +++ b/kubernetes/worker-controller/requirements.txt @@ -4,7 +4,7 @@ # # pip-compile kubernetes/worker-controller/requirements.in # -aiohttp==3.9.0 +aiohttp==3.9.4 # via # -r kubernetes/worker-controller/requirements.in # kubernetes-asyncio diff --git a/requirements-server.txt b/requirements-server.txt index 3fa67537c..3aee6238d 100644 --- a/requirements-server.txt +++ b/requirements-server.txt @@ -211,7 +211,9 @@ numpy==1.25.2 # pandas # pyarrow oasis-data-manager==0.1.3 - # via ods-tools + # via + # -r requirements-server.in + # ods-tools ods-tools==3.2.3 # via -r requirements-server.in packaging==23.2 diff --git a/requirements-worker.txt b/requirements-worker.txt index ee251c2bd..7508954ea 100644 --- a/requirements-worker.txt +++ b/requirements-worker.txt @@ -8,7 +8,7 @@ adlfs==2024.2.0 # via -r requirements-worker.in aiobotocore==2.12.1 # via s3fs -aiohttp==3.9.3 +aiohttp==3.9.4 # via # adlfs # aiobotocore diff --git a/requirements.txt b/requirements.txt index 3e06e6ed8..37e584683 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ adlfs==2024.2.0 # via -r ./requirements-worker.in aiobotocore==2.12.1 # via s3fs -aiohttp==3.9.3 +aiohttp==3.9.4 # via # adlfs # aiobotocore diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 1262e9a04..27fde9658 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -62,5 +62,6 @@ fi set -e docker build -f Dockerfile.api_server $BUILD_ARGS_SERVER -t coreoasis/api_server:dev . docker build -f Dockerfile.model_worker $BUILD_ARGS_WORKER -t coreoasis/model_worker:dev . + # docker-compose -f compose/s3.docker-compose.yml up -d -docker-compose -f up -d +docker-compose -f compose/debug.docker-compose.yml up -d