diff --git a/docker-compose.yml b/docker-compose.yml index 33311619..ff7c07fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,12 +26,13 @@ services: environment: - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true - AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=True - - AIRFLOW__CORE__EXECUTOR=LocalExecutor + - AIRFLOW__CORE__EXECUTOR=DaskExecutor - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@db/airflow - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@db/airflow - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=False - AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth - AIRFLOW__API__AUTH_BACKEND=airflow.api.auth.backend.basic_auth + - AIRFLOW__DASK_CLUSTER_ADDRESS=tcp://dask_scheduler_airflow:8786 - 'AIRFLOW_CONN_LINUX_NETWORK_STACK_BREEDER_SSH={ "conn_type": "ssh", "login": "godon_robot", "host": "10.0.5.53", "port": 22, "extra": { "key_file": "/opt/airflow/credentials/id_rsa" } }' - ARCHIVE_DB_USER=yugabyte - ARCHIVE_DB_PASSWORD=yugabyte @@ -117,6 +118,23 @@ services: ports: - 127.0.0.1:4222:4222 - 127.0.0.1:8222:8222 + # for airflow engine + dask_scheduler_airflow: + build: + context: ./ + dockerfile: ./Dockerfile-dask + hostname: dask-scheduler + ports: + - "8786:8786" + - "8787:8787" + command: ["dask-scheduler"] + dask_worker_airflow: + build: + context: ./ + dockerfile: ./Dockerfile-dask + command: ["dask_scheduler_airflow", "tcp://dask_scheduler:8786"] + deploy: + replicas: 2 # for optuna parallel metaheuristics execution on dask dask_scheduler: build: