control loop: run on dask

Use separate dask worker instances and schedulers for distributed airflow task execution.
godon-dev · Nov 15, 2023 · 6fc48c4 · 6fc48c4
1 parent 79d5903
commit 6fc48c4
Showing 1 changed file with 19 additions and 1 deletion.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -26,12 +26,13 @@ services:
         environment:
             - AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
             - AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=True
-            - AIRFLOW__CORE__EXECUTOR=LocalExecutor
+            - AIRFLOW__CORE__EXECUTOR=DaskExecutor
             - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@db/airflow
             - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:airflow@db/airflow
             - AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=False
             - AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth
             - AIRFLOW__API__AUTH_BACKEND=airflow.api.auth.backend.basic_auth
+            - AIRFLOW__DASK_CLUSTER_ADDRESS=tcp://dask_scheduler_airflow:8786
             - 'AIRFLOW_CONN_LINUX_NETWORK_STACK_BREEDER_SSH={ "conn_type": "ssh", "login": "godon_robot", "host": "10.0.5.53", "port": 22, "extra": { "key_file": "/opt/airflow/credentials/id_rsa" } }'
             - ARCHIVE_DB_USER=yugabyte
             - ARCHIVE_DB_PASSWORD=yugabyte
@@ -117,6 +118,23 @@ services:
         ports:
           - 127.0.0.1:4222:4222
           - 127.0.0.1:8222:8222
+    # for airflow engine
+    dask_scheduler_airflow:
+      build:
+          context: ./
+          dockerfile: ./Dockerfile-dask
+      hostname: dask-scheduler
+      ports:
+        - "8786:8786"
+        - "8787:8787"
+      command: ["dask-scheduler"]
+    dask_worker_airflow:
+      build:
+        context: ./
+        dockerfile: ./Dockerfile-dask
+      command: ["dask_scheduler_airflow", "tcp://dask_scheduler:8786"]
+      deploy:
+        replicas: 2
     # for optuna parallel metaheuristics execution on dask
     dask_scheduler:
       build: