Skip to content

Commit

Permalink
controller engine: stop building image
Browse files Browse the repository at this point in the history
As we orthogonalized the exector and the rest of airflow components,
there is no need for a dedicated airflow build anymore.

Requirements are to be installed in the dask executor work instances
from now.
  • Loading branch information
cherusk committed Nov 15, 2023
1 parent 6fc48c4 commit 314e837
Show file tree
Hide file tree
Showing 2 changed files with 5 additions and 10 deletions.
3 changes: 0 additions & 3 deletions Dockerfile-airflow

This file was deleted.

12 changes: 5 additions & 7 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@
version: '3.4'
services:
control_loop:
build:
context: ./
dockerfile: ./Dockerfile-airflow
image: apache/airflow:2.4.3-python3.9
restart: always
environment:
- AIRFLOW__CORE__ENABLE_XCOM_PICKLING=true
Expand All @@ -32,7 +30,7 @@ services:
- AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=False
- AIRFLOW__API__AUTH_BACKENDS=airflow.api.auth.backend.basic_auth
- AIRFLOW__API__AUTH_BACKEND=airflow.api.auth.backend.basic_auth
- AIRFLOW__DASK_CLUSTER_ADDRESS=tcp://dask_scheduler_airflow:8786
- AIRFLOW__DASK_CLUSTER_ADDRESS=tcp://dask_scheduler_airflow:7786
- 'AIRFLOW_CONN_LINUX_NETWORK_STACK_BREEDER_SSH={ "conn_type": "ssh", "login": "godon_robot", "host": "10.0.5.53", "port": 22, "extra": { "key_file": "/opt/airflow/credentials/id_rsa" } }'
- ARCHIVE_DB_USER=yugabyte
- ARCHIVE_DB_PASSWORD=yugabyte
Expand Down Expand Up @@ -125,14 +123,14 @@ services:
dockerfile: ./Dockerfile-dask
hostname: dask-scheduler
ports:
- "8786:8786"
- "8787:8787"
- "7786:8786"
- "7787:8787"
command: ["dask-scheduler"]
dask_worker_airflow:
build:
context: ./
dockerfile: ./Dockerfile-dask
command: ["dask_scheduler_airflow", "tcp://dask_scheduler:8786"]
command: ["dask-worker", "tcp://dask_scheduler_airflow:7786"]
deploy:
replicas: 2
# for optuna parallel metaheuristics execution on dask
Expand Down

0 comments on commit 314e837

Please sign in to comment.