docker-compose.yml

version: "3"
x-airflow-common: &airflow-common
  # image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.4.0}
  build: .
  environment: &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: CeleryExecutor
    AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
    AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
    AIRFLOW__CORE__FERNET_KEY: ""
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: "true"
    AIRFLOW__CORE__LOAD_EXAMPLES: "false"
    AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth"
    

   
  volumes:
    - ./dags:/opt/airflow/dags
    - ./logs:/opt/airflow/logs
    - ./plugins:/opt/airflow/plugins
    - /var/run/docker.sock:/var/run/docker.sock
  user: "${AIRFLOW_UID:-$(id -u)}:${AIRFLOW_GID:-0}"
  # user: "${AIRFLOW_UID}:50000"
  depends_on:
    redis:
      condition: service_healthy
    postgres:
      condition: service_healthy

services:
  postgres:
    image: postgres:10
    volumes:
      - ./server/db:/opt/db_storage
    ports:
      - "5430:5430"
    environment:
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_USER=airflow
      - POSTGRES_HOST_AUTH_METHOD=trust
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "airflow"]
      interval: 5s
      retries: 5
    restart: always

  redis:
    image: redis:5.0.3
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 30s
      retries: 50
    restart: always

  webserver:
    <<: *airflow-common
    command: webserver
    ports:
      - "8080:8080"
    healthcheck:
      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
      interval: 30s
      timeout: 30s
      retries: 3
    # image: apache/airflow:2.4.0
    restart: always
    depends_on:
      - postgres
      - redis

  airflow-init:
    <<: *airflow-common
    command: version
      # - |
      #   chown -R "${AIRFLOW_UID}:0" /opt/airflow/{logs,dags,plugins}
      
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_UPGRADE: "true"
      _AIRFLOW_WWW_USER_CREATE: "true"
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}

  flower:
    <<: *airflow-common
    depends_on:
      - postgres
      - redis
    ports:
      - "5555:5555"
    command: celery flower
  scheduler:
    <<: *airflow-common
    # image: apache/airflow:2.4.0
    depends_on:
      - webserver
    command: scheduler
  worker:
    <<: *airflow-common
    # image: apache/airflow:2.4.0
    depends_on:
      - scheduler
    ports:
      - "7171:7171"
      - "7272:7272"
    #  - "3002:3002"
    #  - "3003:3003"
    #  - "3004:3004"

    build:
      context: .
      dockerfile: ./Dockerfile
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]

    command: celery worker