docker-compose.yaml

version: "3"

services:
  airflow:
    build: ./airflow
    restart: unless-stopped
    environment:
      - AIRFLOW__CORE__FERNET_KEY=PeszOD9t6wKNdNONTGh-k4qsYbsGHQ8CSqTQZlqPCDk=
      - AIRFLOW_HOME=/opt/airflow
    ports:
      - 8000:8000
    volumes:
      - ./airflow/dags:/opt/airflow/dags
    command: sleep infinity

  spark-master:
    build: ./spark
    restart: unless-stopped
    ports:
      - 4040:4040
      - 7077:7077
      - 8080:8080
      - 18080:18080
    command: sh spark-master.sh

  spark-worker:
    build: ./spark
    restart: unless-stopped
    ports:
      - "8081-8085:8081"
    command: sh spark-worker.sh

  minio:
    image: "minio/minio:latest"
    volumes:
      - ./.storage/minio:/data
    ports:
      - 9000:9000
      - 9001:9001
    environment:
      MINIO_ACCESS_KEY: "root"
      MINIO_SECRET_KEY: "root12345"
    command: server /data --console-address ":9001"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  mariadb:
    image: mariadb:latest
    # image: mariadb/columnstore:latest
    ports:
      - 3306:3306
    environment:
      USE_S3_STORAGE: 0
      MYSQL_ROOT_PASSWORD: root
      MYSQL_USER: admin
      MYSQL_PASSWORD: admin
      MYSQL_DATABASE: metastore_db

  hive:
    build: ./hive
    restart: unless-stopped
    ports:
      - 9083:9083

  presto:
    build: ./presto
    ports:
      - "8888:8888"
    volumes:
      - ./presto/coordinator_config.properties:/opt/presto/etc/config.properties
      - ./presto/etc/catalog:/opt/presto/etc/catalog

  presto-worker:
    build: ./presto
    volumes:
      - ./presto/worker_config.properties:/opt/presto/etc/config.properties
      - ./presto/etc/catalog:/opt/presto/etc/catalog

  superset:
    image: arezamoosavi/superset:latest
    container_name: superset
    environment:
      ADMIN_USERNAME: admin
      ADMIN_PWD: admin
    ports:
      - 8088:8088
    volumes:
      - ./superset:/superset
    restart: unless-stopped