docker-compose.yml

version: "3.7"
        
volumes:
# Monitoring
  prometheus_data: {}
  grafana_data: {}
# Druid
  metadata_data: {}
  druid_middle_var: {}
  historical_var: {}
  druid_broker_var: {}
  druid_coordinator_var: {}
  druid_router_var: {}
# Superset
  superset_home:
  superset_node_modules:
  superset_db:
  redis:
# JuoyteHub
  jupyterhub-data:
  jupyterhub-db:
    
# Superset
x-superset-build: &superset-build
  args:
      NPM_BUILD_CMD: build-dev
  context: ./
  dockerfile: Dockerfile
  target: dev
x-superset-volumes: &superset-volumes
  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
  - ./incubator-superset/docker/docker-init.sh:/app/docker-init.sh
  - ./incubator-superset/docker/pythonpath_dev:/app/pythonpath
  - ./incubator-superset/superset:/app/superset
  - ./incubator-superset/superset-frontend:/app/superset-frontend
  - superset_node_modules:/app/superset-frontend/superset_node_modules
  - superset_home:/app/superset_home

networks:
  monitor-net:

services:
# Monitoring
  prometheus:
    image: prom/prometheus:v2.17.2
    container_name: prometheus
    volumes:
      - ./prometheus:/etc/prometheus
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--storage.tsdb.retention.time=200h'
      - '--web.enable-lifecycle'
    restart: unless-stopped
    ports:
      - "9090:9090"
    networks:
      - monitor-net

  nodeexporter:
    image: prom/node-exporter:v0.18.1
    container_name: nodeexporter
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.rootfs=/rootfs'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
    restart: unless-stopped
    networks:
      - monitor-net

  cadvisor:
    image: gcr.io/google-containers/cadvisor:v0.36.0
    container_name: cadvisor
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:rw
      - /sys:/sys:ro
      - /var/lib/docker:/var/lib/docker:ro
      #- /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux
    restart: unless-stopped
    privileged: true
    networks:
      - monitor-net

  grafana:
    image: grafana/grafana:6.7.3
    container_name: grafana
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning:/etc/grafana/provisioning
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false
    restart: unless-stopped
    ports:
      - "3000:3000"
    networks:
      - monitor-net

  pushgateway:
    image: prom/pushgateway:v1.2.0
    container_name: pushgateway
    restart: unless-stopped
    networks:
      - monitor-net

# Zookeeper
  zookeeper:
    container_name: zookeeper
    image: zookeeper:3.6
    environment:
      - ZOO_MY_ID=1

# Druid
  druid_postgres:
    container_name: druid_postgres
    image: postgres:10
    volumes:
      - metadata_data:/var/lib/postgresql/data
    environment:
      - POSTGRES_PASSWORD=FoolishPassword
      - POSTGRES_USER=druid
      - POSTGRES_DB=druid

  druid_coordinator:
    image: apache/druid:0.17.0
    container_name: druid_coordinator
    volumes:
      - ./storage:/opt/data
      - druid_coordinator_var:/opt/druid/var
    depends_on: 
      - zookeeper
      - druid_postgres
    ports:
      - "8081:8081"
    command:
      - coordinator
    env_file:
      - druid.env

  druid_broker:
    image: apache/druid:0.17.0
    container_name: druid_broker
    volumes:
      - druid_broker_var:/opt/druid/var
    depends_on: 
      - zookeeper
      - druid_postgres
      - druid_coordinator
    ports:
      - "8082:8082"
    command:
      - broker
    env_file:
      - druid.env

  druid_historical:
    image: apache/druid:0.17.0
    container_name: druid_historical
    volumes:
      - ./storage:/opt/data
      - historical_var:/opt/druid/var
    depends_on: 
      - zookeeper
      - druid_postgres
      - druid_coordinator
    ports:
      - "8083:8083"
    command:
      - historical
    env_file:
      - druid.env

  druid_middlemanager:
    image: apache/druid:0.17.0
    container_name: druid_middlemanager
    volumes:
      - ./storage:/opt/data
      - druid_middle_var:/opt/druid/var
    depends_on: 
      - zookeeper
      - druid_postgres
      - druid_coordinator
    ports:
      - "8091:8091"
    command:
      - middleManager
    env_file:
      - druid.env

  druid_router:
    image: apache/druid:0.17.0
    container_name: druid_router
    volumes:
      - druid_router_var:/opt/druid/var
    depends_on:
      - zookeeper
      - druid_postgres
      - druid_coordinator
    ports:
      - "8888:8888"
    command:
      - router
    env_file:
      - druid.env

# Kafka
  kafka:
    image: wurstmeister/kafka:2.12-2.4.1
    container_name: kafka
    ports:
      - "9092:9092"
    environment:
      # KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    depends_on:
      - zookeeper
      - druid_broker
      - druid_coordinator

# NiFi
# Add https://github.com/hortonworks-gallery/nifi-templates
  nifi:
    image: apache/nifi:1.11.4
    ports:
      - 2000
    environment:
      - NIFI_WEB_HTTP_PORT=2000
      - NIFI_CLUSTER_IS_NODE=true
      - NIFI_CLUSTER_NODE_PROTOCOL_PORT=8082
      - NIFI_ZK_CONNECT_STRING=zookeeper:2181
      - NIFI_ELECTION_MAX_WAIT=1 min
    depends_on:
      - zookeeper
      - druid_broker
      - druid_coordinator

# Superset
  redis:
    image: redis:3.2
    container_name: superset_cache
    restart: unless-stopped
    volumes:
      - redis:/data
    depends_on:
      - druid_broker

  superset_db:
    env_file: superset.env
    image: postgres:10
    container_name: superset_db
    restart: unless-stopped
    volumes:
      - superset_db:/var/lib/postgresql/data
    depends_on:
      - druid_broker

  superset:
    env_file: superset.env
    build: *superset-build
    container_name: superset_app
    command: ["flask", "run", "-p", "8088", "--with-threads", "--reload", "--debugger", "--host=0.0.0.0"]
    restart: unless-stopped
    ports:
      - 8088:8088
    depends_on:
      - superset_db
      - redis
      - druid_broker
    volumes: *superset-volumes

  superset-init:
    build: *superset-build
    container_name: superset_init
    command: ["/app/docker-init.sh"]
    env_file: superset.env
    depends_on:
      - superset_db
      - redis
      - druid_broker
    volumes: *superset-volumes

  superset-node:
    image: node:10-jessie
    container_name: superset_node
    command: ["bash", "-c", "cd /app/superset-frontend && npm install --global webpack webpack-cli && npm install && npm run dev"]
    env_file: superset.env
    depends_on:
      - superset_db
      - redis
      - druid_broker
    volumes: *superset-volumes

  superset-worker:
    build: *superset-build
    container_name: superset_worker
    command: ["celery", "worker", "--app=superset.tasks.celery_app:app", "-Ofair"]
    env_file: superset.env
    restart: unless-stopped
    depends_on:
      - superset_db
      - redis
      - druid_broker
    volumes: *superset-volumes

# Zeppelin
  # namenode:
  #   image: bde2020/hadoop-namenode:1.1.0-hadoop2.8-java8
  #   container_name: namenode
  #   restart: unless-stopped
  #   volumes:
  #     - ./data/namenode:/hadoop/dfs/name
  #   environment:
  #     - CLUSTER_NAME=test
  #     - CORE_CONF_fs_defaultFS=hdfs://namenode:8020

  # datanode:
  #   image: bde2020/hadoop-datanode:1.1.0-hadoop2.8-java8
  #   container_name: datanode
  #   restart: unless-stopped
  #   volumes:
  #     - ./data/datanode:/hadoop/dfs/data
  #   environment:
  #     - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
  #   depends_on:
  #     - namenode

  # spark-master:
  #   image: bde2020/spark-master:2.1.0-hadoop2.8-hive-java8
  #   container_name: spark-master
  #   restart: unless-stopped
  #   # ports:
  #   #   - "8080:8080"
  #   #   - "7077:7077"
  #   environment:
  #     - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
  #   depends_on:
  #     - namenode
  #     - datanode

  # spark-worker:
  #   image: bde2020/spark-worker:2.1.0-hadoop2.8-hive-java8
  #   container_name: spark-worker
  #   restart: unless-stopped
  #   environment:
  #     - "SPARK_MASTER=spark://spark-master:7077"
  #   environment:
  #     - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
  #   depends_on:
  #     - spark-master

  zeppelin:
    image: apache/zeppelin:0.9.0
    container_name: zeppelin
    restart: unless-stopped
    ports:
      - 3001:8080
    volumes:
      - ./zeppelin:/opt/zeppelin/notebook
    # environment:
    #   CORE_CONF_fs_defaultFS: "hdfs://namenode:8020"
    #   SPARK_MASTER: "spark://spark-master:7077"
    #   MASTER: "spark://spark-master:7077"
    #   #SPARK_SUBMIT_OPTIONS: "--jars /opt/sansa-examples/jars/sansa-examples-spark-2016-12.jar"
    # depends_on:
    #   - spark-master
    #   - namenode

# JupyterHub
  jupyterhub:
    image: jupyterhub/jupyterhub:1.2
    restart: unless-stopped
    container_name: jupyterhub
    volumes:
      - "./jupyterlab:/data"
    ports:
      - "3002:8000"