docker-compose.yml

version: '2' 
services:
  namenode:
    image: bde2020/hadoop-namenode:1.1.0-hadoop2.8-java8
    container_name: namenode
    volumes:
      - ./data/namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop-hive.env
    ports:
      - 50070:50070
      - 8020:8020
    networks: 
      - bde-net

  resourcemanager:
    image: bde2020/hadoop-resourcemanager:1.1.0-hadoop2.8-java8
    container_name: resourcemanager
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop-hive.env
    ports:
      - 8088:8088
    networks: 
      - bde-net

  historyserver:
    image: bde2020/hadoop-historyserver:1.1.0-hadoop2.8-java8
    container_name: historyserver
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop-hive.env
    ports:
      - 8188:8188
    networks: 
      - bde-net

  datanode:
    image: bde2020/hadoop-datanode:1.1.0-hadoop2.8-java8
    depends_on: 
      - namenode
    volumes:
      - ./data/datanode:/hadoop/dfs/data
    env_file:
      - ./hadoop-hive.env
    ports:
      - 50075:50075
    networks: 
      - bde-net

  nodemanager:
    image: bde2020/hadoop-nodemanager:1.1.0-hadoop2.8-java8
    container_name: nodemanager
    hostname: nodemanager
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop-hive.env
    ports:
      - 8042:8042
    networks: 
      - bde-net

  hive-server:
    # image: bde2020/hive:2.1.0-postgresql-metastore
    image: bde2020/hive:2.3.2-postgresql-metastore
    container_name: hive-server
    env_file:
      - ./hadoop-hive.env
    environment:
      - "HIVE_CORE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore/metastore"
    ports:
      - "10000:10000"
    networks: 
      - bde-net

  hive-metastore:
    # image: bde2020/hive:2.1.0-postgresql-metastore
    image: bde2020/hive:2.3.2-postgresql-metastore
    container_name: hive-metastore
    env_file:
      - ./hadoop-hive.env
    command: /opt/hive/bin/hive --service metastore
    ports:
      - 9083:9083
    networks: 
      - bde-net

  hive-metastore-postgresql:
    # image: bde2020/hive-metastore-postgresql:2.1.0
    image: bde2020/hive-metastore-postgresql:2.3.0
    ports:
      - 5432:5432
    volumes:
      - ./data/postgresql/:/var/lib/postgresql/data
    networks: 
      - bde-net

  spark-master:
    image: bde2020/spark-master:2.1.0-hadoop2.8-hive-java8
    container_name: spark-master
    hostname: spark-master
    volumes:
      - ./copy-jar.sh:/copy-jar.sh
    ports:
      - 8080:8080
      - 7077:7077
    env_file:
      - ./hadoop-hive.env
    networks: 
      - bde-net

  spark-worker:
    image: bde2020/spark-worker:2.1.0-hadoop2.8-hive-java8
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
    ports:
      - "8081:8081"
    env_file:
      - ./hadoop-hive.env
    networks: 
      - bde-net
    
  mysql-server:
    image: mysql:5.7
    container_name: mysql-server
    ports:
      - "3306:3306"
    environment:
      - MYSQL_ROOT_PASSWORD=zhangyang517
    volumes:
      - ./data/mysql:/var/lib/mysql
    networks: 
      - bde-net

  # elasticsearch:
  #   image: elasticsearch:6.5.3
  #   environment:
  #     - discovery.type=single-node
  #   ports:
  #     - "9200:9200"
  #     - "9300:9300"
  #   networks: 
  #     - bde-net

  # kibana:
  #   image: kibana:6.5.3
  #   ports:
  #     - "5601:5601"
  #   networks: 
  #     - bde-net

networks:
  bde-net:
    external: true