docker-compose.yml

---
version: '2'
services:
  namenode:
    image: apache/hadoop:3
    hostname: namenode
    command: ["hdfs", "namenode"]
    depends_on:
      - resourcemanager
      - nodemanager
    ports:
      - 9870:9870
    env_file:
      - ./config-hadoop
    environment:
      ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
    networks:
      - kafka-network
    volumes:
      - hadoop:/home/
    restart: on-failure

  datanode:
    image: apache/hadoop:3
    command: ["hdfs", "datanode"]
    depends_on:
      - namenode
    env_file:
      - ./config-hadoop
    networks:
      - kafka-network
    volumes:
      - hadoop:/home/
    restart: on-failure

  resourcemanager:
    image: apache/hadoop:3
    hostname: resourcemanager
    command: ["yarn", "resourcemanager"]
    ports:
      - 8088:8088
    env_file:
      - ./config-hadoop
    volumes:
      - ./test.sh:/opt/test.sh
    networks:
      - kafka-network
    restart: on-failure

  nodemanager:
    image: apache/hadoop:3
    hostname: nodemanager
    command: ["yarn", "nodemanager"]
    env_file:
      - ./config-hadoop
    networks:
      - kafka-network
    volumes:
      - hadoop:/home/
    restart: on-failure

  zookeeper:
    container_name: zookeeper
    image: confluentinc/cp-zookeeper:5.0.0
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
    networks:
      - kafka-network
    volumes:
      - hadoop:/home/
    restart: on-failure


  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    ports:
      - 8080:8080
    environment:
      KAFKA_CLUSTERS_0_NAME: kafka
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092
      DYNAMIC_CONFIG_ENABLED: 'true'
    networks:
      - kafka-network

  kafka:
    hostname: kafka
    container_name: kafka
    image: confluentinc/cp-kafka:5.0.0
    depends_on:
      - zookeeper
    networks:
      - kafka-network
    ports:
      - "9092:9092"
      - "29092:29092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://kafka:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
    volumes:
      - hadoop:/home/
    restart: on-failure

  producer:
    container_name: producer
    build: ./docker/producer
    volumes:
      - .:/code
    depends_on:
      - kafka
    networks:
      - kafka-network
    restart: on-failure

  consumer:
    container_name: consumer
    build: ./docker/consumer
    volumes:
      - .:/code
    depends_on:
      - kafka
    networks:
      - kafka-network
    restart: on-failure

networks:
  kafka-network:
    external: true

volumes:
  hadoop: