docker-compose.yaml

version: '2.1'
services:
  notebook:
    image: qooba/flink:dev
    ports:
      - "8888:8888"
    depends_on:
      - kafka
      - jobmanager
      - taskmanager
    environment:
      JOB_MANAGER_RPC_ADDRESS: jobmanager
      ZOOKEEPER_CONNECT: zookeeper
      KAFKA_BOOTSTRAP: kafka
      SUBMIT_ARGS: remote -m jobmanager:8081
    volumes:
      - ./data/mlflow:/mlflow
      - ./notebooks:/opt/flink/notebooks
      - ./sql-client-defaults.yaml:/opt/flink/conf/sql-client-defaults.yaml
  mlflow:
    image: qooba/mlflow:dev
    ports:
      - "5000:5000"
    volumes:
      - ./data/mlflow:/mlflow
  # sql-client:
    # image: jark/demo-sql-client:0.2
    # depends_on:
    #   - kafka
    #   - jobmanager
    #   - elasticsearch
    # environment:
    #   FLINK_JOBMANAGER_HOST: jobmanager
    #   ZOOKEEPER_CONNECT: zookeeper
    #   KAFKA_BOOTSTRAP: kafka
    #   MYSQL_HOST: mysql
    #   ES_HOST: elasticsearch
  jobmanager:
    image: flink:1.12.2-scala_2.11
    ports:
      - "8081:8081"
      - "8080:8080"
    command: jobmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
  taskmanager:
    image: flink:1.12.2-scala_2.11
    depends_on:
      - jobmanager
    command: taskmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 10
  # datagen:
  #   image: jark/datagen:0.2
  #   command: "java -classpath /opt/datagen/flink-sql-demo.jar myflink.SourceGenerator --input /opt/datagen/user_behavior.log --output kafka kafka:9094 --speedup 2000"
  #   depends_on:
  #     - kafka
  #   environment:
  #     ZOOKEEPER_CONNECT: zookeeper
  #     KAFKA_BOOTSTRAP: kafka
  # mysql:
  #   image: jark/mysql-example:0.2
  #   ports:
  #     - "3306:3306"
  #   environment:
  #     - MYSQL_ROOT_PASSWORD=123456
  zookeeper:
    image: wurstmeister/zookeeper:3.4.6
    ports:
      - "2181:2181"
  kafka:
    image: wurstmeister/kafka:2.12-2.2.1
    ports:
      - "9092:9092"
      - "9094:9094"
    depends_on:
      - zookeeper
    environment:
      - KAFKA_ADVERTISED_LISTENERS=INSIDE://:9094,OUTSIDE://localhost:9092
      - KAFKA_LISTENERS=INSIDE://:9094,OUTSIDE://:9092
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_CREATE_TOPICS="user_behavior:1:1"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.0
    environment:
      - cluster.name=docker-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - discovery.type=single-node
    ports:
      - "9200:9200"
      - "9300:9300"
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
  kibana:
    image: docker.elastic.co/kibana/kibana:7.6.0
    ports:
      - "5601:5601"