docker-compose.yml

---
version: '3'
services:

  # For better performance (and fewer headaches) comment out any services you aren't using.
  # For services that do run, also consider commenting out
  # - persistent storage
  # - external port mapping
  # as these are frequent sources of contention and random errors.


  ###
  # Base Kafka services
  ###

  # Apache zookeeper does what now?
  zookeeper:
    image: confluentinc/cp-zookeeper:6.1.1
    hostname: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  # The kafka broker
  broker:
    image: confluentinc/cp-server:6.1.1
    hostname: broker
    depends_on:
      - zookeeper
    ports:
      - "19092:19092"
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,LOCAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://broker:9092, LOCAL://localhost:19092
      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      CONFLUENT_METRICS_ENABLE: 'true'
      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'

  # Registry for kafka message schemas
  schema-registry:
    image: confluentinc/cp-schema-registry:6.1.1
    hostname: schema-registry
    depends_on:
      - broker
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081


 

  ###
  # Kafka UIs (landoop)
  ###

  # RESTful proxy for kafka
  rest-proxy:
    image: confluentinc/cp-kafka-rest:6.1.1
    hostname: rest-proxy
    depends_on:
      - broker
      - schema-registry
    ports:
      - 8082:8082
    environment:
      KAFKA_REST_HOST_NAME: rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'
      KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
      KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'

  # viewable at localhost:8093
  landoop-topics-ui:
    image: landoop/kafka-topics-ui
    ports:
      - 8093:8000
    depends_on:
      - rest-proxy
    environment:
      KAFKA_REST_PROXY_URL: http://rest-proxy:8082
      PROXY: 'true'
      
  # viewable at localhost:8094
  landoop-schema-ui:
    image: landoop/schema-registry-ui
    ports:
      - 8094:8000
    depends_on:
      - rest-proxy
      - schema-registry
    environment:
      SCHEMAREGISTRY_URL: "http://schema-registry:8081"
      PROXY: 'true'

  ###
  # Example producers
  ###

  # Example producer for strainmeter bottle files.  contains 1 Day, Hour, and Min file, which it reads, parses, and produces to the kafka topic gtsm_etl
  strain-producer:
    image: unavdocker/strain_bottle_producer_prototype:latest
    hostname: strain-producer
    depends_on:
      - broker
      - schema-registry
    environment:
      BOOTSTRAP_SERVERS: "broker:9092"
      SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
  
#  # Example producer for ggkx (gnss real time positions).  connects to station 
#  ggkx-producer:
#    image: unavdocker/ccp_ggkx_producer:latest
#    hostname: ggkx-producer
#    restart: unless-stopped
#    depends_on:
#      - broker
#      - schema-registry
#    environment:
#      BOOTSTRAP_SERVERS: "broker:9092"
#      SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
#  
#  # Example producer, connects to rtserve and pulls all stations data for HL network, produces as kafka messages to topic binarypackets
#  slinkproducer:
#    image: unavdocker/slinkproducer:v1
#    hostname: slinkproducer
#    container_name: slinkproducer-kafka-2
#    depends_on:
#      - broker
#      - schema-registry
#      - control-center
#    ports:
#      - "3000:3000"
#      - "18000:18000"
#    environment:
#      BROKER_HOSTS: "broker:9092"
#      REGISTRY_URLS: "http://schema-registry:8081"
#      TOPIC_NAME: 'binarypackets'
#      RUN_MODE: 'P'
#      DATA_SELECTION: 'HL_*'
#      DATA_SERVER: 'rtserve.iris.washington.edu:18000'
#      DATA_LOCAL_FOLDER: 'data_producer_kafka_2'
#      START_SLEEP_SECONDS: 90
#  #  volumes:
#  #    - ${HOST_PARENT_FOLDER}/data_producer_kafka_2:/root/slinkkafka/data_producer_kafka_2
  
  
  ###
  # Storage services
  ###

#  # Postgres database
#  postgres:
#    image: postgres
#    volumes:
#      - postgres-data:/var/lib/postgresql/data
#    env_file:
#      # see https://docs.docker.com/compose/environment-variables/#the-env_file-configuration-option
#      - ./docker.env
#    # ports:
#    #   - 8432:5432
#
#  # Redis key/value store
#  redis:
#    image: redis
#    # ports:
#    #   - 6379:6379
#
#  # TimescaleDB
#  timescaledb:
#    image: timescale/timescaledb:latest-pg12
#    environment:
#        POSTGRES_PASSWORD: "password"
#    # ports:
#    #     - "9432:5432"

  localstack:
    image: localstack/localstack
    ports:
      - 4566:4566
    environment:
      - SERVICES=lambda,s3
      - DEBUG=1
      - DATA_DIR=/tmp/localstack/data
      - LAMBDA_EXECUTOR=local
      - LOCALSTACK_API_KEY=key123
      - DOCKER_HOST=unix:///var/run/docker.sock
      - HOST_TMP_FOLDER="${TMPDIR:-/tmp}/localstack"
    volumes:
      - "${TMPDIR:-/tmp}/localstack:/tmp/localstack"
      - "/var/run/docker.sock:/var/run/docker.sock"


  ###
  # Web stack
  ###

#  # Web front-end
#  web:
#    image: nginx
#    volumes:
#      # Define a base nginx path in our subdirectory
#      - ./nginx:/etc/nginx
#      # Map the static subpath to a volume
#      - web-static:/etc/nginx/html/static
#    ports:
#      # The main web interface
#      - 8080:80
#      # HTTPS
#      - 8443:443
#      # Super simple nginx server
#      # - 8181:81
#    depends_on:
#      - django-web
#
#  # Vouch-proxy (CILogon integration with nginx)
#  vouch-proxy:
#    image: voucher/vouch-proxy:latest
#    volumes:
#      - ./vouch-proxy:/config
#    # ports:
#    #   - 9090:9090
#
#
#  ###
#  # Example python applications
#  ###
#
#  # Example web interfaces (collection and distribution)
#  django-web:
#    # GitHub repository image
#    image: ghcr.io/earthscope/es-example-django:main
#    command: gunicorn -c ./gunicorn.conf.py
#    volumes:
#      # AVRO schemas
#      - ./avro_schemas:/avro_schemas:ro
#      # Static files area
#      - web-static:/web-static
#    env_file:
#      # see https://docs.docker.com/compose/environment-variables/#the-env_file-configuration-option
#      - ./docker.env
#    # Local env on top of env_file
#    environment:
#      - DJANGO_PROMETHEUS=1
#    ports:
#      - 8100:8000
#      - 9100:9000
#
#  # Example data archiver
#  django-archiver:
#    # GitHub repository image
#    image: ghcr.io/earthscope/es-example-django:main
#    command: python manage.py run_kafka_consumer
#    depends_on:
#      - broker
#      - schema-registry
#    volumes:
#      # AVRO schemas
#      - ./avro_schemas:/avro_schemas:ro
#    env_file:
#      # see https://docs.docker.com/compose/environment-variables/#the-env_file-configuration-option
#      - ./docker.env
#
## Volumes are local by default, but could be backed by test/prod datasources
#volumes:
#  postgres-data:
#  web-static:
#  prometheus:
#  grafana:

  ###
  # Apache Flink Web stack
  ###
  sql-client:
    image: flink:1.14.2-scala_2.12-java8
    #   to make a shell available.
    #   we are not starting any service
    command: /bin/bash
    stdin_open: true # docker run -i
    tty: true        # docker run -t
    depends_on:
      - broker
      - jobmanager
#      - elasticsearch
    environment:
      FLINK_JOBMANAGER_HOST: jobmanager
      FLINK_JOBMANAGER_REST_PORT: 8083
      ZOOKEEPER_CONNECT: zookeeper
      KAFKA_BOOTSTRAP: broker
      ES_HOST: elasticsearch
    volumes:
      - ./flink/update_flink_install.sh:/opt/flink/update_flink_install.sh

  jobmanager:
    image: flink:1.14.2-scala_2.12-java8
    ports:
      - "8083:8083"
    command: jobmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        # added to bind to a different port
        jobmanager.web.port: 8083

  taskmanager:
    image: flink:1.14.2-scala_2.12-java8
    depends_on:
      - jobmanager
    command: taskmanager
    volumes:
      - ${PWD}/flink/flink-taskmanager-log4j-console.properties:/opt/flink/conf/log4j-console.properties
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        taskmanager.numberOfTaskSlots: 10

#  elasticsearch:
#    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.0
#    environment:
#      - cluster.name=docker-cluster
#      - bootstrap.memory_lock=true
#      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
#      - discovery.type=single-node
#    ports:
#      - "9200:9200"
#      - "9300:9300"
#    ulimits:
#      memlock:
#        soft: -1
#        hard: -1
#      nofile:
#        soft: 65536
#        hard: 65536
#
#  kibana:
#    image: docker.elastic.co/kibana/kibana:7.6.0
#    ports:
#      - "5601:5601"