docker-compose.yml

version: '3'

x-spark-common: &spark-common
  image: bitnami/spark:3.2.4
  volumes:
    - ./jobs:/opt/bitnami/spark/jobs
  command: bin/spark-class org.apache.spark.deploy.worker.Worker ${SPARK_MASTER_URL}
  depends_on:
    - spark-master
  environment:
    SPARK_MODE: worker
    SPARK_WORKER_CORES: ${SPARK_WORKER_CORES}
    SPARK_WORKER_MEMORY: ${SPARK_WORKER_MEMORY}
    SPARK_MASTER_URL: ${SPARK_MASTER_URL}
  networks:
    - app-network

x-python-streamer: &python-streamer-common
  image: python:3.10-slim
  volumes:
    - ./jobs:/app/jobs
    - ./data:/app/data
    - ./requirements.txt:/app/requirements.txt
  working_dir: /app
  env_file:
    - .env
  networks:
    - app-network
  depends_on:
    - broker

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.4.0
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: ${ZOOKEEPER_CLIENT_PORT}
      ZOOKEEPER_TICK_TIME: ${ZOOKEEPER_TICK_TIME}
    healthcheck:
      test: [ 'CMD', 'bash', '-c', "echo 'ruok' | nc localhost ${ZOOKEEPER_CLIENT_PORT}" ]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - app-network

  broker:
    image: confluentinc/cp-server:7.4.0
    hostname: broker
    container_name: broker
    depends_on:
      zookeeper:
        condition: service_healthy
    ports:
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: ${KAFKA_BROKER_ID}
      KAFKA_ZOOKEEPER_CONNECT: ${KAFKA_ZOOKEEPER_CONNECT}
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: ${KAFKA_LISTENER_SECURITY_PROTOCOL_MAP}
      KAFKA_ADVERTISED_LISTENERS: ${KAFKA_ADVERTISED_LISTENERS}
      KAFKA_METRIC_REPORTERS: ${KAFKA_METRIC_REPORTERS}
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: ${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR}
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: ${KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS}
      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: ${KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR}
      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: ${KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR}
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: ${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR}
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: ${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR}
      KAFKA_JMX_PORT: ${KAFKA_JMX_PORT}
      KAFKA_JMX_HOSTNAME: ${KAFKA_JMX_HOSTNAME}
      KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: ${KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL}
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: ${CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS}
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: ${CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS}
      CONFLUENT_METRICS_ENABLE: ${CONFLUENT_METRICS_ENABLE}
      CONFLUENT_SUPPORT_CUSTOMER_ID: ${CONFLUENT_SUPPORT_CUSTOMER_ID}
    healthcheck:
      test: [ 'CMD', 'bash', '-c', "nc -z localhost 9092" ]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - app-network
    restart: on-failure

  spark-master:
    image: bitnami/spark:3.2.4
    volumes:
      - ./jobs:/opt/bitnami/spark/jobs
    command: bin/spark-class org.apache.spark.deploy.master.Master
    ports:
      - "9090:8080"
      - "7077:7077"
    networks:
      - app-network

  twitter-streamer-1:
    <<: *python-streamer-common
    command: /bin/sh -c "pwd && python -m venv venv && cat requirements.txt && venv/bin/pip install -r requirements.txt && venv/bin/python jobs/stream_twitter.py"

  reddit-streamer-1:
    <<: *python-streamer-common
    command: /bin/sh -c "pwd && python -m venv venv && venv/bin/pip install -r requirements.txt && venv/bin/python jobs/stream_reddit.py"

  spark-worker-1:
    <<: *spark-common

  spark-worker-2:
    <<: *spark-common

  flask-backend:
    image: python:3.10-slim
    container_name: flask-backend
    volumes:
      - ./flask-dashboard:/app/flask-dashboard
    command: /bin/sh -c "pip install flask && pip install redis && pip install Flask-Cors && ls && python app/flask-dashboard/app.py"
    ports:
      - "5000:5000"
    networks:
      - app-network

  minio:
    image: minio/minio:latest
    container_name: minio
    ports:
      - "9000:9000" # web console/API
      - "9001:9001" # web admin console
    environment:
      MINIO_ROOT_USER: admin
      MINIO_ROOT_PASSWORD: password
    command: server /data --console-address ":9001"
    volumes:
      - ./minio/data:/data
    networks:
      - app-network
    healthcheck:
      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
      interval: 30s
      timeout: 10s
      retries: 3

  redis:
    image: redis:latest
    ports:
      - "6379:6379"
    volumes:
      - ./redis:/data
    command: [ "redis-server", "--appendonly", "yes" ]
    restart: always
    networks:
      - app-network

networks:
  app-network: