docker-compose.yml

version: '3'

services:
  spark-master: # Spark Master
    container_name: spark-master
    image: docker.io/bitnami/spark:3.2.1
    environment:
      - SPARK_MODE=${SPARK_MODE}
      - SPARK_RPC_AUTHENTICATION_ENABLED=${SPARK_RPC_AUTHENTICATION_ENABLED} # Enable RPC authentication. Default: no
      - SPARK_RPC_ENCRYPTION_ENABLED=${SPARK_RPC_ENCRYPTION_ENABLED} # Enable RPC encryption. Default: no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=${SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED} # Enable local storage encryption: Default no
      - SPARK_SSL_ENABLED=${SPARK_SSL_ENABLED} # Enable SSL configuration. Default: no
    ports:
      - '8080:8080'
      - '7077:7077'
    volumes:
      - type: bind # For disable log only
        source: ./conf/log4j.properties # For disable log only
        target: /opt/bitnami/spark/conf/log4j.properties # For disable log only
      - ".:/opt/bitnami/spark/source_code:rw" # Name volume for container to access the local root of this project
    networks:
      - spark_nw
  spark-worker: # Spark Worker
    container_name: spark-worker
    image: docker.io/bitnami/spark:3.2.1
    environment:
      - SPARK_MODE=worker # Cluster mode starting Apache Spark. Valid values: master, worker. Default: master
      - SPARK_MASTER_URL=${SPARK_MASTER_URL} # Url where the worker can find the master. Only needed when spark mode is worker. Default: spark://spark-master:7077
      - SPARK_WORKER_MEMORY=${SPARK_WORKER_MEMORY}
      - SPARK_WORKER_CORES=${SPARK_WORKER_CORES}
      - SPARK_RPC_AUTHENTICATION_ENABLED=${SPARK_RPC_AUTHENTICATION_ENABLED} # Enable RPC authentication. Default: no
      - SPARK_RPC_ENCRYPTION_ENABLED=${SPARK_RPC_ENCRYPTION_ENABLED} # Enable RPC encryption. Default: no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=${SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED} # Enable local storage encryption: Default no
      - SPARK_SSL_ENABLED=${SPARK_SSL_ENABLED} # Enable SSL configuration. Default: no
    volumes:
      - type: bind # For disable log only
        source: ./conf/log4j.properties # For disable log only
        target: /opt/bitnami/spark/conf/log4j.properties # For disable log only
    networks:
      - spark_nw
    depends_on:
      - spark-master

networks:
  spark_nw:
    driver: bridge