diff --git a/README.md b/README.md index 6f720517..d47f7e0b 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,13 @@ If you wish to use multi-line YAML or some other delimiter between your topic de For example, `KAFKA_CREATE_TOPICS_SEPARATOR: "$$'\n"'` would use a newline to split the topic definitions. Syntax has to follow docker-compose escaping rules, and [ANSI-C](https://www.gnu.org/software/bash/manual/html_node/ANSI_002dC-Quoting.html) quoting. +Another environment variable can be added to perform further configuration changes: ```KAFKA_CONFIGS```. For example: + + environment: + - KAFKA_CONFIGS=topics:Topic1:add:retention.ms=604800000 cleanup.policy=delete segment.bytes=1024 + +The format for this variable mirrors the ```kafka-configs.sh``` script. For multi-line YAML a separate delimiter variable ```KAFKA_CREATE_TOPICS_SEPARATOR``` is available. + ## Advertised hostname You can configure the advertised hostname in different ways diff --git a/create-topics.sh b/create-topics.sh index 9d89540c..8e72afc0 100755 --- a/create-topics.sh +++ b/create-topics.sh @@ -1,9 +1,5 @@ #!/bin/bash -if [[ -z "$KAFKA_CREATE_TOPICS" ]]; then - exit 0 -fi - if [[ -z "$START_TIMEOUT" ]]; then START_TIMEOUT=600 fi @@ -26,24 +22,45 @@ if $start_timeout_exceeded; then exit 1 fi -# Expected format: -# name:partitions:replicas:cleanup.policy -IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOPICS; do - echo "creating topics: $topicToCreate" - IFS=':' read -r -a topicConfig <<< "$topicToCreate" - config= - if [ -n "${topicConfig[3]}" ]; then - config="--config=cleanup.policy=${topicConfig[3]}" - fi - COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\ - --create \\ - --zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\ - --topic ${topicConfig[0]} \\ - --partitions ${topicConfig[1]} \\ - --replication-factor ${topicConfig[2]} \\ - ${config} \\ - --if-not-exists &" - eval "${COMMAND}" -done +if [[ -n "$KAFKA_CREATE_TOPICS" ]]; then + # Expected format: + # name:partitions:replicas:cleanup.policy + IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOPICS; do + echo "creating topics: $topicToCreate" + IFS=':' read -r -a topicConfig <<< "$topicToCreate" + config= + if [ -n "${topicConfig[3]}" ]; then + config="--config=cleanup.policy=${topicConfig[3]}" + fi + COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\ + --create \\ + --zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\ + --topic ${topicConfig[0]} \\ + --partitions ${topicConfig[1]} \\ + --replication-factor ${topicConfig[2]} \\ + ${config} \\ + --if-not-exists &" + eval "${COMMAND}" + done +fi + +wait + +if [[ -n "$KAFKA_CONFIGS" ]]; then + # Expected format: + # type:name:add|delete:a=b c=d + IFS="${KAFKA_CONFIG_SEPARATOR-,}"; for configToChange in $KAFKA_CONFIGS; do + echo "changing config: $configToChange" + IFS=':' read -r -a entityConfig <<< "$configToChange" + config="${entityConfig[3]}" + COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-configs.sh \\ + --zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\ + --entity-type ${entityConfig[0]} \\ + --entity-name ${entityConfig[1]} \\ + --alter \\ + --${entityConfig[2]}-config ${config// /,} &" + eval "${COMMAND}" + done +fi wait diff --git a/start-kafka.sh b/start-kafka.sh index 51567abf..b3c9ed76 100755 --- a/start-kafka.sh +++ b/start-kafka.sh @@ -21,6 +21,7 @@ fi create-topics.sh & unset KAFKA_CREATE_TOPICS +unset KAFKA_CONFIGS # DEPRECATED: but maintained for compatibility with older brokers pre 0.9.0 (https://issues.apache.org/jira/browse/KAFKA-1809) if [[ -z "$KAFKA_ADVERTISED_PORT" && \