From 914704d2a41d754dacd2379606f35c839194363c Mon Sep 17 00:00:00 2001
From: Ron Arts <ron.arts@novantum.com>
Date: Wed, 23 May 2018 05:23:26 +0200
Subject: [PATCH 1/4] Add KAFKA_CONFIGS variable to do additional configuration
 at creation time

---
 Dockerfile       |  4 +--
 README.md        |  7 +++++
 configs.sh       | 66 ++++++++++++++++++++++++++++++++++++++++++++++++
 create-topics.sh | 49 -----------------------------------
 start-kafka.sh   |  3 ++-
 5 files changed, 77 insertions(+), 52 deletions(-)
 create mode 100755 configs.sh
 delete mode 100755 create-topics.sh

diff --git a/Dockerfile b/Dockerfile
index 6740cc1a..d6e51c1b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,12 +13,12 @@ ENV KAFKA_VERSION=$kafka_version \
 
 ENV PATH=${PATH}:${KAFKA_HOME}/bin
 
-COPY download-kafka.sh start-kafka.sh broker-list.sh create-topics.sh /tmp/
+COPY download-kafka.sh start-kafka.sh broker-list.sh configs.sh /tmp/
 
 RUN apk add --no-cache bash curl jq docker \
  && mkdir /opt \
  && chmod a+x /tmp/*.sh \
- && mv /tmp/start-kafka.sh /tmp/broker-list.sh /tmp/create-topics.sh /usr/bin \
+ && mv /tmp/start-kafka.sh /tmp/broker-list.sh /tmp/configs.sh /usr/bin \
  && sync && /tmp/download-kafka.sh \
  && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt \
  && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \
diff --git a/README.md b/README.md
index 6f720517..d47f7e0b 100644
--- a/README.md
+++ b/README.md
@@ -76,6 +76,13 @@ If you wish to use multi-line YAML or some other delimiter between your topic de
 
 For example, `KAFKA_CREATE_TOPICS_SEPARATOR: "$$'\n"'` would use a newline to split the topic definitions. Syntax has to follow docker-compose escaping rules, and [ANSI-C](https://www.gnu.org/software/bash/manual/html_node/ANSI_002dC-Quoting.html) quoting.
 
+Another environment variable can be added to perform further configuration changes: ```KAFKA_CONFIGS```. For example:
+
+    environment:
+      - KAFKA_CONFIGS=topics:Topic1:add:retention.ms=604800000 cleanup.policy=delete segment.bytes=1024
+
+The format for this variable mirrors the ```kafka-configs.sh``` script. For multi-line YAML a separate delimiter variable ```KAFKA_CREATE_TOPICS_SEPARATOR``` is available.
+
 ## Advertised hostname
 
 You can configure the advertised hostname in different ways
diff --git a/configs.sh b/configs.sh
new file mode 100755
index 00000000..8e72afc0
--- /dev/null
+++ b/configs.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+if [[ -z "$START_TIMEOUT" ]]; then
+    START_TIMEOUT=600
+fi
+
+start_timeout_exceeded=false
+count=0
+step=10
+while netstat -lnt | awk '$4 ~ /:'"$KAFKA_PORT"'$/ {exit 1}'; do
+    echo "waiting for kafka to be ready"
+    sleep $step;
+    count=$((count + step))
+    if [ $count -gt $START_TIMEOUT ]; then
+        start_timeout_exceeded=true
+        break
+    fi
+done
+
+if $start_timeout_exceeded; then
+    echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
+    exit 1
+fi
+
+if [[ -n "$KAFKA_CREATE_TOPICS" ]]; then
+    # Expected format:
+    # name:partitions:replicas:cleanup.policy
+        IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOPICS; do
+        echo "creating topics: $topicToCreate"
+        IFS=':' read -r -a topicConfig <<< "$topicToCreate"
+        config=
+        if [ -n "${topicConfig[3]}" ]; then
+            config="--config=cleanup.policy=${topicConfig[3]}"
+        fi
+        COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\
+    		--create \\
+		    --zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\
+    		--topic ${topicConfig[0]} \\
+		    --partitions ${topicConfig[1]} \\
+    		--replication-factor ${topicConfig[2]} \\
+		    ${config} \\
+    		--if-not-exists &"
+        eval "${COMMAND}"
+    done
+fi
+
+wait
+
+if [[ -n "$KAFKA_CONFIGS" ]]; then
+    # Expected format:
+    # type:name:add|delete:a=b c=d
+        IFS="${KAFKA_CONFIG_SEPARATOR-,}"; for configToChange in $KAFKA_CONFIGS; do
+        echo "changing config: $configToChange"
+        IFS=':' read -r -a entityConfig <<< "$configToChange"
+        config="${entityConfig[3]}"
+        COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-configs.sh \\
+		    --zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\
+    		--entity-type ${entityConfig[0]} \\
+    		--entity-name ${entityConfig[1]} \\
+        --alter \\
+        --${entityConfig[2]}-config ${config// /,} &"
+        eval "${COMMAND}"
+    done
+fi
+
+wait
diff --git a/create-topics.sh b/create-topics.sh
deleted file mode 100755
index 9d89540c..00000000
--- a/create-topics.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/bash
-
-if [[ -z "$KAFKA_CREATE_TOPICS" ]]; then
-    exit 0
-fi
-
-if [[ -z "$START_TIMEOUT" ]]; then
-    START_TIMEOUT=600
-fi
-
-start_timeout_exceeded=false
-count=0
-step=10
-while netstat -lnt | awk '$4 ~ /:'"$KAFKA_PORT"'$/ {exit 1}'; do
-    echo "waiting for kafka to be ready"
-    sleep $step;
-    count=$((count + step))
-    if [ $count -gt $START_TIMEOUT ]; then
-        start_timeout_exceeded=true
-        break
-    fi
-done
-
-if $start_timeout_exceeded; then
-    echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
-    exit 1
-fi
-
-# Expected format:
-#   name:partitions:replicas:cleanup.policy
-IFS="${KAFKA_CREATE_TOPICS_SEPARATOR-,}"; for topicToCreate in $KAFKA_CREATE_TOPICS; do
-    echo "creating topics: $topicToCreate"
-    IFS=':' read -r -a topicConfig <<< "$topicToCreate"
-    config=
-    if [ -n "${topicConfig[3]}" ]; then
-        config="--config=cleanup.policy=${topicConfig[3]}"
-    fi
-    COMMAND="JMX_PORT='' ${KAFKA_HOME}/bin/kafka-topics.sh \\
-		--create \\
-		--zookeeper ${KAFKA_ZOOKEEPER_CONNECT} \\
-		--topic ${topicConfig[0]} \\
-		--partitions ${topicConfig[1]} \\
-		--replication-factor ${topicConfig[2]} \\
-		${config} \\
-		--if-not-exists &"
-    eval "${COMMAND}"
-done
-
-wait
diff --git a/start-kafka.sh b/start-kafka.sh
index 51567abf..1577576a 100755
--- a/start-kafka.sh
+++ b/start-kafka.sh
@@ -19,8 +19,9 @@ if [[ -z "$KAFKA_PORT" ]]; then
     export KAFKA_PORT=9092
 fi
 
-create-topics.sh &
+configs.sh &
 unset KAFKA_CREATE_TOPICS
+unset KAFKA_CONFIGS
 
 # DEPRECATED: but maintained for compatibility with older brokers pre 0.9.0 (https://issues.apache.org/jira/browse/KAFKA-1809)
 if [[ -z "$KAFKA_ADVERTISED_PORT" && \

From d8c6e5d9db9b52397be1b20e1c46ad5dfc041b10 Mon Sep 17 00:00:00 2001
From: Ron Arts <ron.arts@novantum.com>
Date: Wed, 23 May 2018 05:35:49 +0200
Subject: [PATCH 2/4] script changed name, adapt travis

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 7bce2155..c406534f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,7 @@ before_script:
 
 script:
   # Shellcheck main source files
-  - shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh
+  - shellcheck -s bash broker-list.sh configs.sh start-kafka.sh download-kafka.sh
   - cd test
   # Shellcheck the tests
   - shellcheck -x -e SC1090 -s bash *sh

From 20af68c4ce3f13abec8033556d95a021e18375c5 Mon Sep 17 00:00:00 2001
From: Ron Arts <ron.arts@novantum.com>
Date: Wed, 23 May 2018 05:45:26 +0200
Subject: [PATCH 3/4] Should not have renamed create-topics.sh

---
 .travis.yml                    | 2 +-
 configs.sh => create-topics.sh | 0
 start-kafka.sh                 | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename configs.sh => create-topics.sh (100%)

diff --git a/.travis.yml b/.travis.yml
index c406534f..7bce2155 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,7 +17,7 @@ before_script:
 
 script:
   # Shellcheck main source files
-  - shellcheck -s bash broker-list.sh configs.sh start-kafka.sh download-kafka.sh
+  - shellcheck -s bash broker-list.sh create-topics.sh start-kafka.sh download-kafka.sh
   - cd test
   # Shellcheck the tests
   - shellcheck -x -e SC1090 -s bash *sh
diff --git a/configs.sh b/create-topics.sh
similarity index 100%
rename from configs.sh
rename to create-topics.sh
diff --git a/start-kafka.sh b/start-kafka.sh
index 1577576a..b3c9ed76 100755
--- a/start-kafka.sh
+++ b/start-kafka.sh
@@ -19,7 +19,7 @@ if [[ -z "$KAFKA_PORT" ]]; then
     export KAFKA_PORT=9092
 fi
 
-configs.sh &
+create-topics.sh &
 unset KAFKA_CREATE_TOPICS
 unset KAFKA_CONFIGS
 

From 7d93191b494196907db7f9523b9d331901bfecc8 Mon Sep 17 00:00:00 2001
From: Ron Arts <ron.arts@novantum.com>
Date: Wed, 23 May 2018 10:28:49 +0200
Subject: [PATCH 4/4] Shoot! Don't commit stuff in the middle of the night

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index d6e51c1b..6740cc1a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,12 +13,12 @@ ENV KAFKA_VERSION=$kafka_version \
 
 ENV PATH=${PATH}:${KAFKA_HOME}/bin
 
-COPY download-kafka.sh start-kafka.sh broker-list.sh configs.sh /tmp/
+COPY download-kafka.sh start-kafka.sh broker-list.sh create-topics.sh /tmp/
 
 RUN apk add --no-cache bash curl jq docker \
  && mkdir /opt \
  && chmod a+x /tmp/*.sh \
- && mv /tmp/start-kafka.sh /tmp/broker-list.sh /tmp/configs.sh /usr/bin \
+ && mv /tmp/start-kafka.sh /tmp/broker-list.sh /tmp/create-topics.sh /usr/bin \
  && sync && /tmp/download-kafka.sh \
  && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt \
  && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz \