From 80f29039ef0cb17000841b54a3c7d165581ffba3 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 14 Aug 2018 14:50:15 +0200 Subject: [PATCH 01/13] Kafka and importantly Zookeeper upgraded to 2.0.0 --- avro-tools/test/rest-curl.yml | 2 +- kafka/50kafka.yml | 2 +- kafka/test/kafkacat.yml | 2 +- kafka/test/produce-consume.yml | 6 +++--- maintenance/preferred-replica-election-job.yml | 2 +- maintenance/reassign-paritions-job.yml | 2 +- maintenance/replication-factor-increase-job.yml | 2 +- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/avro-tools/test/rest-curl.yml b/avro-tools/test/rest-curl.yml index 7a42219b..0b491cff 100644 --- a/avro-tools/test/rest-curl.yml +++ b/avro-tools/test/rest-curl.yml @@ -114,7 +114,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 659c9d4d..fad17e6e 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -42,7 +42,7 @@ spec: mountPath: /etc/kafka containers: - name: broker - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index f551fbdd..6f1aa5b5 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -72,7 +72,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index d5aaf8b6..7634ac13 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -55,7 +55,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 command: - ./bin/kafka-topics.sh - --zookeeper @@ -95,7 +95,7 @@ spec: spec: containers: - name: producer - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 @@ -114,7 +114,7 @@ spec: - name: shared mountPath: /shared - name: consumer - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 diff --git a/maintenance/preferred-replica-election-job.yml b/maintenance/preferred-replica-election-job.yml index f3cfa83e..16060d82 100644 --- a/maintenance/preferred-replica-election-job.yml +++ b/maintenance/preferred-replica-election-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 command: - ./bin/kafka-preferred-replica-election.sh - --zookeeper diff --git a/maintenance/reassign-paritions-job.yml b/maintenance/reassign-paritions-job.yml index 317d9110..d7eab61d 100644 --- a/maintenance/reassign-paritions-job.yml +++ b/maintenance/reassign-paritions-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/maintenance/replication-factor-increase-job.yml b/maintenance/replication-factor-increase-job.yml index 00a13ae8..40781af5 100644 --- a/maintenance/replication-factor-increase-job.yml +++ b/maintenance/replication-factor-increase-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index d923ee97..9132f9e3 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -33,7 +33,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 88b781af..46f67976 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -36,7 +36,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties From 226db874bef93c68d8def3649da9eabeb9916c4f Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Sep 2018 14:36:41 +0200 Subject: [PATCH 02/13] Same Kafka, newer Java 8 JRE, newer Debian --- avro-tools/test/rest-curl.yml | 2 +- kafka/50kafka.yml | 2 +- kafka/test/kafkacat.yml | 2 +- kafka/test/produce-consume.yml | 6 +++--- maintenance/preferred-replica-election-job.yml | 2 +- maintenance/reassign-paritions-job.yml | 2 +- maintenance/replication-factor-increase-job.yml | 2 +- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/avro-tools/test/rest-curl.yml b/avro-tools/test/rest-curl.yml index 0b491cff..bea98595 100644 --- a/avro-tools/test/rest-curl.yml +++ b/avro-tools/test/rest-curl.yml @@ -114,7 +114,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index fad17e6e..8cc4c289 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -42,7 +42,7 @@ spec: mountPath: /etc/kafka containers: - name: broker - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index 6f1aa5b5..47bec3c1 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -72,7 +72,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index 7634ac13..b8325d6d 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -55,7 +55,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 command: - ./bin/kafka-topics.sh - --zookeeper @@ -95,7 +95,7 @@ spec: spec: containers: - name: producer - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 @@ -114,7 +114,7 @@ spec: - name: shared mountPath: /shared - name: consumer - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 diff --git a/maintenance/preferred-replica-election-job.yml b/maintenance/preferred-replica-election-job.yml index 16060d82..f8f75012 100644 --- a/maintenance/preferred-replica-election-job.yml +++ b/maintenance/preferred-replica-election-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 command: - ./bin/kafka-preferred-replica-election.sh - --zookeeper diff --git a/maintenance/reassign-paritions-job.yml b/maintenance/reassign-paritions-job.yml index d7eab61d..cb9d8a5c 100644 --- a/maintenance/reassign-paritions-job.yml +++ b/maintenance/reassign-paritions-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/maintenance/replication-factor-increase-job.yml b/maintenance/replication-factor-increase-job.yml index 40781af5..9cf2306c 100644 --- a/maintenance/replication-factor-increase-job.yml +++ b/maintenance/replication-factor-increase-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 9132f9e3..3e78e7e3 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -33,7 +33,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 46f67976..d8276846 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -36,7 +36,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 + image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties From 79c4c9250b341d850433f0c00a679ae2394160bd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Sep 2018 14:46:14 +0200 Subject: [PATCH 03/13] Bumps JMX exporter to a version based on the same JRE image --- prometheus/50-kafka-jmx-exporter-patch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/50-kafka-jmx-exporter-patch.yml b/prometheus/50-kafka-jmx-exporter-patch.yml index 7876ef47..8b116dfe 100644 --- a/prometheus/50-kafka-jmx-exporter-patch.yml +++ b/prometheus/50-kafka-jmx-exporter-patch.yml @@ -14,7 +14,7 @@ spec: spec: containers: - name: metrics - image: solsson/kafka-prometheus-jmx-exporter@sha256:a23062396cd5af1acdf76512632c20ea6be76885dfc20cd9ff40fb23846557e8 + image: solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 command: - java - -XX:+UnlockExperimentalVMOptions From 7f77c18cb7dab05aa5082ceac85690a01e33b0f8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 05:26:06 +0100 Subject: [PATCH 04/13] Runs Kafka 2.1.0 on Oracle open source JDK 11.0.1 This is the first Kafka release to officially support Java >8. We should stabilize on it ASAP because of improved docker support in JDK 10+. See https://blog.docker.com/2018/04/improved-docker-container-integration-with-java-10/ --- avro-tools/test/rest-curl.yml | 2 +- kafka/50kafka.yml | 2 +- kafka/test/kafkacat.yml | 2 +- kafka/test/produce-consume.yml | 6 +++--- maintenance/preferred-replica-election-job.yml | 2 +- maintenance/reassign-paritions-job.yml | 2 +- maintenance/replication-factor-increase-job.yml | 2 +- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/avro-tools/test/rest-curl.yml b/avro-tools/test/rest-curl.yml index bea98595..c01a037e 100644 --- a/avro-tools/test/rest-curl.yml +++ b/avro-tools/test/rest-curl.yml @@ -114,7 +114,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 8cc4c289..378ce85c 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -42,7 +42,7 @@ spec: mountPath: /etc/kafka containers: - name: broker - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index 47bec3c1..1e4645e1 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -72,7 +72,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index b8325d6d..0e841a58 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -55,7 +55,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 command: - ./bin/kafka-topics.sh - --zookeeper @@ -95,7 +95,7 @@ spec: spec: containers: - name: producer - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 @@ -114,7 +114,7 @@ spec: - name: shared mountPath: /shared - name: consumer - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 diff --git a/maintenance/preferred-replica-election-job.yml b/maintenance/preferred-replica-election-job.yml index f8f75012..704d9fc0 100644 --- a/maintenance/preferred-replica-election-job.yml +++ b/maintenance/preferred-replica-election-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 command: - ./bin/kafka-preferred-replica-election.sh - --zookeeper diff --git a/maintenance/reassign-paritions-job.yml b/maintenance/reassign-paritions-job.yml index cb9d8a5c..e56e21bd 100644 --- a/maintenance/reassign-paritions-job.yml +++ b/maintenance/reassign-paritions-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/maintenance/replication-factor-increase-job.yml b/maintenance/replication-factor-increase-job.yml index 9cf2306c..48742722 100644 --- a/maintenance/replication-factor-increase-job.yml +++ b/maintenance/replication-factor-increase-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 3e78e7e3..69de9832 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -33,7 +33,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index d8276846..95b5f23a 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -36,7 +36,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:2.0@sha256:b6db39a1039c5bb2f8e5c11bb71756dcc3c5ab3758703ca6229e8387199e04f0 + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties From 4ae82cde88a1381d37ad06891717a4804c166863 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 07:46:47 +0100 Subject: [PATCH 05/13] Initutils with latest Debian and kubectl --- kafka/50kafka.yml | 2 +- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 378ce85c..afda4847 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -20,7 +20,7 @@ spec: terminationGracePeriodSeconds: 30 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2 + image: solsson/kafka-initutils@sha256:2cdb90ea514194d541c7b869ac15d2d530ca64889f56e270161fe4e5c3d076ea env: - name: NODE_NAME valueFrom: diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 69de9832..154e90db 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -22,7 +22,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2 + image: solsson/kafka-initutils@sha256:2cdb90ea514194d541c7b869ac15d2d530ca64889f56e270161fe4e5c3d076ea command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] volumeMounts: - name: configmap diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 95b5f23a..7d2101d4 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -22,7 +22,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2 + image: solsson/kafka-initutils@sha256:2cdb90ea514194d541c7b869ac15d2d530ca64889f56e270161fe4e5c3d076ea command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] env: - name: ID_OFFSET From d4b5bf365a068663b5a67da86e16636b69b33881 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 07:47:11 +0100 Subject: [PATCH 06/13] JMX exporter on Kafka's JDK 11 image --- prometheus/50-kafka-jmx-exporter-patch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/50-kafka-jmx-exporter-patch.yml b/prometheus/50-kafka-jmx-exporter-patch.yml index 8b116dfe..fd0ce84e 100644 --- a/prometheus/50-kafka-jmx-exporter-patch.yml +++ b/prometheus/50-kafka-jmx-exporter-patch.yml @@ -14,7 +14,7 @@ spec: spec: containers: - name: metrics - image: solsson/kafka-prometheus-jmx-exporter@sha256:6f82e2b0464f50da8104acd7363fb9b995001ddff77d248379f8788e78946143 + image: solsson/kafka-prometheus-jmx-exporter@sha256:d237a12cc0cde42b539bcb5efc0008ba5e6ca1351b7843ed52bd574d181c5efd command: - java - -XX:+UnlockExperimentalVMOptions From 182a8bf4c3aedfdc079bde69f57a88b76fc2f599 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 08:27:23 +0100 Subject: [PATCH 07/13] Pixy 0.16 for Kafka 2.1.0 support --- pixy/pixy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixy/pixy.yml b/pixy/pixy.yml index 7226aa6b..8d4ea2c4 100644 --- a/pixy/pixy.yml +++ b/pixy/pixy.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: pixy - image: mailgun/kafka-pixy:0.15.0@sha256:088210d53945a0db5f93921ceff3a79c012449b7845baebe8898452741764e7c + image: mailgun/kafka-pixy:0.16.0@sha256:2a2d426f9fe17ede9c21b2f8b7418d8787293e39327b638fd6c403f3726597fb ports: - containerPort: 80 command: From 7f494710477abb29b02b12132db8faf27ce1035f Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 08:30:21 +0100 Subject: [PATCH 08/13] Use workloads api v1 everywhere --- avro-tools/rest.yml | 2 +- avro-tools/schemas.yml | 2 +- avro-tools/test/rest-curl.yml | 2 +- kafka/test/kafkacat.yml | 2 +- kafka/test/produce-consume.yml | 2 +- kafka/test/replication-config.yml | 2 +- linkedin-burrow/burrow.yml | 2 +- maintenance/test/replicated-partitions.yml | 2 +- prometheus/50-kafka-jmx-exporter-patch.yml | 2 +- yahoo-kafka-manager/kafka-manager.yml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/avro-tools/rest.yml b/avro-tools/rest.yml index be299e6a..1f02cefe 100644 --- a/avro-tools/rest.yml +++ b/avro-tools/rest.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: avro-rest diff --git a/avro-tools/schemas.yml b/avro-tools/schemas.yml index b811d17d..964f988e 100644 --- a/avro-tools/schemas.yml +++ b/avro-tools/schemas.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: avro-schemas diff --git a/avro-tools/test/rest-curl.yml b/avro-tools/test/rest-curl.yml index c01a037e..2cdadd24 100644 --- a/avro-tools/test/rest-curl.yml +++ b/avro-tools/test/rest-curl.yml @@ -129,7 +129,7 @@ spec: - "1" restartPolicy: Never --- -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: rest-curl diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index 1e4645e1..7f2eca6b 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -87,7 +87,7 @@ spec: - "3" restartPolicy: Never --- -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: ReplicaSet metadata: name: kafkacat diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index 0e841a58..44e54db2 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -70,7 +70,7 @@ spec: - "3" restartPolicy: Never --- -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: ReplicaSet metadata: name: produce-consume diff --git a/kafka/test/replication-config.yml b/kafka/test/replication-config.yml index ec528071..68c0872a 100644 --- a/kafka/test/replication-config.yml +++ b/kafka/test/replication-config.yml @@ -45,7 +45,7 @@ data: exit 0 --- -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: ReplicaSet metadata: name: replication-config diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml index c6a3cf3e..1e9ff47c 100644 --- a/linkedin-burrow/burrow.yml +++ b/linkedin-burrow/burrow.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: burrow diff --git a/maintenance/test/replicated-partitions.yml b/maintenance/test/replicated-partitions.yml index fc535371..72728b7d 100644 --- a/maintenance/test/replicated-partitions.yml +++ b/maintenance/test/replicated-partitions.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: replicated-partitions diff --git a/prometheus/50-kafka-jmx-exporter-patch.yml b/prometheus/50-kafka-jmx-exporter-patch.yml index fd0ce84e..572b3769 100644 --- a/prometheus/50-kafka-jmx-exporter-patch.yml +++ b/prometheus/50-kafka-jmx-exporter-patch.yml @@ -1,6 +1,6 @@ # meant to be applied using # kubectl --namespace kafka patch statefulset kafka --patch "$(cat prometheus/50-kafka-jmx-exporter-patch.yml )" -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: StatefulSet metadata: name: kafka diff --git a/yahoo-kafka-manager/kafka-manager.yml b/yahoo-kafka-manager/kafka-manager.yml index b801982f..8d659041 100644 --- a/yahoo-kafka-manager/kafka-manager.yml +++ b/yahoo-kafka-manager/kafka-manager.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta2 +apiVersion: apps/v1 kind: Deployment metadata: name: kafka-manager From 1ecae522573673b9cbe0927985feda7f178f5e71 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 10:37:50 +0100 Subject: [PATCH 09/13] Runs latest eventrouter --- events-kube/events-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/events-kube/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml index 41fe35e3..6a90e667 100644 --- a/events-kube/events-kube-kafka.yml +++ b/events-kube/events-kube-kafka.yml @@ -30,7 +30,7 @@ spec: serviceAccount: eventrouter containers: - name: kube-eventrouter - image: gcr.io/heptio-images/eventrouter@sha256:30e36ce7bad4a7c539e0a0cb1833d309089919fb0ef0c165ee28aabe97740d02 + image: gcr.io/heptio-images/eventrouter@sha256:e613b48c6235426fa334867d661118322b4e1973c23e2e8cf5c066b982cc8596 resources: requests: memory: "5Mi" From f9162a646890b795dfe8db35baa376a27434eb72 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 14:36:35 +0100 Subject: [PATCH 10/13] Resource limit awareness is default in Java 10+ which probably makes the discussion in #112 outdated. Instead we could discuss -XX:MaxRAMPercentage and related flags from http://bugs.java.com/view_bug.do?bug_id=JDK-8186248 --- prometheus/50-kafka-jmx-exporter-patch.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/prometheus/50-kafka-jmx-exporter-patch.yml b/prometheus/50-kafka-jmx-exporter-patch.yml index 572b3769..179cbb5e 100644 --- a/prometheus/50-kafka-jmx-exporter-patch.yml +++ b/prometheus/50-kafka-jmx-exporter-patch.yml @@ -17,10 +17,6 @@ spec: image: solsson/kafka-prometheus-jmx-exporter@sha256:d237a12cc0cde42b539bcb5efc0008ba5e6ca1351b7843ed52bd574d181c5efd command: - java - - -XX:+UnlockExperimentalVMOptions - - -XX:+UseCGroupMemoryLimitForHeap - - -XX:MaxRAMFraction=1 - - -XshowSettings:vm - -jar - jmx_prometheus_httpserver.jar - "5556" From 6e81906d55265b6de0d97dbff085e4e5e0bced46 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 14:40:41 +0100 Subject: [PATCH 11/13] Documents recent releases, and the upcoming one --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d4da96ac..b98121aa 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,13 @@ If you begin to rely on this kafka setup we recommend you fork, for example to e ## Version history -| tag | k8s ≥ | highlights | +| tag | k8s ≥ | highlights | | ----- | ------ | ---------- | -| 4.x | 1.9+ | Kafka 1.1 dynamic config | -| v4.1 | 1.9+ | Kafka 1.0.1 new [default](#148) [config](#170) | -| v3.2 | 1.9.4, 1.8.9, 1.7.14 | Required for read-only ConfigMaps [#162](https://github.com/Yolean/kubernetes-kafka/issues/162) [#163](https://github.com/Yolean/kubernetes-kafka/pull/163) [k8s #58720](https://github.com/kubernetes/kubernetes/pull/58720) | +| v5.0 | 1.11+ | Destabilize because in Docker we want Java 11 [#197](https://github.com/Yolean/kubernetes-kafka/pull/197) [#191](https://github.com/Yolean/kubernetes-kafka/pull/191) | +| v4.3 | 1.9+ | Adds a prpper shutdown hook [207](https://github.com/Yolean/kubernetes-kafka/pull/207) | +| v4.2 | 1.9+ | Kafka 1.0.2 and tools upgrade | +| v4.1 | 1.9+ | Kafka 1.0.1 new [default](#148) [config](#170) | +| v3.2 | 1.9.4, 1.8.9, 1.7.14 | Required for read-only ConfigMaps [#162](https://github.com/Yolean/kubernetes-kafka/issues/162) [#163](https://github.com/Yolean/kubernetes-kafka/pull/163) [k8s #58720](https://github.com/kubernetes/kubernetes/pull/58720) | | v3.1 | 1.8 | The painstaking path to `min.insync.replicas`=2 | | v3.0 | 1.8 | [Outside access](#78), [modern manifests](#84), [bootstrap.kafka](#52) | | v2.1 | 1.5 | Kafka 1.0, the init script concept | From d088b259a5cf071fe341d3a64c648e10ec1ba3dc Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 15:14:21 +0100 Subject: [PATCH 12/13] We want a memory limit low enough for people to bump into it as soon as they apply any non-testing load, but still high enough to support for example this repo's eventrouter + tests. --- kafka/50kafka.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index afda4847..41940062 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -61,7 +61,12 @@ spec: resources: requests: cpu: 100m - memory: 512Mi + memory: 100Mi + limits: + # This limit was intentionally set low as a reminder that + # the entire Yolean/kubernetes-kafka is meant to be tweaked + # before you run production workloads + memory: 600Mi readinessProbe: tcpSocket: port: 9092 From 1fcbfbd2fa5624b70f1794c309e7e2a1ac233ff6 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 15:40:24 +0100 Subject: [PATCH 13/13] Low memory limits for zookeeper too, same argument as stated in the 50kafka.yml manifest --- zookeeper/50pzoo.yml | 2 ++ zookeeper/51zoo.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 154e90db..ecf88926 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -51,6 +51,8 @@ spec: requests: cpu: 10m memory: 100Mi + limits: + memory: 100Mi readinessProbe: exec: command: diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 7d2101d4..aea5763d 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -54,6 +54,8 @@ spec: requests: cpu: 10m memory: 100Mi + limits: + memory: 100Mi readinessProbe: exec: command: