From 642c620d0074c8547fc2165bb95490d2fce04ce4 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 1 Mar 2017 14:50:01 +0100 Subject: [PATCH 1/6] Uses latest jre image --- docker-kafka-persistent/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-kafka-persistent/Dockerfile b/docker-kafka-persistent/Dockerfile index 81200562..4db24ede 100644 --- a/docker-kafka-persistent/Dockerfile +++ b/docker-kafka-persistent/Dockerfile @@ -1,5 +1,5 @@ -FROM openjdk:8u102-jre +FROM openjdk:8u121-jre ENV kafka_version=0.10.1.1 ENV scala_version=2.11.8 From 42ea2182f63541ffbe256bdd353e300bf73ba003 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 1 Mar 2017 14:50:21 +0100 Subject: [PATCH 2/6] Upgrades Kafka to latest release --- docker-kafka-persistent/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-kafka-persistent/Dockerfile b/docker-kafka-persistent/Dockerfile index 4db24ede..8e593ce4 100644 --- a/docker-kafka-persistent/Dockerfile +++ b/docker-kafka-persistent/Dockerfile @@ -1,7 +1,7 @@ FROM openjdk:8u121-jre -ENV kafka_version=0.10.1.1 +ENV kafka_version=0.10.2.0 ENV scala_version=2.11.8 ENV kafka_bin_version=2.11-$kafka_version From 3739b24219d6850dcf7a08b5aac8f7e9a9e39c76 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 1 Mar 2017 14:50:52 +0100 Subject: [PATCH 3/6] Upgrades Scala to latest, supported as of Kafka 0.10.2 --- docker-kafka-persistent/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-kafka-persistent/Dockerfile b/docker-kafka-persistent/Dockerfile index 8e593ce4..678f6f2c 100644 --- a/docker-kafka-persistent/Dockerfile +++ b/docker-kafka-persistent/Dockerfile @@ -2,8 +2,8 @@ FROM openjdk:8u121-jre ENV kafka_version=0.10.2.0 -ENV scala_version=2.11.8 -ENV kafka_bin_version=2.11-$kafka_version +ENV scala_version=2.12.1 +ENV kafka_bin_version=2.12-$kafka_version RUN curl -SLs "http://www.scala-lang.org/files/archive/scala-$scala_version.deb" -o scala.deb \ && dpkg -i scala.deb \ From a385740ddf05532f48c73f2aba60a4ab44b61b6e Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 1 Mar 2017 15:02:59 +0100 Subject: [PATCH 4/6] Adds namespace to kubectl commands, matching that in yamls --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 492af465..8cfaecdf 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Alternatively create [PV](http://kubernetes.io/docs/user-guide/persistent-volume ./bootstrap/pv.sh kubectl create -f ./bootstrap/pvc.yml # check that claims are bound -kubectl get pvc +kubectl -n kafka get pvc ``` ## Set up Zookeeper @@ -49,7 +49,7 @@ kubectl create -f ./ You might want to verify in logs that Kafka found its own DNS name(s) correctly. Look for records like: ``` -kubectl logs kafka-0 | grep "Registered broker" +kubectl -n kafka logs kafka-0 | grep "Registered broker" # INFO Registered broker 0 at path /brokers/ids/0 with addresses: PLAINTEXT -> EndPoint(kafka-0.broker.kafka.svc.cluster.local,9092,PLAINTEXT) ``` @@ -79,5 +79,5 @@ kubectl create -f test/21consumer-test1.yml Testing and retesting... delete the namespace. PVs are outside namespaces so delete them too. ``` kubectl delete namespace kafka -rm -R ./data/ && kubectl delete pv datadir-kafka-0 datadir-kafka-1 datadir-kafka-2 +rm -R ./data/ && kubectl -n kafka delete pv datadir-kafka-0 datadir-kafka-1 datadir-kafka-2 ``` From 1110d598c8dfe705f78fc101cf697738e65d97c6 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 1 Mar 2017 15:08:43 +0100 Subject: [PATCH 5/6] Makes PV match PVC, avoiding small initial storage becase resize is difficult in GKE --- README.md | 2 +- bootstrap/pv-template.yml | 6 +++--- bootstrap/pv.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8cfaecdf..11175391 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Alternatively create [PV](http://kubernetes.io/docs/user-guide/persistent-volume ``` ./bootstrap/pv.sh -kubectl create -f ./bootstrap/pvc.yml +kubectl create -f ./10pvc.yml # check that claims are bound kubectl -n kafka get pvc ``` diff --git a/bootstrap/pv-template.yml b/bootstrap/pv-template.yml index e58bfb2b..befb6b6c 100644 --- a/bootstrap/pv-template.yml +++ b/bootstrap/pv-template.yml @@ -10,7 +10,7 @@ spec: accessModes: - ReadWriteOnce capacity: - storage: 100Mi + storage: 200Gi hostPath: path: /tmp/k8s-data/datadir-kafka-0 --- @@ -25,7 +25,7 @@ spec: accessModes: - ReadWriteOnce capacity: - storage: 100Mi + storage: 200Gi hostPath: path: /tmp/k8s-data/datadir-kafka-1 --- @@ -40,6 +40,6 @@ spec: accessModes: - ReadWriteOnce capacity: - storage: 100Mi + storage: 200Gi hostPath: path: /tmp/k8s-data/datadir-kafka-2 diff --git a/bootstrap/pv.sh b/bootstrap/pv.sh index 78bf7f5d..c12787bc 100755 --- a/bootstrap/pv.sh +++ b/bootstrap/pv.sh @@ -1,6 +1,6 @@ #!/bin/bash -echo "Note that in for example GKE a PetSet will have PersistentVolume(s) and PersistentVolumeClaim(s) created for it automatically" +echo "Note that in for example GKE a StatefulSet will have PersistentVolume(s) and PersistentVolumeClaim(s) created for it automatically" dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" path="$dir/data" From b9340fe5d45b9aca8a0c11b924698624666e941e Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 1 Mar 2017 15:40:13 +0100 Subject: [PATCH 6/6] 0.10.2 adds to the confusion about consumer args, #21 --- test/21consumer-test1.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/21consumer-test1.yml b/test/21consumer-test1.yml index aff5944f..33218738 100644 --- a/test/21consumer-test1.yml +++ b/test/21consumer-test1.yml @@ -17,8 +17,8 @@ spec: image: solsson/kafka:0.10.0.1 command: - ./bin/kafka-console-consumer.sh - - --zookeeper - - zookeeper:2181 + - --bootstrap-server + - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 - --topic - test1 - --from-beginning