From c6314666e9cad10944d587d90c8d003a780a52dd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 8 Jan 2018 12:57:58 +0100 Subject: [PATCH 1/4] Adds the required RBAC for the init script to set the kafka-broker-id label --- rbac-namespace-default/pod-labler.yml | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 rbac-namespace-default/pod-labler.yml diff --git a/rbac-namespace-default/pod-labler.yml b/rbac-namespace-default/pod-labler.yml new file mode 100644 index 00000000..78816a3a --- /dev/null +++ b/rbac-namespace-default/pod-labler.yml @@ -0,0 +1,41 @@ +# To see if init containers need RBAC: +# +# $ kubectl exec kafka-0 -- cat /etc/kafka/server.properties | grep broker.rack +# #init#broker.rack=# zone lookup failed, see -c init-config logs +# $ kubectl logs -c init-config kafka-0 +# ++ kubectl get node some-node '-o=go-template={{index .metadata.labels "failure-domain.beta.kubernetes.io/zone"}}' +# Error from server (Forbidden): User "system:serviceaccount:kafka:default" cannot get nodes at the cluster scope.: "Unknown user \"system:serviceaccount:kafka:default\"" +# +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: pod-labler + namespace: kafka + labels: + origin: github.com_Yolean_kubernetes-kafka +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - update + - patch +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kafka-pod-labler + namespace: kafka + labels: + origin: github.com_Yolean_kubernetes-kafka +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pod-labler +subjects: +- kind: ServiceAccount + name: default + namespace: kafka From 8811b0264591fba3299ace4d537db09613bacb56 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 8 Jan 2018 13:02:47 +0100 Subject: [PATCH 2/4] Revert "Deprecates our own statefulset pod label kafka-broker-id" This reverts commit 096e3bdfba31d2918520011886a6a0d1f1b08c84. --- README.md | 2 -- kafka/10broker-config.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/README.md b/README.md index 4ba9a0ce..c3495647 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,6 @@ For clusters that enfoce [RBAC](https://kubernetes.io/docs/admin/authorization/r kubectl apply -f rbac-namespace-default/ ``` -For example rack awareness can fail without this, `logs -c init-config` showing `Error from server (Forbidden): pods "kafka-0" is forbidden: User "system:serviceaccount:kafka:default" cannot get pods in the namespace "kafka": Unknown user "system:serviceaccount:kafka:default"`. - ## Tests Tests are based on the [kube-test](https://github.com/Yolean/kube-test) concept. diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index bc1d55db..d11eeafc 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -23,7 +23,6 @@ data: sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties fi - # This requires additional RBAC, and won't be needed after https://github.com/kubernetes/kubernetes/pull/55329 kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') From f76e192ab4cdd5d0cc833690276a5eb3e7aa715d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 8 Jan 2018 13:21:59 +0100 Subject: [PATCH 3/4] Sets other useful labels, for #78 and #56 --- kafka/10broker-config.yml | 7 +++++-- rbac-namespace-default/pod-labler.yml | 9 ++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index d11eeafc..55a0c3b7 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -21,6 +21,7 @@ data: sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/" /etc/kafka/server.properties else sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties + kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-rack=$ZONE fi kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID @@ -29,8 +30,10 @@ data: if [ $? -ne 0 ]; then echo "Outside (i.e. cluster-external access) host lookup command failed" else - OUTSIDE_HOST=${OUTSIDE_HOST}:3240${KAFKA_BROKER_ID} - sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}|" /etc/kafka/server.properties + OUTSIDE_PORT=3240${KAFKA_BROKER_ID} + sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|" /etc/kafka/server.properties + kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-listener-outside-host=$OUTSIDE_HOST + kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-listener-outside-port=$OUTSIDE_PORT fi } diff --git a/rbac-namespace-default/pod-labler.yml b/rbac-namespace-default/pod-labler.yml index 78816a3a..92745aff 100644 --- a/rbac-namespace-default/pod-labler.yml +++ b/rbac-namespace-default/pod-labler.yml @@ -1,10 +1,9 @@ # To see if init containers need RBAC: # -# $ kubectl exec kafka-0 -- cat /etc/kafka/server.properties | grep broker.rack -# #init#broker.rack=# zone lookup failed, see -c init-config logs -# $ kubectl logs -c init-config kafka-0 -# ++ kubectl get node some-node '-o=go-template={{index .metadata.labels "failure-domain.beta.kubernetes.io/zone"}}' -# Error from server (Forbidden): User "system:serviceaccount:kafka:default" cannot get nodes at the cluster scope.: "Unknown user \"system:serviceaccount:kafka:default\"" +# $ kubectl -n kafka logs kafka-2 -c init-config +# ... +# + kubectl -n kafka label pod kafka-2 kafka-broker-id=2 +# Error from server (Forbidden): pods "kafka-2" is forbidden: User "system:serviceaccount:kafka:default" cannot get pods in the namespace "kafka": Unknown user "system:serviceaccount:kafka:default" # --- kind: Role From cc27bc113cfe2d295a59336236ea4d69ca4c38f3 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 8 Jan 2018 16:15:45 +0100 Subject: [PATCH 4/4] Sets all labels at the end of the init script, to speed up a bit and reduce the risk that it interferes with actual config. Also if the init script goes crash looping you won't get warnings about labels already being set. --- kafka/10broker-config.yml | 13 ++++++++----- rbac-namespace-default/pod-labler.yml | 1 - 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 55a0c3b7..28462328 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -11,6 +11,8 @@ data: KAFKA_BROKER_ID=${HOSTNAME##*-} sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties + LABELS="kafka-broker-id=$KAFKA_BROKER_ID" + hash kubectl 2>/dev/null || { sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/" /etc/kafka/server.properties } && { @@ -21,19 +23,20 @@ data: sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/" /etc/kafka/server.properties else sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties - kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-rack=$ZONE + LABELS="$LABELS kafka-broker-rack=$ZONE" fi - kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID - OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') if [ $? -ne 0 ]; then echo "Outside (i.e. cluster-external access) host lookup command failed" else OUTSIDE_PORT=3240${KAFKA_BROKER_ID} sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|" /etc/kafka/server.properties - kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-listener-outside-host=$OUTSIDE_HOST - kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-listener-outside-port=$OUTSIDE_PORT + LABELS="$LABELS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" + fi + + if [ ! -z "$LABELS" ]; then + kubectl -n $POD_NAMESPACE label pod $POD_NAME $LABELS || echo "Failed to label $POD_NAMESPACE.$POD_NAME - RBAC issue?" fi } diff --git a/rbac-namespace-default/pod-labler.yml b/rbac-namespace-default/pod-labler.yml index 92745aff..bd488b00 100644 --- a/rbac-namespace-default/pod-labler.yml +++ b/rbac-namespace-default/pod-labler.yml @@ -2,7 +2,6 @@ # # $ kubectl -n kafka logs kafka-2 -c init-config # ... -# + kubectl -n kafka label pod kafka-2 kafka-broker-id=2 # Error from server (Forbidden): pods "kafka-2" is forbidden: User "system:serviceaccount:kafka:default" cannot get pods in the namespace "kafka": Unknown user "system:serviceaccount:kafka:default" # ---