From 77a3bbf74e7320d6c1f86cd168f5aef48dc34ffe Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Mon, 18 Jul 2022 04:55:25 +0000 Subject: [PATCH 1/8] change number of broker --- kafka/50kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index a001f1d5..c04d162a 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -8,7 +8,7 @@ spec: matchLabels: app: kafka serviceName: "kafka" - replicas: 3 + replicas: 5 updateStrategy: type: RollingUpdate podManagementPolicy: Parallel From 67cd84d7fafa11843d3547500dddc537fdae0cdf Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Mon, 18 Jul 2022 05:28:29 +0000 Subject: [PATCH 2/8] delete unnecessary dir --- 01-test-namespace.yml | 5 - cruise-control/11cruise-control-config.yml | 380 ------------------ .../20kafka-broker-reporter-patch.yml | 25 -- cruise-control/40cruise-control-service.yml | 12 - cruise-control/50cruise-control.yml | 51 --- cruise-control/README.md | 26 -- cruise-control/topic-create.yml | 24 -- events-kube/README.md | 5 - events-kube/config.yml | 16 - events-kube/events-kube-kafka.yml | 47 --- events-kube/rbac/cluster-events-watcher.yml | 39 -- events-kube/topic-create.yaml | 24 -- native/distroless.yaml | 9 - native/kustomization.yaml | 17 - native/native-image-zookeeper.yaml | 29 -- nonroot/entrypoint-from-image.yaml | 2 - nonroot/fsgroup-65534.yaml | 4 - nonroot/kustomization.yaml | 44 -- nonroot/nonroot-image-kafka.yaml | 15 - nonroot/nonroot-image-zookeeper.yaml | 31 -- pixy/Kustomization | 3 - pixy/pixy-service.yml | 12 - pixy/pixy.yml | 30 -- yahoo-kafka-manager/Kustomization | 3 - yahoo-kafka-manager/kafka-manager-service.yml | 12 - yahoo-kafka-manager/kafka-manager.yml | 26 -- 26 files changed, 891 deletions(-) delete mode 100644 01-test-namespace.yml delete mode 100644 cruise-control/11cruise-control-config.yml delete mode 100644 cruise-control/20kafka-broker-reporter-patch.yml delete mode 100644 cruise-control/40cruise-control-service.yml delete mode 100644 cruise-control/50cruise-control.yml delete mode 100644 cruise-control/README.md delete mode 100644 cruise-control/topic-create.yml delete mode 100644 events-kube/README.md delete mode 100644 events-kube/config.yml delete mode 100644 events-kube/events-kube-kafka.yml delete mode 100644 events-kube/rbac/cluster-events-watcher.yml delete mode 100644 events-kube/topic-create.yaml delete mode 100644 native/distroless.yaml delete mode 100644 native/kustomization.yaml delete mode 100644 native/native-image-zookeeper.yaml delete mode 100644 nonroot/entrypoint-from-image.yaml delete mode 100644 nonroot/fsgroup-65534.yaml delete mode 100644 nonroot/kustomization.yaml delete mode 100644 nonroot/nonroot-image-kafka.yaml delete mode 100644 nonroot/nonroot-image-zookeeper.yaml delete mode 100644 pixy/Kustomization delete mode 100644 pixy/pixy-service.yml delete mode 100644 pixy/pixy.yml delete mode 100644 yahoo-kafka-manager/Kustomization delete mode 100644 yahoo-kafka-manager/kafka-manager-service.yml delete mode 100644 yahoo-kafka-manager/kafka-manager.yml diff --git a/01-test-namespace.yml b/01-test-namespace.yml deleted file mode 100644 index fbb6e0ef..00000000 --- a/01-test-namespace.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: test-kafka diff --git a/cruise-control/11cruise-control-config.yml b/cruise-control/11cruise-control-config.yml deleted file mode 100644 index b964d199..00000000 --- a/cruise-control/11cruise-control-config.yml +++ /dev/null @@ -1,380 +0,0 @@ -kind: ConfigMap -metadata: - name: broker-cruise-control-config - namespace: kafka -apiVersion: v1 -data: - cruisecontrol.properties: |- - # - # Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. - # - - # This is an example property file for Kafka Cruise Control. See KafkaCruiseControlConfig for more details. - - # Configuration for the metadata client. - # ======================================= - - # The Kafka cluster to control. - bootstrap.servers=bootstrap:9092 - - # The maximum interval in milliseconds between two metadata refreshes. - #metadata.max.age.ms=300000 - - # Client id for the Cruise Control. It is used for the metadata client. - #client.id=kafka-cruise-control - - # The size of TCP send buffer bytes for the metadata client. - #send.buffer.bytes=131072 - - # The size of TCP receive buffer size for the metadata client. - #receive.buffer.bytes=131072 - - # The time to wait before disconnect an idle TCP connection. - #connections.max.idle.ms=540000 - - # The time to wait before reconnect to a given host. - #reconnect.backoff.ms=50 - - # The time to wait for a response from a host after sending a request. - #request.timeout.ms=30000 - - - # Configurations for the load monitor - # ======================================= - - # The number of metric fetcher thread to fetch metrics for the Kafka cluster - num.metric.fetchers=1 - - # The metric sampler class - metric.sampler.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.CruiseControlMetricsReporterSampler - # Configurations for CruiseControlMetricsReporterSampler - metric.reporter.topic.pattern=__CruiseControlMetrics - - # The sample store class name - sample.store.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.KafkaSampleStore - - # The config for the Kafka sample store to save the partition metric samples - partition.metric.sample.store.topic=__KafkaCruiseControlPartitionMetricSamples - - # The config for the Kafka sample store to save the model training samples - broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples - - # The replication factor of Kafka metric sample store topic - sample.store.topic.replication.factor=2 - - # The config for the number of Kafka sample store consumer threads - num.sample.loading.threads=8 - - # The partition assignor class for the metric samplers - metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor - - # The metric sampling interval in milliseconds - metric.sampling.interval.ms=120000 - - # The partition metrics window size in milliseconds - partition.metrics.window.ms=300000 - - # The number of partition metric windows to keep in memory - num.partition.metrics.windows=1 - - # The minimum partition metric samples required for a partition in each window - min.samples.per.partition.metrics.window=1 - - # The broker metrics window size in milliseconds - broker.metrics.window.ms=300000 - - # The number of broker metric windows to keep in memory - num.broker.metrics.windows=20 - - # The minimum broker metric samples required for a partition in each window - min.samples.per.broker.metrics.window=1 - - # The configuration for the BrokerCapacityConfigFileResolver (supports JBOD and non-JBOD broker capacities) - capacity.config.file=config/capacity.json - #capacity.config.file=config/capacityJBOD.json - - # Configurations for the analyzer - # ======================================= - - # The list of goals to optimize the Kafka cluster for with pre-computed proposals - default.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal - - # The list of supported goals - goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.kafkaassigner.KafkaAssignerDiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.kafkaassigner.KafkaAssignerEvenRackAwareGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PreferredLeaderElectionGoal - - # The list of supported hard goals - hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal - - # The minimum percentage of well monitored partitions out of all the partitions - min.monitored.partition.percentage=0.95 - - # The balance threshold for CPU - cpu.balance.threshold=1.1 - - # The balance threshold for disk - disk.balance.threshold=1.1 - - # The balance threshold for network inbound utilization - network.inbound.balance.threshold=1.1 - - # The balance threshold for network outbound utilization - network.outbound.balance.threshold=1.1 - - # The balance threshold for the replica count - replica.count.balance.threshold=1.1 - - # The capacity threshold for CPU in percentage - cpu.capacity.threshold=0.8 - - # The capacity threshold for disk in percentage - disk.capacity.threshold=0.8 - - # The capacity threshold for network inbound utilization in percentage - network.inbound.capacity.threshold=0.8 - - # The capacity threshold for network outbound utilization in percentage - network.outbound.capacity.threshold=0.8 - - # The threshold to define the cluster to be in a low CPU utilization state - cpu.low.utilization.threshold=0.0 - - # The threshold to define the cluster to be in a low disk utilization state - disk.low.utilization.threshold=0.0 - - # The threshold to define the cluster to be in a low network inbound utilization state - network.inbound.low.utilization.threshold=0.0 - - # The threshold to define the cluster to be in a low disk utilization state - network.outbound.low.utilization.threshold=0.0 - - # The metric anomaly percentile upper threshold - metric.anomaly.percentile.upper.threshold=90.0 - - # The metric anomaly percentile lower threshold - metric.anomaly.percentile.lower.threshold=10.0 - - # How often should the cached proposal be expired and recalculated if necessary - proposal.expiration.ms=60000 - - # The maximum number of replicas that can reside on a broker at any given time. - max.replicas.per.broker=10000 - - # The number of threads to use for proposal candidate precomputing. - num.proposal.precompute.threads=1 - - # the topics that should be excluded from the partition movement. - #topics.excluded.from.partition.movement - - # Configurations for the executor - # ======================================= - - # The zookeeper connect of the Kafka cluster - zookeeper.connect=zookeeper:2181/ - - # The max number of partitions to move in/out on a given broker at a given time. - num.concurrent.partition.movements.per.broker=10 - - # The interval between two execution progress checks. - execution.progress.check.interval.ms=10000 - - - # Configurations for anomaly detector - # ======================================= - - # The goal violation notifier class - anomaly.notifier.class=com.linkedin.kafka.cruisecontrol.detector.notifier.SelfHealingNotifier - - # The metric anomaly finder class - metric.anomaly.finder.class=com.linkedin.kafka.cruisecontrol.detector.KafkaMetricAnomalyFinder - - # The anomaly detection interval - anomaly.detection.interval.ms=10000 - - # The goal violation to detect. - anomaly.detection.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal - - # The interested metrics for metric anomaly analyzer. - metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_MAX,BROKER_PRODUCE_LOCAL_TIME_MS_MEAN,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MAX,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MAX,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MEAN,BROKER_LOG_FLUSH_TIME_MS_MAX,BROKER_LOG_FLUSH_TIME_MS_MEAN - - ## Adjust accordingly if your metrics reporter is an older version and does not produce these metrics. - #metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_50TH,BROKER_PRODUCE_LOCAL_TIME_MS_999TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_50TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_999TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_50TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_999TH,BROKER_LOG_FLUSH_TIME_MS_50TH,BROKER_LOG_FLUSH_TIME_MS_999TH - - # The zk path to store failed broker information. - failed.brokers.zk.path=/CruiseControlBrokerList - - # Topic config provider class - topic.config.provider.class=com.linkedin.kafka.cruisecontrol.config.KafkaTopicConfigProvider - - # The cluster configurations for the KafkaTopicConfigProvider - cluster.configs.file=config/clusterConfigs.json - - # The maximum time in milliseconds to store the response and access details of a completed user task. - completed.user.task.retention.time.ms=21600000 - - # The maximum time in milliseconds to retain the demotion history of brokers. - demotion.history.retention.time.ms=86400000 - - # The maximum number of completed user tasks for which the response and access details will be cached. - max.cached.completed.user.tasks=100 - - # The maximum number of user tasks for concurrently running in async endpoints across all users. - max.active.user.tasks=5 - - # Enable self healing for all anomaly detectors, unless the particular anomaly detector is explicitly disabled - self.healing.enabled=true - - # Enable self healing for broker failure detector - #self.healing.broker.failure.enabled=true - - # Enable self healing for goal violation detector - #self.healing.goal.violation.enabled=true - - # Enable self healing for metric anomaly detector - #self.healing.metric.anomaly.enabled=true - - - # configurations for the webserver - # ================================ - - # HTTP listen port - webserver.http.port=9090 - - # HTTP listen address - webserver.http.address=0.0.0.0 - - # Whether CORS support is enabled for API or not - webserver.http.cors.enabled=false - - # Value for Access-Control-Allow-Origin - webserver.http.cors.origin=http://localhost:8080/ - - # Value for Access-Control-Request-Method - webserver.http.cors.allowmethods=OPTIONS,GET,POST - - # Headers that should be exposed to the Browser (Webapp) - # This is a special header that is used by the - # User Tasks subsystem and should be explicitly - # Enabled when CORS mode is used as part of the - # Admin Interface - webserver.http.cors.exposeheaders=User-Task-ID - - # REST API default prefix - # (dont forget the ending *) - webserver.api.urlprefix=/kafkacruisecontrol/* - - # Location where the Cruise Control frontend is deployed - webserver.ui.diskpath=./cruise-control-ui/dist/ - - # URL path prefix for UI - # (dont forget the ending *) - webserver.ui.urlprefix=/* - - # Time After which request is converted to Async - webserver.request.maxBlockTimeMs=10000 - - # Default Session Expiry Period - webserver.session.maxExpiryTimeMs=60000 - - # Session cookie path - webserver.session.path=/ - - # Server Access Logs - webserver.accesslog.enabled=true - - # Location of HTTP Request Logs - webserver.accesslog.path=access.log - - # HTTP Request Log retention days - webserver.accesslog.retention.days=14 - - capacityJBOD.json: |- - { - "brokerCapacities":[ - { - "brokerId": "-1", - "capacity": { - "DISK": {"/tmp/kafka-logs-1": "100000", "/tmp/kafka-logs-2": "100000", "/tmp/kafka-logs-3": "50000", - "/tmp/kafka-logs-4": "50000", "/tmp/kafka-logs-5": "150000", "/tmp/kafka-logs-6": "50000"}, - "CPU": "100", - "NW_IN": "10000", - "NW_OUT": "10000" - }, - "doc": "The default capacity for a broker with multiple logDirs each on a separate heterogeneous disk." - }, - { - "brokerId": "0", - "capacity": { - "DISK": {"/tmp/kafka-logs": "500000"}, - "CPU": "100", - "NW_IN": "50000", - "NW_OUT": "50000" - }, - "doc": "This overrides the capacity for broker 0. This broker is not a JBOD broker." - }, - { - "brokerId": "1", - "capacity": { - "DISK": {"/tmp/kafka-logs-1": "250000", "/tmp/kafka-logs-2": "250000"}, - "CPU": "100", - "NW_IN": "50000", - "NW_OUT": "50000" - }, - "doc": "This overrides the capacity for broker 1. This broker is a JBOD broker." - } - ] - } - - capacity.json: |- - { - "brokerCapacities":[ - { - "brokerId": "-1", - "capacity": { - "DISK": "100000", - "CPU": "100", - "NW_IN": "10000", - "NW_OUT": "10000" - }, - "doc": "This is the default capacity. Capacity unit used for disk is in MB, cpu is in percentage, network throughput is in KB." - }, - { - "brokerId": "0", - "capacity": { - "DISK": "500000", - "CPU": "100", - "NW_IN": "50000", - "NW_OUT": "50000" - }, - "doc": "This overrides the capacity for broker 0." - } - ] - } - - clusterConfigs.json: |- - { - "min.insync.replicas": 2 - } - - log4j2.xml: |- - - - - - - - - - - - - - - - log4j.properties: |- - log4j.rootLogger = INFO, FILE - - log4j.appender.FILE=org.apache.log4j.FileAppender - log4j.appender.FILE.File=/dev/stdout - - log4j.appender.FILE.layout=org.apache.log4j.PatternLayout - log4j.appender.FILE.layout.conversionPattern=%-6r [%15.15t] %-5p %30.30c %x - %m%n diff --git a/cruise-control/20kafka-broker-reporter-patch.yml b/cruise-control/20kafka-broker-reporter-patch.yml deleted file mode 100644 index f3305ecd..00000000 --- a/cruise-control/20kafka-broker-reporter-patch.yml +++ /dev/null @@ -1,25 +0,0 @@ -# meant to be applied using -# kubectl --namespace kafka patch statefulset kafka --patch "$(cat cruise-control/20kafka-broker-reporter-patch.yml)" -metadata: - name: kafka - namespace: kafka -spec: - template: - spec: - initContainers: - - name: cruise-control-reporter - image: solsson/kafka-cruise-control@sha256:c70eae329b4ececba58e8cf4fa6e774dd2e0205988d8e5be1a70e622fcc46716 - command: - - /bin/bash - - -cex - - | - cp -v /opt/cruise-control/cruise-control/build/dependant-libs/cruise-control-metrics-reporter.jar /opt/kafka/libs/extensions/cruise-control-metrics-reporter.jar - echo -e "\n\nmetric.reporters = com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter" | tee -a /etc/kafka/server.properties - volumeMounts: - - name: config - mountPath: /etc/kafka - - name: extensions - mountPath: /opt/kafka/libs/extensions - $setElementOrder/initContainers: - - name: init-config - - name: cruise-control-reporter diff --git a/cruise-control/40cruise-control-service.yml b/cruise-control/40cruise-control-service.yml deleted file mode 100644 index dcb8f243..00000000 --- a/cruise-control/40cruise-control-service.yml +++ /dev/null @@ -1,12 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: cruise-control - namespace: kafka -spec: - selector: - app: cruise-control - ports: - - protocol: TCP - port: 8090 - targetPort: 8090 diff --git a/cruise-control/50cruise-control.yml b/cruise-control/50cruise-control.yml deleted file mode 100644 index d2892b8c..00000000 --- a/cruise-control/50cruise-control.yml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cruise-control - namespace: kafka -spec: - selector: - matchLabels: - app: cruise-control - replicas: 1 - template: - metadata: - labels: - app: cruise-control - annotations: - spec: - terminationGracePeriodSeconds: 30 - initContainers: - - name: init-config - image: busybox@sha256:2a03a6059f21e150ae84b0973863609494aad70f0a80eaeb64bddd8d92465812 - command: ['/bin/sh'] - args: [ '-c', 'cp /etc/cruise-control-configmap/* /opt/cruise-control/config'] - volumeMounts: - - name: configmap - mountPath: /etc/cruise-control-configmap - - name: config - mountPath: /opt/cruise-control/config - containers: - - name: cruise-control - image: solsson/kafka-cruise-control@sha256:2c2d113ec3d960bfa75e8e51d1fed6a2f2818e329990f61c895e611403ba64d0 - imagePullPolicy: IfNotPresent - ports: - - name: api - containerPort: 8090 - resources: - requests: - cpu: 100m - memory: 512Mi - readinessProbe: - tcpSocket: - port: 8090 - timeoutSeconds: 1 - volumeMounts: - - name: config - mountPath: /opt/cruise-control/config - volumes: - - name: configmap - configMap: - name: broker-cruise-control-config - - name: config - emptyDir: {} diff --git a/cruise-control/README.md b/cruise-control/README.md deleted file mode 100644 index c27ba2f1..00000000 --- a/cruise-control/README.md +++ /dev/null @@ -1,26 +0,0 @@ -## Cruise Control - -Cruise Control is used to automate the dynamic workload rebalance and self-healing of a Kafka cluster. This tool will allow you to add, replace or remove nodes and the cluster will be automatically adjusted. Partitions will be rebalanced based on resource usage of CPU, network, disk, etc. - -*Disclaimer*: It is important to understand Cruise Control will modify the Kafka cluster without operator intervention. Bugs or misconfiguration may cause loss of data or denial of service. You bear the responsibility of configuring and testing properly and taking precautions based on the importance of your data. - -### Configuration - -There are several configuration files that need to be mounted in `/opt/cruise-control/config`. The files in `11cruise-control-config.yml` are the defaults from [the Cruise Control GitHub repo, migrate_to_kafka_2_0 branch](https://github.com/linkedin/cruise-control/tree/migrate_to_kafka_2_0/config). The significant modification from the GitHub repo is that self healing has been enabled using `self.healing.enabled=true`. - -Following are the files in `11cruise-control-config.yml`. Nearly all changes you would make are in `cruisecontrol.properties`. - -- cruisecontrol.properties -- capacityJBOD.json -- capacity.json -- clusterConfigs.json -- log4j2.xml -- log4j.properties - -### Patching - -Cruise control requires broker metrics to make informed decisions. Each broker runs a metric collector that pushes metrics into a topic, by default named `__CruiseControlMetrics`. Configuring the collector requires patching the broker StatefulSet. An example command to apply this patch is below. - -```shell -$ kubectl --namespace kafka patch statefulset kafka --patch "$(cat cruise-control/20kafka-broker-reporter-patch.yml)" -``` diff --git a/cruise-control/topic-create.yml b/cruise-control/topic-create.yml deleted file mode 100644 index c8730c15..00000000 --- a/cruise-control/topic-create.yml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: topic-cruise-control-metrics - namespace: kafka -spec: - template: - spec: - containers: - - name: topic-create - image: solsson/kafka:native-cli@sha256:fbf29c59182fb87921c5199783d2d5796856ecbfe34a9c03eca658b3cf50f3c4 - command: - - ./bin/kafka-topics.sh - - --zookeeper - - zookeeper.kafka.svc.cluster.local:2181 - - --create - - --if-not-exists - - --topic - - __CruiseControlMetrics - resources: - limits: - cpu: 100m - memory: 20Mi - restartPolicy: Never diff --git a/events-kube/README.md b/events-kube/README.md deleted file mode 100644 index e2eff305..00000000 --- a/events-kube/README.md +++ /dev/null @@ -1,5 +0,0 @@ -## Kubernetes event streaming - -This is meant as input for Ops work and automation. - -Using https://github.com/heptiolabs/eventrouter with Kafka sink. diff --git a/events-kube/config.yml b/events-kube/config.yml deleted file mode 100644 index 2f40f15f..00000000 --- a/events-kube/config.yml +++ /dev/null @@ -1,16 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: eventrouter-cm - namespace: kube-system - annotations: - origin: github.com/Yolean/kubernetes-kafka -data: - config.json: |- - { - "sink": "kafka", - "kafkaBrokers": "bootstrap.kafka:9092", - "kafkaTopic": "ops.kube-events.stream.json", - "kafkaAsync": false, - "kafkaRetryMax": 5 - } diff --git a/events-kube/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml deleted file mode 100644 index 6a90e667..00000000 --- a/events-kube/events-kube-kafka.yml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: eventrouter - namespace: kube-system - labels: - app: eventrouter - annotations: - origin: github.com/Yolean/kubernetes-kafka -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - # prefer duplicate events over missed - maxUnavailable: 0 - maxSurge: 1 - selector: - matchLabels: - app: eventrouter - template: - metadata: - labels: - app: eventrouter - tier: control-plane-addons - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8080" - spec: - serviceAccount: eventrouter - containers: - - name: kube-eventrouter - image: gcr.io/heptio-images/eventrouter@sha256:e613b48c6235426fa334867d661118322b4e1973c23e2e8cf5c066b982cc8596 - resources: - requests: - memory: "5Mi" - cpu: "2m" - limits: - memory: "20Mi" - cpu: "100m" - volumeMounts: - - name: config-volume - mountPath: /etc/eventrouter - volumes: - - name: config-volume - configMap: - name: eventrouter-cm diff --git a/events-kube/rbac/cluster-events-watcher.yml b/events-kube/rbac/cluster-events-watcher.yml deleted file mode 100644 index 3d9ef4d7..00000000 --- a/events-kube/rbac/cluster-events-watcher.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: eventrouter - namespace: kube-system - annotations: - origin: github.com/Yolean/kubernetes-kafka ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: eventrouter - annotations: - origin: github.com/Yolean/kubernetes-kafka -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - get - - watch - - list ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: eventrouter - annotations: - origin: github.com/Yolean/kubernetes-kafka -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: eventrouter -subjects: -- kind: ServiceAccount - name: eventrouter - namespace: kube-system diff --git a/events-kube/topic-create.yaml b/events-kube/topic-create.yaml deleted file mode 100644 index 2cf4f062..00000000 --- a/events-kube/topic-create.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: topic-create-event-kube - namespace: kafka -spec: - template: - spec: - containers: - - name: topic-create - image: solsson/kafka:native-cli@sha256:fbf29c59182fb87921c5199783d2d5796856ecbfe34a9c03eca658b3cf50f3c4 - command: - - ./bin/kafka-topics.sh - - --zookeeper - - zookeeper.kafka.svc.cluster.local:2181 - - --create - - --if-not-exists - - --topic - - ops.kube-events.stream.json - resources: - limits: - cpu: 100m - memory: 20Mi - restartPolicy: Never diff --git a/native/distroless.yaml b/native/distroless.yaml deleted file mode 100644 index e00b5205..00000000 --- a/native/distroless.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# The more specific removes are to make sure that there was a shell that we're removing -- op: remove - path: /spec/template/spec/containers/0/readinessProbe/exec -- op: remove - path: /spec/template/spec/containers/0/readinessProbe -- op: remove - path: /spec/template/spec/containers/0/lifecycle/preStop/exec -- op: remove - path: /spec/template/spec/containers/0/lifecycle/preStop diff --git a/native/kustomization.yaml b/native/kustomization.yaml deleted file mode 100644 index 6a489db7..00000000 --- a/native/kustomization.yaml +++ /dev/null @@ -1,17 +0,0 @@ -bases: -- ../nonroot -patchesStrategicMerge: -- native-image-zookeeper.yaml -patchesJson6902: -- target: - group: apps - version: v1 - kind: StatefulSet - name: pzoo - path: distroless.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: zoo - path: distroless.yaml \ No newline at end of file diff --git a/native/native-image-zookeeper.yaml b/native/native-image-zookeeper.yaml deleted file mode 100644 index 5d38e65d..00000000 --- a/native/native-image-zookeeper.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: pzoo -spec: - template: - spec: - containers: - - name: zookeeper - image: solsson/kafka:native-zookeeper-server-start@sha256:ba3a0632240b8906a3b5bb6441e98ad9d9de73cb716b156ca68f1b435c819e8b - resources: - requests: - cpu: 10m - memory: 25Mi ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: zoo -spec: - template: - spec: - containers: - - name: zookeeper - image: solsson/kafka:native-zookeeper-server-start@sha256:ba3a0632240b8906a3b5bb6441e98ad9d9de73cb716b156ca68f1b435c819e8b - resources: - requests: - cpu: 10m - memory: 25Mi diff --git a/nonroot/entrypoint-from-image.yaml b/nonroot/entrypoint-from-image.yaml deleted file mode 100644 index fe110c35..00000000 --- a/nonroot/entrypoint-from-image.yaml +++ /dev/null @@ -1,2 +0,0 @@ -- op: remove - path: /spec/template/spec/containers/0/command diff --git a/nonroot/fsgroup-65534.yaml b/nonroot/fsgroup-65534.yaml deleted file mode 100644 index 5ebd6cba..00000000 --- a/nonroot/fsgroup-65534.yaml +++ /dev/null @@ -1,4 +0,0 @@ -- op: add - path: /spec/template/spec/securityContext - value: - fsGroup: 65534 diff --git a/nonroot/kustomization.yaml b/nonroot/kustomization.yaml deleted file mode 100644 index a3526dfb..00000000 --- a/nonroot/kustomization.yaml +++ /dev/null @@ -1,44 +0,0 @@ -bases: -- ../rbac-namespace-default -- ../kafka -- ../zookeeper -patchesStrategicMerge: -- nonroot-image-kafka.yaml -- nonroot-image-zookeeper.yaml -patchesJson6902: -- target: - group: apps - version: v1 - kind: StatefulSet - name: kafka - path: fsgroup-65534.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: pzoo - path: fsgroup-65534.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: zoo - path: fsgroup-65534.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: kafka - path: entrypoint-from-image.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: pzoo - path: entrypoint-from-image.yaml -- target: - group: apps - version: v1 - kind: StatefulSet - name: zoo - path: entrypoint-from-image.yaml diff --git a/nonroot/nonroot-image-kafka.yaml b/nonroot/nonroot-image-kafka.yaml deleted file mode 100644 index 8d124f4b..00000000 --- a/nonroot/nonroot-image-kafka.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: kafka -spec: - template: - spec: - initContainers: - - name: init-config - image: solsson/kafka:initutils-nonroot@sha256:8988aca5b34feabe8d7d4e368f74b2ede398f692c7e99a38b262a938d475812c - containers: - - name: broker - image: solsson/kafka:2.5.1-kafka-server-start@sha256:e4d34530e500eb9724e7778595f4cb244df228336090bb7ed5f7dd7433d4d41d - args: - - /etc/kafka/server.properties.$(POD_NAME) diff --git a/nonroot/nonroot-image-zookeeper.yaml b/nonroot/nonroot-image-zookeeper.yaml deleted file mode 100644 index 880a33d2..00000000 --- a/nonroot/nonroot-image-zookeeper.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: pzoo -spec: - template: - spec: - initContainers: - - name: init-config - image: solsson/kafka:initutils-nonroot@sha256:8988aca5b34feabe8d7d4e368f74b2ede398f692c7e99a38b262a938d475812c - containers: - - name: zookeeper - image: solsson/kafka:2.5.1-zookeeper-server-start@sha256:b3af82c547b8188fa303520901eee6a526c6e34d87cfd78c1569a3a2c96ad5cd - args: - - /etc/kafka/zookeeper.properties.scale-$(REPLICAS).$(POD_NAME) ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: zoo -spec: - template: - spec: - initContainers: - - name: init-config - image: solsson/kafka:initutils-nonroot@sha256:8988aca5b34feabe8d7d4e368f74b2ede398f692c7e99a38b262a938d475812c - containers: - - name: zookeeper - image: solsson/kafka:2.5.1-zookeeper-server-start@sha256:b3af82c547b8188fa303520901eee6a526c6e34d87cfd78c1569a3a2c96ad5cd - args: - - /etc/kafka/zookeeper.properties.scale-$(REPLICAS).$(POD_NAME) diff --git a/pixy/Kustomization b/pixy/Kustomization deleted file mode 100644 index 17eaa10f..00000000 --- a/pixy/Kustomization +++ /dev/null @@ -1,3 +0,0 @@ -resources: -- pixy-service.yml -- pixy.yml diff --git a/pixy/pixy-service.yml b/pixy/pixy-service.yml deleted file mode 100644 index c15b7379..00000000 --- a/pixy/pixy-service.yml +++ /dev/null @@ -1,12 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: pixy - namespace: kafka -spec: - selector: - app: pixy - ports: - - name: web - protocol: TCP - port: 80 diff --git a/pixy/pixy.yml b/pixy/pixy.yml deleted file mode 100644 index 190195b4..00000000 --- a/pixy/pixy.yml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: pixy - namespace: kafka - labels: - app: pixy -spec: - replicas: 2 - selector: - matchLabels: - app: pixy - template: - metadata: - labels: - app: pixy - spec: - containers: - - name: pixy - image: mailgun/kafka-pixy:0.17.0@sha256:0b5f4795c0b0d80729fa7415ec70ae4d411e152c6149656dddf01b18184792e0 - ports: - - containerPort: 80 - command: - - kafka-pixy - - -kafkaPeers - - bootstrap.kafka:9092 - - -zookeeperPeers - - zookeeper.kafka:2181 - - -tcpAddr - - 0.0.0.0:80 diff --git a/yahoo-kafka-manager/Kustomization b/yahoo-kafka-manager/Kustomization deleted file mode 100644 index d32e595c..00000000 --- a/yahoo-kafka-manager/Kustomization +++ /dev/null @@ -1,3 +0,0 @@ -resources: -- kafka-manager-service.yml -- kafka-manager.yml diff --git a/yahoo-kafka-manager/kafka-manager-service.yml b/yahoo-kafka-manager/kafka-manager-service.yml deleted file mode 100644 index 3d26adf3..00000000 --- a/yahoo-kafka-manager/kafka-manager-service.yml +++ /dev/null @@ -1,12 +0,0 @@ -kind: Service -apiVersion: v1 -metadata: - name: kafka-manager - namespace: kafka -spec: - selector: - app: kafka-manager - ports: - - protocol: TCP - port: 80 - targetPort: 80 diff --git a/yahoo-kafka-manager/kafka-manager.yml b/yahoo-kafka-manager/kafka-manager.yml deleted file mode 100644 index 16eea0b0..00000000 --- a/yahoo-kafka-manager/kafka-manager.yml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kafka-manager - namespace: kafka -spec: - replicas: 1 - selector: - matchLabels: - app: kafka-manager - template: - metadata: - labels: - app: kafka-manager - spec: - containers: - - name: kafka-manager - image: solsson/kafka-manager@sha256:9da595ecbb733074a1d3c6091a1e0c384da4f4e1f19f4e16276062278da8e592 - ports: - - containerPort: 80 - env: - - name: ZK_HOSTS - value: zookeeper.kafka:2181 - command: - - ./bin/kafka-manager - - -Dhttp.port=80 \ No newline at end of file From 67e222c6ff99b17fc8bd159d8b27fa98bad27e6a Mon Sep 17 00:00:00 2001 From: joyfulbean Date: Mon, 18 Jul 2022 08:59:02 +0000 Subject: [PATCH 3/8] change to my version of broker and zookeeper set --- kafka/Kustomization | 8 +- ...trap-service.yml => bootstrap-service.yml} | 0 ...{10broker-config.yml => broker-config.yml} | 0 kafka/{20dns.yml => dns.yml} | 0 kafka/{50kafka.yml => kafka.yml} | 2 +- 00-namespace.yml => namespace.yml | 0 ...patch.yml => kafka-jmx-exporter-patch.yml} | 0 prometheus/kustomization.yaml | 4 +- ...-metrics-config.yml => metrics-config.yml} | 0 zookeeper/21zoo-service.yml | 15 --- zookeeper/51zoo.yml | 102 ------------------ zookeeper/Kustomization | 10 +- zookeeper/{30service.yml => dns.yml} | 0 .../{20pzoo-service.yml => pzoo-service.yml} | 0 zookeeper/{50pzoo.yml => pzoo.yml} | 2 +- ...{10zookeeper-config.yml => zoo-config.yml} | 13 ++- 16 files changed, 20 insertions(+), 136 deletions(-) rename kafka/{30bootstrap-service.yml => bootstrap-service.yml} (100%) rename kafka/{10broker-config.yml => broker-config.yml} (100%) rename kafka/{20dns.yml => dns.yml} (100%) rename kafka/{50kafka.yml => kafka.yml} (99%) rename 00-namespace.yml => namespace.yml (100%) rename prometheus/{50-kafka-jmx-exporter-patch.yml => kafka-jmx-exporter-patch.yml} (100%) rename prometheus/{10-metrics-config.yml => metrics-config.yml} (100%) delete mode 100644 zookeeper/21zoo-service.yml delete mode 100644 zookeeper/51zoo.yml rename zookeeper/{30service.yml => dns.yml} (100%) rename zookeeper/{20pzoo-service.yml => pzoo-service.yml} (100%) rename zookeeper/{50pzoo.yml => pzoo.yml} (99%) rename zookeeper/{10zookeeper-config.yml => zoo-config.yml} (86%) diff --git a/kafka/Kustomization b/kafka/Kustomization index 0590b6b5..351a643f 100644 --- a/kafka/Kustomization +++ b/kafka/Kustomization @@ -1,5 +1,5 @@ resources: -- 10broker-config.yml -- 20dns.yml -- 30bootstrap-service.yml -- 50kafka.yml +- broker-config.yml +- dns.yml +- bootstrap-service.yml +- kafka.yml diff --git a/kafka/30bootstrap-service.yml b/kafka/bootstrap-service.yml similarity index 100% rename from kafka/30bootstrap-service.yml rename to kafka/bootstrap-service.yml diff --git a/kafka/10broker-config.yml b/kafka/broker-config.yml similarity index 100% rename from kafka/10broker-config.yml rename to kafka/broker-config.yml diff --git a/kafka/20dns.yml b/kafka/dns.yml similarity index 100% rename from kafka/20dns.yml rename to kafka/dns.yml diff --git a/kafka/50kafka.yml b/kafka/kafka.yml similarity index 99% rename from kafka/50kafka.yml rename to kafka/kafka.yml index c04d162a..0a15d729 100644 --- a/kafka/50kafka.yml +++ b/kafka/kafka.yml @@ -8,7 +8,7 @@ spec: matchLabels: app: kafka serviceName: "kafka" - replicas: 5 + replicas: 7 updateStrategy: type: RollingUpdate podManagementPolicy: Parallel diff --git a/00-namespace.yml b/namespace.yml similarity index 100% rename from 00-namespace.yml rename to namespace.yml diff --git a/prometheus/50-kafka-jmx-exporter-patch.yml b/prometheus/kafka-jmx-exporter-patch.yml similarity index 100% rename from prometheus/50-kafka-jmx-exporter-patch.yml rename to prometheus/kafka-jmx-exporter-patch.yml diff --git a/prometheus/kustomization.yaml b/prometheus/kustomization.yaml index 81c00fed..21eac282 100644 --- a/prometheus/kustomization.yaml +++ b/prometheus/kustomization.yaml @@ -4,6 +4,6 @@ bases: #- ../kafka #- ../variants/scale-1 resources: -- 10-metrics-config.yml +- metrics-config.yml patchesStrategicMerge: -- 50-kafka-jmx-exporter-patch.yml +- kafka-jmx-exporter-patch.yml diff --git a/prometheus/10-metrics-config.yml b/prometheus/metrics-config.yml similarity index 100% rename from prometheus/10-metrics-config.yml rename to prometheus/metrics-config.yml diff --git a/zookeeper/21zoo-service.yml b/zookeeper/21zoo-service.yml deleted file mode 100644 index 53beaeb7..00000000 --- a/zookeeper/21zoo-service.yml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: zoo - namespace: kafka -spec: - ports: - - port: 2888 - name: peer - - port: 3888 - name: leader-election - clusterIP: None - selector: - app: zookeeper - storage: persistent-regional diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml deleted file mode 100644 index 52e930f7..00000000 --- a/zookeeper/51zoo.yml +++ /dev/null @@ -1,102 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: zoo - namespace: kafka -spec: - selector: - matchLabels: - app: zookeeper - storage: persistent-regional - serviceName: "zoo" - replicas: 2 - updateStrategy: - type: RollingUpdate - podManagementPolicy: Parallel - template: - metadata: - labels: - app: zookeeper - storage: persistent-regional - annotations: - spec: - terminationGracePeriodSeconds: 10 - initContainers: - - name: init-config - image: solsson/kafka:initutils@sha256:8988aca5b34feabe8d7d4e368f74b2ede398f692c7e99a38b262a938d475812c - command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ID_OFFSET - value: "4" - volumeMounts: - - name: configmap - mountPath: /etc/kafka-configmap - - name: config - mountPath: /etc/kafka - - name: data - mountPath: /var/lib/zookeeper - containers: - - name: zookeeper - image: solsson/kafka:2.5.1@sha256:5c52620bd8e1bcd47805eb8ca285843168e1684aa27f1ae11ce330c3e12f6b0c - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KAFKA_LOG4J_OPTS - value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties - command: - - ./bin/zookeeper-server-start.sh - - /etc/kafka/zookeeper.properties.scale-5.$(POD_NAME) - lifecycle: - preStop: - exec: - command: ["sh", "-ce", "kill -s TERM 1; while $(kill -0 1 2>/dev/null); do sleep 1; done"] - ports: - - containerPort: 2181 - name: client - - containerPort: 2888 - name: peer - - containerPort: 3888 - name: leader-election - resources: - requests: - cpu: 10m - memory: 100Mi - limits: - memory: 120Mi - readinessProbe: - exec: - command: - - /bin/sh - - -c - - '[ "imok" = "$(echo ruok | nc -w 1 -q 1 127.0.0.1 2181)" ]' - timeoutSeconds: 2 - periodSeconds: 30 - volumeMounts: - - name: config - mountPath: /etc/kafka - - name: data - mountPath: /var/lib/zookeeper - volumes: - - name: configmap - configMap: - name: zookeeper-config - - name: config - emptyDir: {} - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi diff --git a/zookeeper/Kustomization b/zookeeper/Kustomization index 978b228a..16255850 100644 --- a/zookeeper/Kustomization +++ b/zookeeper/Kustomization @@ -1,7 +1,5 @@ resources: -- 10zookeeper-config.yml -- 20pzoo-service.yml -- 21zoo-service.yml -- 30service.yml -- 50pzoo.yml -- 51zoo.yml +- zoo-config.yml +- pzoo-service.yml +- dns.yml +- pzoo.yml diff --git a/zookeeper/30service.yml b/zookeeper/dns.yml similarity index 100% rename from zookeeper/30service.yml rename to zookeeper/dns.yml diff --git a/zookeeper/20pzoo-service.yml b/zookeeper/pzoo-service.yml similarity index 100% rename from zookeeper/20pzoo-service.yml rename to zookeeper/pzoo-service.yml diff --git a/zookeeper/50pzoo.yml b/zookeeper/pzoo.yml similarity index 99% rename from zookeeper/50pzoo.yml rename to zookeeper/pzoo.yml index cdaae36b..e79a973e 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/pzoo.yml @@ -9,7 +9,7 @@ spec: app: zookeeper storage: persistent serviceName: "pzoo" - replicas: 3 + replicas: 5 updateStrategy: type: RollingUpdate podManagementPolicy: Parallel diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/zoo-config.yml similarity index 86% rename from zookeeper/10zookeeper-config.yml rename to zookeeper/zoo-config.yml index 57728474..a48eb1d6 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/zoo-config.yml @@ -19,15 +19,15 @@ data: for N in $(seq $PZOO_REPLICAS); do echo "server.$N=pzoo-$(( $N - 1 )).pzoo.$POD_NAMESPACE.svc.cluster.local:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done for N in $(seq $(( $REPLICAS - $PZOO_REPLICAS ))); do echo "server.$(( $PZOO_REPLICAS + $N ))=zoo-$(( $N - 1 )).zoo.$POD_NAMESPACE.svc.cluster.local:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done } - ln -s /etc/kafka/zookeeper.properties /etc/kafka/zookeeper.properties.scale-$REPLICAS.$POD_NAME + ln -s /etc/kafka/zookeeper.properties /etc/kafka/zookeeper.properties.scale-5.$POD_NAME zookeeper.properties: | - 4lw.commands.whitelist=ruok + 4lw.commands.whitelist=stat,ruok,conf,isro,mntr tickTime=2000 dataDir=/var/lib/zookeeper/data dataLogDir=/var/lib/zookeeper/log clientPort=2181 - maxClientCnxns=3 + maxClientCnxns=0 initLimit=5 syncLimit=2 tcpKeepAlive=true @@ -35,8 +35,11 @@ data: server.1=pzoo-0.pzoo:2888:3888:participant server.2=pzoo-1.pzoo:2888:3888:participant server.3=pzoo-2.pzoo:2888:3888:participant - server.4=zoo-0.zoo:2888:3888:participant - server.5=zoo-1.zoo:2888:3888:participant + server.4=pzoo-3.pzoo:2888:3888:participant + server.5=pzoo-4.pzoo:2888:3888:participant + admin.enableServer=false + autopurge.snapRetainCount=3 + autopurge.purgeInterval=24 log4j.properties: |- log4j.rootLogger=INFO, stdout From 9832df4007d5ff84fc6f2164e30214ac4a061b30 Mon Sep 17 00:00:00 2001 From: joyfulbean Date: Tue, 19 Jul 2022 00:43:17 +0000 Subject: [PATCH 4/8] chagne burrow-config --- linkedin-burrow/burrow-config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/linkedin-burrow/burrow-config.yml b/linkedin-burrow/burrow-config.yml index 5fe6dda1..548ab285 100644 --- a/linkedin-burrow/burrow-config.yml +++ b/linkedin-burrow/burrow-config.yml @@ -6,27 +6,27 @@ apiVersion: v1 data: burrow.toml: |- [zookeeper] - servers=[ "zookeeper:2181" ] + servers=[ "pzoo-0:2181", "pzoo-1:2181","pzoo-2:2181","pzoo-3:2181","pzoo-4:2181" ] timeout=6 root-path="/burrow" [cluster.local] class-name="kafka" - servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] + servers=[ "kafka-0", "kafka-1:9092", "kafka-2:9092", "kafka-3:9092", "kafka-4:9092", "kafka-5:9092", "kafka-6:9092" ] topic-refresh=60 offset-refresh=30 [consumer.local] class-name="kafka" cluster="local" - servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] + servers=[ "kafka-0:9092", "kafka-1:9092", "kafka-2:9092", "kafka-3:9092", "kafka-4:9092", "kafka-5:9092", "kafka-6:9092" ] group-blacklist="" group-whitelist="" [consumer.local_zk] class-name="kafka_zk" cluster="local" - servers=[ "zookeeper:2181" ] + servers=[ pzoo-0:2181", "pzoo-1:2181","pzoo-2:2181","pzoo-3:2181","pzoo-4:2181" ] zookeeper-path="/local" zookeeper-timeout=30 group-blacklist="" From 18d14a11576ddaa257482b6057967e0fba1a920e Mon Sep 17 00:00:00 2001 From: joyfulbean Date: Tue, 19 Jul 2022 04:58:19 +0000 Subject: [PATCH 5/8] working burrow --- linkedin-burrow/burrow-config.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/linkedin-burrow/burrow-config.yml b/linkedin-burrow/burrow-config.yml index 548ab285..f1d41ceb 100644 --- a/linkedin-burrow/burrow-config.yml +++ b/linkedin-burrow/burrow-config.yml @@ -6,31 +6,27 @@ apiVersion: v1 data: burrow.toml: |- [zookeeper] - servers=[ "pzoo-0:2181", "pzoo-1:2181","pzoo-2:2181","pzoo-3:2181","pzoo-4:2181" ] + servers=[ "zookeeper:2181" ] timeout=6 root-path="/burrow" - [cluster.local] class-name="kafka" - servers=[ "kafka-0", "kafka-1:9092", "kafka-2:9092", "kafka-3:9092", "kafka-4:9092", "kafka-5:9092", "kafka-6:9092" ] + servers=[ "bootstrap:9092" ] topic-refresh=60 offset-refresh=30 - [consumer.local] class-name="kafka" cluster="local" - servers=[ "kafka-0:9092", "kafka-1:9092", "kafka-2:9092", "kafka-3:9092", "kafka-4:9092", "kafka-5:9092", "kafka-6:9092" ] + servers=[ "bootstrap:9092" ] group-blacklist="" group-whitelist="" - [consumer.local_zk] class-name="kafka_zk" cluster="local" - servers=[ pzoo-0:2181", "pzoo-1:2181","pzoo-2:2181","pzoo-3:2181","pzoo-4:2181" ] + servers=[ "zookeeper:2181" ] zookeeper-path="/local" zookeeper-timeout=30 group-blacklist="" group-whitelist="" - [httpserver.default] address=":8000" From f68ae6d5b5e1b445da879634aa46db7ba9901680 Mon Sep 17 00:00:00 2001 From: joyfulbean Date: Tue, 19 Jul 2022 05:01:44 +0000 Subject: [PATCH 6/8] add CMAK --- kubernetes-kafka | 1 + yahoo-kafka-manager/Kustomization | 3 +++ yahoo-kafka-manager/kafka-manager-service.yml | 12 +++++++++ yahoo-kafka-manager/kafka-manager.yml | 26 +++++++++++++++++++ 4 files changed, 42 insertions(+) create mode 160000 kubernetes-kafka create mode 100644 yahoo-kafka-manager/Kustomization create mode 100644 yahoo-kafka-manager/kafka-manager-service.yml create mode 100644 yahoo-kafka-manager/kafka-manager.yml diff --git a/kubernetes-kafka b/kubernetes-kafka new file mode 160000 index 00000000..9832df40 --- /dev/null +++ b/kubernetes-kafka @@ -0,0 +1 @@ +Subproject commit 9832df4007d5ff84fc6f2164e30214ac4a061b30 diff --git a/yahoo-kafka-manager/Kustomization b/yahoo-kafka-manager/Kustomization new file mode 100644 index 00000000..d32e595c --- /dev/null +++ b/yahoo-kafka-manager/Kustomization @@ -0,0 +1,3 @@ +resources: +- kafka-manager-service.yml +- kafka-manager.yml diff --git a/yahoo-kafka-manager/kafka-manager-service.yml b/yahoo-kafka-manager/kafka-manager-service.yml new file mode 100644 index 00000000..3d26adf3 --- /dev/null +++ b/yahoo-kafka-manager/kafka-manager-service.yml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: kafka-manager + namespace: kafka +spec: + selector: + app: kafka-manager + ports: + - protocol: TCP + port: 80 + targetPort: 80 diff --git a/yahoo-kafka-manager/kafka-manager.yml b/yahoo-kafka-manager/kafka-manager.yml new file mode 100644 index 00000000..16eea0b0 --- /dev/null +++ b/yahoo-kafka-manager/kafka-manager.yml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kafka-manager + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: kafka-manager + template: + metadata: + labels: + app: kafka-manager + spec: + containers: + - name: kafka-manager + image: solsson/kafka-manager@sha256:9da595ecbb733074a1d3c6091a1e0c384da4f4e1f19f4e16276062278da8e592 + ports: + - containerPort: 80 + env: + - name: ZK_HOSTS + value: zookeeper.kafka:2181 + command: + - ./bin/kafka-manager + - -Dhttp.port=80 \ No newline at end of file From 9bc4084ad2e7ce6dcb5802557f389b9b5892442c Mon Sep 17 00:00:00 2001 From: joyfulbean Date: Tue, 19 Jul 2022 05:04:47 +0000 Subject: [PATCH 7/8] delete kubernetes-kafka --- kubernetes-kafka | 1 - 1 file changed, 1 deletion(-) delete mode 160000 kubernetes-kafka diff --git a/kubernetes-kafka b/kubernetes-kafka deleted file mode 160000 index 9832df40..00000000 --- a/kubernetes-kafka +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9832df4007d5ff84fc6f2164e30214ac4a061b30 From 8dc5e34bec57d50a1af23012211c866f8c9dcde4 Mon Sep 17 00:00:00 2001 From: joyfulbean Date: Wed, 20 Jul 2022 12:36:16 +0000 Subject: [PATCH 8/8] add node-exporter --- kafka/bootstrap-service.yml | 1 + prometheus/kafka-jmx-exporter-svc.yml | 16 ++++++ prometheus/node-exporter-daemonset.yml | 53 +++++++++++++++++++ prometheus/node-exporter-svc.yml | 18 +++++++ yahoo-kafka-manager/kafka-manager-service.yml | 2 + yahoo-kafka-manager/kafka-manager.yml | 4 +- 6 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 prometheus/kafka-jmx-exporter-svc.yml create mode 100644 prometheus/node-exporter-daemonset.yml create mode 100644 prometheus/node-exporter-svc.yml diff --git a/kafka/bootstrap-service.yml b/kafka/bootstrap-service.yml index 7c2a3376..ab0b8e84 100644 --- a/kafka/bootstrap-service.yml +++ b/kafka/bootstrap-service.yml @@ -5,6 +5,7 @@ metadata: name: bootstrap namespace: kafka spec: + type: ClusterIP ports: - port: 9092 selector: diff --git a/prometheus/kafka-jmx-exporter-svc.yml b/prometheus/kafka-jmx-exporter-svc.yml new file mode 100644 index 00000000..9d0ec49e --- /dev/null +++ b/prometheus/kafka-jmx-exporter-svc.yml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: kafka-jmx-exporter + name: meteric + namespace: kafka +spec: + ports: + - name: http + port: 5556 + nodePort: 30001 + protocol: TCP + type: NodePort + selector: + k8s-app: kafka-jmx-exporter diff --git a/prometheus/node-exporter-daemonset.yml b/prometheus/node-exporter-daemonset.yml new file mode 100644 index 00000000..41b28ace --- /dev/null +++ b/prometheus/node-exporter-daemonset.yml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-exporter + namespace: kafka + labels: + app: node-exporter +spec: + selector: + matchLabels: + app: node-exporter + template: + metadata: + labels: + app: node-exporter + spec: + hostNetwork: true + hostIPC: true + hostPID: true + containers: + - name: node-exporter + image: prom/node-exporter:v1.0.1 + imagePullPolicy: IfNotPresent + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + resources: + requests: + cpu: 10m + memory: 100Mi + limits: + cpu: 100m + memory: 100Mi + ports: + - name: scrape + containerPort: 9100 + hostPort: 9100 + volumeMounts: + - mountPath: /host/proc + name: proc + readOnly: true + - mountPath: /host/sys + name: sys + readOnly: true + volumes: + - name: proc + hostPath: + path: /proc + type: "" + - name: sys + hostPath: + path: /sys + type: "" diff --git a/prometheus/node-exporter-svc.yml b/prometheus/node-exporter-svc.yml new file mode 100644 index 00000000..bfbc37e4 --- /dev/null +++ b/prometheus/node-exporter-svc.yml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/scrape: "true" + name: node-exporter-http + namespace: kafka + labels: + app: node-exporter +spec: + type: NodePort + selector: + app: node-exporter + ports: + - name: scrape + port: 9100 + nodePort: 30002 + protocol: TCP diff --git a/yahoo-kafka-manager/kafka-manager-service.yml b/yahoo-kafka-manager/kafka-manager-service.yml index 3d26adf3..0a28c12d 100644 --- a/yahoo-kafka-manager/kafka-manager-service.yml +++ b/yahoo-kafka-manager/kafka-manager-service.yml @@ -4,9 +4,11 @@ metadata: name: kafka-manager namespace: kafka spec: + type: NodePort selector: app: kafka-manager ports: - protocol: TCP port: 80 targetPort: 80 + nodePort: 30010 diff --git a/yahoo-kafka-manager/kafka-manager.yml b/yahoo-kafka-manager/kafka-manager.yml index 16eea0b0..5727f031 100644 --- a/yahoo-kafka-manager/kafka-manager.yml +++ b/yahoo-kafka-manager/kafka-manager.yml @@ -20,7 +20,7 @@ spec: - containerPort: 80 env: - name: ZK_HOSTS - value: zookeeper.kafka:2181 + value: zookeeper:2181 command: - ./bin/kafka-manager - - -Dhttp.port=80 \ No newline at end of file + - -Dhttp.port=80