Feature/kafka upgrade

charts/akto-protection/templates/deployment.yaml

@@ -71,23 +71,7 @@ spec:
         imagePullPolicy: {{ .Values.runtime.aktoApiSecurityProtection.imagePullPolicy }}
         name: akto-api-security-runtime
         resources: {{- toYaml .Values.runtime.aktoApiSecurityProtection.resources |  nindent 10 }}
-      - env:
-        - name: ZOOKEEPER_CLIENT_PORT
-          value: {{ quote .Values.runtime.zoo1.env.zookeeperClientPort }}
-        - name: ZOOKEEPER_SERVERS
-          value: {{ quote .Values.runtime.zoo1.env.zookeeperServers }}
-        - name: ZOOKEEPER_SERVER_ID
-          value: {{ quote .Values.runtime.zoo1.env.zookeeperServerId }}
-        - name: KAFKA_OPTS
-          value: {{ quote .Values.runtime.zoo1.env.kafkaOpts }}
-        - name: KUBERNETES_CLUSTER_DOMAIN
-          value: {{ quote .Values.kubernetesClusterDomain }}
-        image: {{ .Values.runtime.zoo1.image.repository }}:{{ .Values.runtime.zoo1.image.tag
-          | default .Chart.AppVersion }}
-        name: zoo1
-        ports:
-        - containerPort: 2181
-        resources: {{- toYaml .Values.runtime.zoo1.resources | nindent 10 }}
+
       - env:
         - name: MY_POD_NAMESPACE
           valueFrom:
@@ -105,6 +89,16 @@ spec:
           value: {{ quote .Values.runtime.kafka1.env.kafkaInterBrokerListenerName }}
         - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
           value: {{ quote .Values.runtime.kafka1.env.kafkaListenerSecurityProtocolMap }}
+        - name: KAFKA_PROCESS_ROLES
+          value: {{ quote .Values.runtime.kafka1.env.kafkaProcessRoles }}
+        - name: KAFKA_NODE_ID
+          value: {{ quote .Values.runtime.kafka1.env.kafkaNodeId }}
+        - name: KAFKA_CONTROLLER_QUORUM_VOTERS
+          value: {{ quote .Values.runtime.kafka1.env.kafkaControllerQuorumVoters }}
+        - name: KAFKA_CONTROLLER_LISTENER_NAMES
+          value: {{ quote .Values.runtime.kafka1.env.kafkaControllerListenerNames }}
+        - name: KAFKA_LISTENERS
+          value: {{ quote .Values.runtime.kafka1.env.kafkaListeners }}
         - name: KAFKA_LOG_CLEANER_ENABLE
           value: {{ quote .Values.runtime.kafka1.env.kafkaLogCleanerEnable }}
         - name: KAFKA_LOG_RETENTION_BYTES
@@ -123,8 +117,8 @@ spec:
         - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
           value: {{ quote .Values.runtime.kafka1.env.kafkaTransactionStateLogReplicationFactor
             }}
-        - name: KAFKA_ZOOKEEPER_CONNECT
-          value: {{ quote .Values.runtime.kafka1.env.kafkaZookeeperConnect }}
+        - name: CLUSTER_ID
+          value: {{ quote .Values.runtime.kafka1.env.kafkaClusterId }}
         - name: KUBERNETES_CLUSTER_DOMAIN
           value: {{ quote .Values.kubernetesClusterDomain }}
         image: {{ .Values.runtime.kafka1.image.repository }}:{{ .Values.runtime.kafka1.image.tag
@@ -134,6 +128,7 @@ spec:
         - containerPort: 9092
         - containerPort: 19092
         - containerPort: 29092
+        - containerPort: 9093
         - containerPort: 9999
         resources: {{- toYaml .Values.runtime.kafka1.resources | nindent 10 }}
       restartPolicy: Always

charts/akto-protection/values.yaml

@@ -36,8 +36,12 @@ runtime:
       kafkaCleanupPolicy: delete
       kafkaCreateTopics: akto.api.protection:3:3
       kafkaInterBrokerListenerName: LISTENER_DOCKER_EXTERNAL_LOCALHOST
-      kafkaListenerSecurityProtocolMap: LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT,
-        LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT
+      kafkaListenerSecurityProtocolMap: CONTROLLER:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT
+      kafkaProcessRoles: "broker,controller"
+      kafkaNodeId: "1"
+      kafkaControllerQuorumVoters: "1@localhost:9093"
+      kafkaControllerListenerNames: "CONTROLLER"
+      kafkaListeners: "CONTROLLER://0.0.0.0:9093,LISTENER_DOCKER_EXTERNAL_LOCALHOST://0.0.0.0:29092,LISTENER_DOCKER_EXTERNAL_DIFFHOST://0.0.0.0:9092"
       kafkaLogCleanerEnable: "true"
       kafkaLogRetentionBytes: "10737418240"
       kafkaLogRetentionCheckIntervalMs: "60000"
@@ -46,10 +50,10 @@ runtime:
       kafkaOffsetsTopicReplicationFactor: "1"
       kafkaTransactionStateLogMinIsr: "1"
       kafkaTransactionStateLogReplicationFactor: "1"
-      kafkaZookeeperConnect: 127.0.0.1:2181
+      kafkaClusterId: "c6a1b8e2-4f2a-4b2a-9c3f-1a2b3c4d5e6f"
     image:
       repository: confluentinc/cp-kafka
-      tag: 6.2.11-1-ubi8
+      tag: 8.1.0
     resources:
       requests:
         cpu: "50m"
@@ -62,22 +66,7 @@ runtime:
     targetPort: 9092
   replicas: 1
   type: ClusterIP
-  zoo1:
-    env:
-      kafkaOpts: -Dzookeeper.admin.enableServer=false
-      zookeeperClientPort: "2181"
-      zookeeperServerId: "1"
-      zookeeperServers: zoo1:2888:3888
-    image:
-      repository: confluentinc/cp-zookeeper
-      tag: 6.2.11-1-ubi8
-    resources:
-      requests:
-        cpu: "5m"
-        memory: "500Mi"
-      limits:
-        cpu: 2
-        memory: "4Gi"
+
 
 
 keel:

charts/akto-runtime/templates/deployment.yaml

@@ -71,23 +71,7 @@ spec:
         imagePullPolicy: {{ .Values.runtime.aktoApiSecurityRuntime.imagePullPolicy }}
         name: akto-api-security-runtime
         resources: {{- toYaml .Values.runtime.aktoApiSecurityRuntime.resources |  nindent 10 }}
-      - env:
-        - name: ZOOKEEPER_CLIENT_PORT
-          value: {{ quote .Values.runtime.zoo1.env.zookeeperClientPort }}
-        - name: ZOOKEEPER_SERVERS
-          value: {{ quote .Values.runtime.zoo1.env.zookeeperServers }}
-        - name: ZOOKEEPER_SERVER_ID
-          value: {{ quote .Values.runtime.zoo1.env.zookeeperServerId }}
-        - name: KAFKA_OPTS
-          value: {{ quote .Values.runtime.zoo1.env.kafkaOpts }}
-        - name: KUBERNETES_CLUSTER_DOMAIN
-          value: {{ quote .Values.kubernetesClusterDomain }}
-        image: {{ .Values.runtime.zoo1.image.repository }}:{{ .Values.runtime.zoo1.image.tag
-          | default .Chart.AppVersion }}
-        name: zoo1
-        ports:
-        - containerPort: 2181
-        resources: {{- toYaml .Values.runtime.zoo1.resources | nindent 10 }}
+
       - env:
         - name: MY_POD_NAMESPACE
           valueFrom:
@@ -105,6 +89,16 @@ spec:
           value: {{ quote .Values.runtime.kafka1.env.kafkaInterBrokerListenerName }}
         - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
           value: {{ quote .Values.runtime.kafka1.env.kafkaListenerSecurityProtocolMap }}
+        - name: KAFKA_PROCESS_ROLES
+          value: {{ quote .Values.runtime.kafka1.env.kafkaProcessRoles }}
+        - name: KAFKA_NODE_ID
+          value: {{ quote .Values.runtime.kafka1.env.kafkaNodeId }}
+        - name: KAFKA_CONTROLLER_QUORUM_VOTERS
+          value: {{ quote .Values.runtime.kafka1.env.kafkaControllerQuorumVoters }}
+        - name: KAFKA_CONTROLLER_LISTENER_NAMES
+          value: {{ quote .Values.runtime.kafka1.env.kafkaControllerListenerNames }}
+        - name: KAFKA_LISTENERS
+          value: {{ quote .Values.runtime.kafka1.env.kafkaListeners }}
         - name: KAFKA_LOG_CLEANER_ENABLE
           value: {{ quote .Values.runtime.kafka1.env.kafkaLogCleanerEnable }}
         - name: KAFKA_LOG_RETENTION_BYTES
@@ -123,8 +117,8 @@ spec:
         - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
           value: {{ quote .Values.runtime.kafka1.env.kafkaTransactionStateLogReplicationFactor
             }}
-        - name: KAFKA_ZOOKEEPER_CONNECT
-          value: {{ quote .Values.runtime.kafka1.env.kafkaZookeeperConnect }}
+        - name: CLUSTER_ID
+          value: {{ quote .Values.runtime.kafka1.env.kafkaClusterId }}
         - name: KUBERNETES_CLUSTER_DOMAIN
           value: {{ quote .Values.kubernetesClusterDomain }}
         image: {{ .Values.runtime.kafka1.image.repository }}:{{ .Values.runtime.kafka1.image.tag
@@ -134,6 +128,7 @@ spec:
         - containerPort: 9092
         - containerPort: 19092
         - containerPort: 29092
+        - containerPort: 9093
         - containerPort: 9999
         resources: {{- toYaml .Values.runtime.kafka1.resources | nindent 10 }}
       restartPolicy: Always
@@ -291,6 +286,6 @@ spec:
         name: akto-api-security-threat-backend
         ports:
         - containerPort: 9090
-        resources: {{- toYaml .Values.runtime.zoo1.resources | nindent 10 }}
+        resources: {{- toYaml .Values.threat_client.aktoApiSecurityThreatBackend.resources | nindent 10 }}
       restartPolicy: Always
 {{ end }}

charts/akto-runtime/values.yaml

@@ -40,8 +40,12 @@ runtime:
       kafkaCleanupPolicy: delete
       kafkaCreateTopics: akto.api.logs:3:3
       kafkaInterBrokerListenerName: LISTENER_DOCKER_EXTERNAL_LOCALHOST
-      kafkaListenerSecurityProtocolMap: LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT,
-        LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT
+      kafkaListenerSecurityProtocolMap: CONTROLLER:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT
+      kafkaProcessRoles: "broker,controller"
+      kafkaNodeId: "1"
+      kafkaControllerQuorumVoters: "1@localhost:9093"
+      kafkaControllerListenerNames: "CONTROLLER"
+      kafkaListeners: "CONTROLLER://0.0.0.0:9093,LISTENER_DOCKER_EXTERNAL_LOCALHOST://0.0.0.0:29092,LISTENER_DOCKER_EXTERNAL_DIFFHOST://0.0.0.0:9092"
       kafkaLogCleanerEnable: "true"
       kafkaLogRetentionBytes: "10737418240"
       kafkaLogRetentionCheckIntervalMs: "60000"
@@ -50,10 +54,10 @@ runtime:
       kafkaOffsetsTopicReplicationFactor: "1"
       kafkaTransactionStateLogMinIsr: "1"
       kafkaTransactionStateLogReplicationFactor: "1"
-      kafkaZookeeperConnect: 127.0.0.1:2181
+      kafkaClusterId: "c6a1b8e2-4f2a-4b2a-9c3f-1a2b3c4d5e6f"
     image:
       repository: confluentinc/cp-kafka
-      tag: 6.2.11-1-ubi8
+      tag: 8.1.0
     resources:
       requests:
         cpu: 1
@@ -66,22 +70,6 @@ runtime:
     targetPort: 9092
   replicas: 1
   type: ClusterIP
-  zoo1:
-    env:
-      kafkaOpts: -Dzookeeper.admin.enableServer=false
-      zookeeperClientPort: "2181"
-      zookeeperServerId: "1"
-      zookeeperServers: zoo1:2888:3888
-    image:
-      repository: confluentinc/cp-zookeeper
-      tag: 6.2.11-1-ubi8
-    resources:
-      requests:
-        cpu: 1
-        memory: "2Gi"
-      limits:
-        cpu: 2
-        memory: "4Gi"
 
 keel:
   keel:

charts/akto-setup/templates/deployment.yaml

@@ -225,23 +225,7 @@ spec:
         imagePullPolicy: {{ .Values.runtime.aktoApiSecurityRuntime.imagePullPolicy }}
         name: akto-api-security-runtime
         resources: {{- toYaml .Values.runtime.aktoApiSecurityRuntime.resources |  nindent 10 }}
-      - env:
-        - name: ZOOKEEPER_CLIENT_PORT
-          value: {{ quote .Values.runtime.zoo1.env.zookeeperClientPort }}
-        - name: ZOOKEEPER_SERVERS
-          value: {{ quote .Values.runtime.zoo1.env.zookeeperServers }}
-        - name: ZOOKEEPER_SERVER_ID
-          value: {{ quote .Values.runtime.zoo1.env.zookeeperServerId }}
-        - name: KAFKA_OPTS
-          value: {{ quote .Values.runtime.zoo1.env.kafkaOpts }}
-        - name: KUBERNETES_CLUSTER_DOMAIN
-          value: {{ quote .Values.kubernetesClusterDomain }}
-        image: {{ .Values.runtime.zoo1.image.repository }}:{{ .Values.runtime.zoo1.image.tag
-          | default .Chart.AppVersion }}       
-        name: zoo1
-        ports:
-        - containerPort: 2181
-        resources: {{- toYaml .Values.runtime.zoo1.resources | nindent 10 }}
+
       - env:
         - name: MY_POD_NAMESPACE
           valueFrom:
@@ -259,6 +243,16 @@ spec:
           value: {{ quote .Values.runtime.kafka1.env.kafkaInterBrokerListenerName }}
         - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
           value: {{ quote .Values.runtime.kafka1.env.kafkaListenerSecurityProtocolMap }}
+        - name: KAFKA_PROCESS_ROLES
+          value: {{ quote .Values.runtime.kafka1.env.kafkaProcessRoles }}
+        - name: KAFKA_NODE_ID
+          value: {{ quote .Values.runtime.kafka1.env.kafkaNodeId }}
+        - name: KAFKA_CONTROLLER_QUORUM_VOTERS
+          value: {{ quote .Values.runtime.kafka1.env.kafkaControllerQuorumVoters }}
+        - name: KAFKA_CONTROLLER_LISTENER_NAMES
+          value: {{ quote .Values.runtime.kafka1.env.kafkaControllerListenerNames }}
+        - name: KAFKA_LISTENERS
+          value: {{ quote .Values.runtime.kafka1.env.kafkaListeners }}
         - name: KAFKA_LOG_CLEANER_ENABLE
           value: {{ quote .Values.runtime.kafka1.env.kafkaLogCleanerEnable }}
         - name: KAFKA_LOG_RETENTION_BYTES
@@ -277,8 +271,8 @@ spec:
         - name: KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
           value: {{ quote .Values.runtime.kafka1.env.kafkaTransactionStateLogReplicationFactor
             }}
-        - name: KAFKA_ZOOKEEPER_CONNECT
-          value: {{ quote .Values.runtime.kafka1.env.kafkaZookeeperConnect }}
+        - name: CLUSTER_ID
+          value: {{ quote .Values.runtime.kafka1.env.kafkaClusterId }}
         - name: KUBERNETES_CLUSTER_DOMAIN
           value: {{ quote .Values.kubernetesClusterDomain }}
         image: {{ .Values.runtime.kafka1.image.repository }}:{{ .Values.runtime.kafka1.image.tag
@@ -288,6 +282,7 @@ spec:
         - containerPort: 9092
         - containerPort: 19092
         - containerPort: 29092
+        - containerPort: 9093
         - containerPort: 9999
         resources: {{- toYaml .Values.runtime.kafka1.resources | nindent 10 }}
       restartPolicy: Always

charts/akto-setup/values.yaml

@@ -80,8 +80,12 @@ runtime:
       kafkaCleanupPolicy: delete
       kafkaCreateTopics: akto.api.logs:3:3
       kafkaInterBrokerListenerName: LISTENER_DOCKER_EXTERNAL_LOCALHOST
-      kafkaListenerSecurityProtocolMap: LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT,
-        LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT
+      kafkaListenerSecurityProtocolMap: CONTROLLER:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT
+      kafkaProcessRoles: "broker,controller"
+      kafkaNodeId: "1"
+      kafkaControllerQuorumVoters: "1@localhost:9093"
+      kafkaControllerListenerNames: "CONTROLLER"
+      kafkaListeners: "CONTROLLER://0.0.0.0:9093,LISTENER_DOCKER_EXTERNAL_LOCALHOST://0.0.0.0:29092,LISTENER_DOCKER_EXTERNAL_DIFFHOST://0.0.0.0:9092"
       kafkaLogCleanerEnable: "true"
       kafkaLogRetentionBytes: "10737418240"
       kafkaLogRetentionCheckIntervalMs: "60000"
@@ -90,10 +94,10 @@ runtime:
       kafkaOffsetsTopicReplicationFactor: "1"
       kafkaTransactionStateLogMinIsr: "1"
       kafkaTransactionStateLogReplicationFactor: "1"
-      kafkaZookeeperConnect: 127.0.0.1:2181
+      kafkaClusterId: "c6a1b8e2-4f2a-4b2a-9c3f-1a2b3c4d5e6f"
     image:
       repository: confluentinc/cp-kafka
-      tag: 6.2.11-1-ubi8
+      tag: 8.1.0
     resources:
       requests:
         cpu: 1
@@ -106,22 +110,7 @@ runtime:
     targetPort: 9092
   replicas: 1
   type: ClusterIP
-  zoo1:
-    env:
-      kafkaOpts: -Dzookeeper.admin.enableServer=false
-      zookeeperClientPort: "2181"
-      zookeeperServerId: "1"
-      zookeeperServers: zoo1:2888:3888
-    image:
-      repository: confluentinc/cp-zookeeper
-      tag: 6.2.11-1-ubi8
-    resources:
-      requests:
-        cpu: 1
-        memory: "2Gi"
-      limits:
-        cpu: 2
-        memory: "4Gi"
+
 testing:
   aktoApiSecurityTesting:
     env: