ci: update kafka config

Jenkinsfile

@@ -284,7 +284,7 @@ pipeline {
                 dir('installer') {
                     script {
                         def images = ['jmeter-master:${JMETER_TAG}',
-                                    'kafka:3.6.1',
+                                    'kafka:3.7.0',
                                     'mysql:8.0.35',
                                     'redis:6.2.6',
                                     'minio:RELEASE.2023-04-13T03-08-07Z',
@@ -327,7 +327,7 @@ pipeline {
                         ${IMAGE_PREFIX}/node-controller:${RELEASE} \\
                         ${IMAGE_PREFIX}/data-streaming:${RELEASE} \\
                         ${IMAGE_PREFIX}/jmeter-master:${JMETER_TAG} \\
-                        ${IMAGE_PREFIX}/kafka:3.6.1 \\
+                        ${IMAGE_PREFIX}/kafka:3.7.0 \\
                         ${IMAGE_PREFIX}/mysql:8.0.35 \\
                         ${IMAGE_PREFIX}/redis:6.2.6 \\
                         ${IMAGE_PREFIX}/minio:RELEASE.2023-04-13T03-08-07Z \\

metersphere/conf/kafka/config/server.properties

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+advertised.listeners=PLAINTEXT_HOST://localhost:9092,PLAINTEXT://broker:19092
+controller.listener.names=CONTROLLER
+group.initial.rebalance.delay.ms=0
+inter.broker.listener.name=PLAINTEXT
+listener.security.protocol.map=PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+log.dirs=/tmp/kraft-combined-logs
+offsets.topic.replication.factor=1
+process.roles=broker
+transaction.state.log.min.isr=1
+transaction.state.log.replication.factor=1
+log.retention.ms=60000
+max.request.size=524288000
+message.max.bytes=524288000
+replica.fetch.max.bytes=524288000
+fetch.message.max_bytes=524288000
+partition.fetch.bytes=524288000

metersphere/docker-compose-kafka.yml

@@ -1,7 +1,7 @@
 version: "2.1"
 services:
   kafka:
-    image: '${MS_IMAGE_PREFIX}/kafka:3.6.1'
+    image: '${MS_IMAGE_PREFIX}/kafka:3.7.0'
     container_name: kafka
     ports:
       - '${MS_KAFKA_PORT}:9092'
@@ -12,21 +12,17 @@ services:
       retries: 50
     restart: always
     environment:
-      ALLOW_PLAINTEXT_LISTENER: 'yes'
-      KAFKA_CFG_LOG_RETENTION_MS: 60000
-      KAFKA_CFG_MAX_REQUEST_SIZE: 524288000
-      KAFKA_CFG_MESSAGE_MAX_BYTES: 524288000
-      KAFKA_CFG_REPLICA_FETCH_MAX_BYTES: 524288000
-      KAFKA_CFG_FETCH_MESSAGE_MAX_BYTES: 524288000
-      KAFKA_CFG_PARTITION_FETCH_BYTES: 524288000
-      KAFKA_HEAP_OPTS: '-Xmx512m -Xms256m'
-      KAFKA_CFG_NODE_ID: 0
-      KAFKA_CFG_PROCESS_ROLES: controller,broker
-      KAFKA_CFG_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
-      KAFKA_CFG_ADVERTISED_LISTENERS: PLAINTEXT://${MS_KAFKA_HOST}:${MS_KAFKA_PORT}
-      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
-      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka:9093
-      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      # Environment variables used by kafka scripts will be needed in case of File input.
+      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
+      # Set properties not provided in the file input
+      KAFKA_NODE_ID: 1
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
+      KAFKA_LISTENERS: 'CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:19092'
+      # Override an existing property
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://${MS_KAFKA_HOST}:${MS_KAFKA_PORT},PLAINTEXT://kafka:19092'
+    volumes:
+      - ${MS_BASE}/metersphere/conf/kafka/config:/mnt/shared/config
     networks:
       - ms-network
   data-streaming: