feat: update helm and kubeblocks

Signed-off-by: wangxye <[email protected]>
AutoMQ · Oct 31, 2023 · 2e1747a · 2e1747a
1 parent 34b7327
commit 2e1747a
Show file tree

Hide file tree

Showing 19 changed files with 824 additions and 15 deletions.
diff --git a/rocketmq-k8s-helm/clean-ci.sh b/rocketmq-k8s-helm/clean-ci.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+NAMESPACE=$1
+
+if [[ -z $NAMESPACE ]]; then
+  NAMESPACE="default"
+  echo "info: NAMESPACE is empty, use default namespace: $NAMESPACE"
+fi
+
+helm uninstall s3-localstack --namespace $NAMESPACE
+helm uninstall mysql --namespace $NAMESPACE
+helm uninstall rocketmq-on-s3 --namespace $NAMESPACE
+
+kubectl delete -f deploy/init-db-configmap.yaml --namespace $NAMESPACE
diff --git a/rocketmq-k8s-helm/deploy-ci.sh b/rocketmq-k8s-helm/deploy-ci.sh
@@ -1,18 +1,84 @@
 #!/bin/bash
-# echo $KUBE_CONFIG_DATA | base64 -d > ./config
 
-# DB_SCRIPTS=$1
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
+ROCKETMQ_REPO=$1
+ROCKETMQ_VERSION=$2
+NAMESPACE=$3
+
+if [ $# -lt 2 ]; then
+  echo -e "Usage: sh $0 ROCKETMQ_REPO ROCKETMQ_VERSION\nExample: sh $0 rocketmq-on-s3 latest"
+  exit 1
+fi
+
+if [[ -z $NAMESPACE ]]; then
+  NAMESPACE="default"
+  echo "info: NAMESPACE is empty, use default namespace: $NAMESPACE"
+fi
+
+echo "************************************"
+echo "*          init config...          *"
+echo "************************************"
+
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+REPO_DIR=$(dirname "$(dirname "$SCRIPT_DIR")")
+echo "SCRIPT_DIR: $SCRIPT_DIR"
+echo "REPO_DIR: $REPO_DIR"
+cd "$SCRIPT_DIR" || exit 1
+if [ ! -f "deploy/ddl.sql" ]
+then
+  cp "$REPO_DIR/controller/src/main/resources/ddl.sql" deploy/
+fi
+
+# make ddl to configmap
+if [ ! -f "deploy/init-db-configmap.yaml" ]
+then
+  cp deploy/configmap-template.yaml deploy/init-db-configmap.yaml
+  sed 's/^/    /' deploy/ddl.sql >> deploy/init-db-configmap.yaml
+fi
+
+kubectl apply -f deploy/init-db-configmap.yaml --namespace $NAMESPACE
+
+echo "************************************"
+echo "*     Create env and deploy...     *"
+echo "************************************"
+
+echo ${ROCKETMQ_REPO}:${ROCKETMQ_VERSION}: ${NAMESPACE} deploy start
+
+echo "Deploying s3-localstack..."
 # deploy s3-localstack
 helm repo add localstack-charts https://localstack.github.io/helm-charts
-helm install s3-localstack localstack-charts/localstack -f deploy/localstack_s3.yaml  
-#   --kubeconfig ./config
+helm install s3-localstack localstack-charts/localstack -f deploy/localstack_s3.yaml --namespace $NAMESPACE
+
+# Wait for s3-localstack to be ready
+kubectl rollout status --watch --timeout=120s statefulset/s3-localstack --namespace $NAMESPACE
 
+echo "Deploying MySQL..."
 # deploy mysql
 helm repo add bitnami https://charts.bitnami.com/bitnami
+helm install mysql bitnami/mysql -f deploy/mysql.yaml --namespace $NAMESPACE
+
+# Wait for mysql to be ready
+kubectl rollout status --watch --timeout=120s statefulset/mysql --namespace $NAMESPACE
 
-helm install mysql bitnami/mysql -f deploy/mysql.yaml
+echo "Deploying rocketmq..."
 # deploy rocketmq-on-s3
-helm install rocketmq-on-s3 ../rocketmq-k8s-helm/ -f deploy/helm_sample_values.yaml
+helm install rocketmq-on-s3 . -f deploy/helm_sample_values.yaml --set image.repository=$ROCKETMQ_REPO --set image.tag=$ROCKETMQ_VERSION --namespace $NAMESPACE
 
+kubectl rollout status --watch --timeout=360s statefulset/rocketmq-on-s3-broker --namespace $NAMESPACE
 
+echo "Deployment successful!"
diff --git a/rocketmq-k8s-helm/deploy/configmap-template.yaml b/rocketmq-k8s-helm/deploy/configmap-template.yaml
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql-initdb-config
+data:
+  initdb.sql: |
+    use metadata;
diff --git a/rocketmq-k8s-helm/deploy/ddl.sql b/rocketmq-k8s-helm/deploy/ddl.sql
@@ -46,6 +46,7 @@ CREATE TABLE IF NOT EXISTS topic
     id                   BIGINT       NOT NULL PRIMARY KEY AUTO_INCREMENT,
     name                 VARCHAR(255) NOT NULL,
     queue_num            INT          NOT NULL DEFAULT 0,
+    retention_hours      INT          NOT NULL DEFAULT 72,
     status               TINYINT               DEFAULT 0,
     create_time          DATETIME              DEFAULT current_timestamp,
     update_time          DATETIME              DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,

diff --git a/rocketmq-k8s-helm/deploy/helm_sample_values.yaml b/rocketmq-k8s-helm/deploy/helm_sample_values.yaml
@@ -28,6 +28,6 @@ broker:
 
 
   db:
-    url: "jdbc:mysql://10.1.0.119:3306/metadata"
+    url: "jdbc:mysql://mysql:3306/metadata"
     userName: "root"
-    password: "root"
+    password: "passward"
diff --git a/rocketmq-k8s-helm/deploy/init-db-configmap.yaml b/rocketmq-k8s-helm/deploy/init-db-configmap.yaml
@@ -0,0 +1,200 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mysql-initdb-config
+data:
+  initdb.sql: |
+    use metadata;
+    /*
+     * Licensed to the Apache Software Foundation (ASF) under one or more
+     * contributor license agreements.  See the NOTICE file distributed with
+     * this work for additional information regarding copyright ownership.
+     * The ASF licenses this file to You under the Apache License, Version 2.0
+     * (the "License"); you may not use this file except in compliance with
+     * the License.  You may obtain a copy of the License at
+     *
+     *     http://www.apache.org/licenses/LICENSE-2.0
+     *
+     * Unless required by applicable law or agreed to in writing, software
+     * distributed under the License is distributed on an "AS IS" BASIS,
+     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     * See the License for the specific language governing permissions and
+     * limitations under the License.
+     */
+    CREATE TABLE IF NOT EXISTS lease
+    (
+        node_id         INT      NOT NULL,
+        epoch           INT      NOT NULL DEFAULT 0,
+        expiration_time DATETIME NOT NULL
+    );
+    
+    INSERT INTO lease(node_id, epoch, expiration_time)
+    VALUES (0, 0, TIMESTAMP('2023-01-01'));
+    
+    CREATE TABLE IF NOT EXISTS node
+    (
+        id          INT          NOT NULL PRIMARY KEY AUTO_INCREMENT,
+        name        VARCHAR(255) NOT NULL,
+        instance_id VARCHAR(255),
+        volume_id   VARCHAR(255),
+        hostname    VARCHAR(255),
+        vpc_id      VARCHAR(255),
+        address     VARCHAR(255) NOT NULL,
+        epoch       INT          NOT NULL DEFAULT 0,
+        create_time DATETIME              DEFAULT CURRENT_TIMESTAMP,
+        update_time DATETIME              DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+        UNIQUE INDEX idx_name (name),
+        UNIQUE INDEX idx_host_name (hostname),
+        UNIQUE INDEX idx_address (address)
+    );
+    
+    CREATE TABLE IF NOT EXISTS topic
+    (
+        id                   BIGINT       NOT NULL PRIMARY KEY AUTO_INCREMENT,
+        name                 VARCHAR(255) NOT NULL,
+        queue_num            INT          NOT NULL DEFAULT 0,
+        retention_hours      INT          NOT NULL DEFAULT 72,
+        status               TINYINT               DEFAULT 0,
+        create_time          DATETIME              DEFAULT current_timestamp,
+        update_time          DATETIME              DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+        accept_message_types TEXT         NOT NULL,
+        UNIQUE INDEX idx_topic_name (name)
+    );
+    
+    CREATE TABLE IF NOT EXISTS queue_assignment
+    (
+        topic_id    BIGINT  NOT NULL,
+        queue_id    INT     NOT NULL,
+        src_node_id INT     NOT NULL,
+        dst_node_id INT     NOT NULL,
+        status      TINYINT NOT NULL,
+        create_time DATETIME DEFAULT CURRENT_TIMESTAMP,
+        update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+    );
+    
+    CREATE TABLE IF NOT EXISTS stream
+    (
+        id           BIGINT    NOT NULL PRIMARY KEY AUTO_INCREMENT,
+        topic_id     BIGINT    NOT NULL,
+        queue_id     INT       NOT NULL,
+        stream_role  TINYINT   NOT NULL DEFAULT 0,
+        group_id     BIGINT    NULL,
+        src_node_id  INT,
+        dst_node_id  INT,
+        epoch        BIGINT    NOT NULL DEFAULT -1,
+        range_id     INT       NOT NULL DEFAULT -1,
+        start_offset BIGINT    NOT NULL DEFAULT 0,
+        state        INT       NOT NULL DEFAULT 1,
+        create_time  TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        update_time  TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+        UNIQUE INDEX idx_queue (topic_id, queue_id, group_id, stream_role)
+    );
+    
+    CREATE TABLE IF NOT EXISTS consumer_group
+    (
+        id                   BIGINT       NOT NULL PRIMARY KEY AUTO_INCREMENT,
+        name                 VARCHAR(255) NOT NULL,
+        status               TINYINT      NOT NULL DEFAULT 0,
+        dead_letter_topic_id BIGINT,
+        max_delivery_attempt INT          NOT NULL DEFAULT 16,
+        group_type           TINYINT      NOT NULL,
+        create_time          DATETIME              DEFAULT CURRENT_TIMESTAMP,
+        update_time          DATETIME              DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+        UNIQUE INDEX idx_name (name)
+    );
+    
+    CREATE TABLE IF NOT EXISTS subscription
+    (
+        id          BIGINT       NOT NULL PRIMARY KEY AUTO_INCREMENT,
+        group_id    BIGINT       NOT NULL,
+        topic_id    BIGINT       NOT NULL,
+        expression  VARCHAR(255) NOT NULL,
+        create_time DATETIME DEFAULT CURRENT_TIMESTAMP,
+        update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+        UNIQUE INDEX idx_subscription_group_topic (group_id, topic_id)
+    );
+    
+    CREATE TABLE IF NOT EXISTS group_progress
+    (
+        group_id     BIGINT NOT NULL,
+        topic_id     BIGINT NOT NULL,
+        queue_id     INT    NOT NULL,
+        queue_offset BIGINT NOT NULL,
+        UNIQUE INDEX idx_group_progress (group_id, topic_id, queue_id)
+    );
+    
+    CREATE TABLE IF NOT EXISTS `range`
+    (
+        id           BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT,
+        range_id     INT    NOT NULL,
+        stream_id    BIGINT NOT NULL,
+        epoch        BIGINT NOT NULL,
+        start_offset BIGINT NOT NULL,
+        end_offset   BIGINT NOT NULL,
+        node_id      INT    NOT NULL,
+        UNIQUE INDEX idx_stream_range (stream_id, range_id),
+        INDEX idx_stream_start_offset (stream_id, start_offset)
+    );
+    
+    CREATE TABLE IF NOT EXISTS s3object
+    (
+        id                            BIGINT  NOT NULL PRIMARY KEY,
+        object_size                   BIGINT,
+        stream_id                     BIGINT,
+        prepared_timestamp            TIMESTAMP(3)  NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+        committed_timestamp           TIMESTAMP(3),
+        expired_timestamp             TIMESTAMP(3)  NOT NULL,
+        marked_for_deletion_timestamp TIMESTAMP(3),
+        state                         TINYINT NOT NULL DEFAULT 1
+    );
+    
+    CREATE TABLE IF NOT EXISTS s3streamobject
+    (
+        id                  BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT,
+        stream_id           BIGINT NOT NULL,
+        start_offset        BIGINT NOT NULL,
+        end_offset          BIGINT NOT NULL,
+        object_id           BIGINT NOT NULL,
+        object_size         BIGINT NOT NULL,
+        base_data_timestamp BIGINT,
+        committed_timestamp BIGINT,
+        created_timestamp   BIGINT,
+        UNIQUE INDEX uk_s3_stream_object_object_id (object_id),
+        INDEX idx_s3_stream_object_stream_id (stream_id, start_offset)
+    );
+    
+    CREATE TABLE IF NOT EXISTS s3walobject
+    (
+        object_id           BIGINT   NOT NULL,
+        object_size         BIGINT   NOT NULL,
+        node_id             INT      NOT NULL,
+        sequence_id         BIGINT   NOT NULL,
+        sub_streams         LONGTEXT NOT NULL, -- immutable
+        base_data_timestamp BIGINT,
+        committed_timestamp BIGINT,
+        created_timestamp   BIGINT,
+        UNIQUE INDEX uk_s3_wal_object_node_sequence_id (node_id, sequence_id),
+        INDEX idx_s3_wal_object_object_id (object_id)
+    );
+    
+    CREATE TABLE IF NOT EXISTS sequence (
+        name VARCHAR(255) NOT NULL,
+        next BIGINT NOT NULL DEFAULT 1,
+        UNIQUE INDEX idx_name(name)
+    );
+    INSERT INTO sequence(name) VALUES ('S3_OBJECT_ID_SEQ');
diff --git a/rocketmq-k8s-helm/deploy/localstack_s3.yaml b/rocketmq-k8s-helm/deploy/localstack_s3.yaml
@@ -1,8 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 enableStartupScripts: true
 
 service:
-  type: LoadBalancer
+  type: ClusterIP
 
 startupScriptContent: |
     #!/bin/bash