[improvement](meta) Switch meta serialization to gson 1 (apache#36125)

## Proposed changes Issue Number: Switch meta serialization to gson Contains the following classes： ~~JournalEntity~~ DatabaseInfo ~~PartitionPersistInfo~~ ConsistencyCheckInfo Timestamp BrokerMgr.ModifyBrokerInfo LoadErrorHub.Param TableInfo --------- Co-authored-by: zhangpeicheng <[email protected]>
morningman · Jun 21, 2024 · 869828c · 869828c
1 parent f550945
commit 869828c
Show file tree

Hide file tree

Showing 18 changed files with 393 additions and 94 deletions.
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java b/fe/fe-common/src/main/java/org/apache/doris/common/FeMetaVersion.java
@@ -80,15 +80,18 @@ public final class FeMetaVersion {
     public static final int VERSION_129 = 129;
 
     public static final int VERSION_130 = 130;
+
     // for java-udtf add a bool field to write
     public static final int VERSION_131 = 131;
+
     // For transaction insert
     public static final int VERSION_132 = 132;
     // for expr serde
     public static final int VERSION_133 = 133;
-
+    // For mate gson
+    public static final int VERSION_134 = 134;
     // note: when increment meta version, should assign the latest version to VERSION_CURRENT
-    public static final int VERSION_CURRENT = VERSION_133;
+    public static final int VERSION_CURRENT = VERSION_134;
 
     // all logs meta version should >= the minimum version, so that we could remove many if clause, for example
     // if (FE_METAVERSION < VERSION_94) ...

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabaseQuotaStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterDatabaseQuotaStmt.java
@@ -27,11 +27,19 @@
 import org.apache.doris.qe.ConnectContext;
 
 import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
 
 public class AlterDatabaseQuotaStmt extends DdlStmt {
+    @SerializedName("db")
     private String dbName;
+
+    @SerializedName("qt")
     private QuotaType quotaType;
+
+    @SerializedName("qv")
     private String quotaValue;
+
+    @SerializedName("q")
     private long quota;
 
     public enum QuotaType {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java
@@ -21,6 +21,7 @@
 import org.apache.doris.analysis.ModifyBrokerClause;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.DdlException;
+import org.apache.doris.common.FeMetaVersion;
 import org.apache.doris.common.Pair;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
@@ -29,11 +30,13 @@
 import org.apache.doris.common.proc.ProcResult;
 import org.apache.doris.common.util.NetUtils;
 import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.persist.gson.GsonUtils;
 
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.google.gson.annotations.SerializedName;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -383,7 +386,9 @@ public ProcResult fetchResult() {
     }
 
     public static class ModifyBrokerInfo implements Writable {
+        @SerializedName(value = "n")
         public String brokerName;
+        @SerializedName(value = "a")
         public List<FsBroker> brokerAddresses;
 
         public ModifyBrokerInfo() {
@@ -396,13 +401,20 @@ public ModifyBrokerInfo(String brokerName, List<FsBroker> brokerAddresses) {
 
         @Override
         public void write(DataOutput out) throws IOException {
-            Text.writeString(out, brokerName);
-            out.writeInt(brokerAddresses.size());
-            for (FsBroker address : brokerAddresses) {
-                address.write(out);
+            Text.writeString(out, GsonUtils.GSON.toJson(this));
+        }
+
+        public static ModifyBrokerInfo read(DataInput in) throws IOException {
+            if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_134) {
+                ModifyBrokerInfo modifyBrokerInfo = new ModifyBrokerInfo();
+                modifyBrokerInfo.readFields(in);
+                return modifyBrokerInfo;
+            } else {
+                return GsonUtils.GSON.fromJson(Text.readString(in), ModifyBrokerInfo.class);
             }
         }
 
+        @Deprecated
         public void readFields(DataInput in) throws IOException {
             brokerName = Text.readString(in);
             int size = in.readInt();
@@ -411,11 +423,5 @@ public void readFields(DataInput in) throws IOException {
                 brokerAddresses.add(FsBroker.readIn(in));
             }
         }
-
-        public static ModifyBrokerInfo readIn(DataInput in) throws IOException {
-            ModifyBrokerInfo info = new ModifyBrokerInfo();
-            info.readFields(in);
-            return info;
-        }
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
@@ -46,6 +46,7 @@
 import com.google.gson.JsonPrimitive;
 import com.google.gson.JsonSerializationContext;
 import com.google.gson.JsonSerializer;
+import com.google.gson.annotations.SerializedName;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -59,9 +60,13 @@
 
 public class PartitionKey implements Comparable<PartitionKey>, Writable {
     private static final Logger LOG = LogManager.getLogger(PartitionKey.class);
+    @SerializedName("ks")
     private List<LiteralExpr> keys;
+    @SerializedName("hk")
     private List<String> originHiveKeys;
+    @SerializedName("ts")
     private List<PrimitiveType> types;
+    @SerializedName("isD")
     private boolean isDefaultListPartitionKey = false;
 
     // constructor for partition prune

diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
@@ -224,8 +224,7 @@ public void readFields(DataInput in) throws IOException {
             }
             case OperationType.OP_ALTER_DB:
             case OperationType.OP_RENAME_DB: {
-                data = new DatabaseInfo();
-                ((DatabaseInfo) data).readFields(in);
+                data = DatabaseInfo.read(in);
                 isRead = true;
                 break;
             }
@@ -293,8 +292,7 @@ public void readFields(DataInput in) throws IOException {
             case OperationType.OP_RENAME_TABLE:
             case OperationType.OP_RENAME_ROLLUP:
             case OperationType.OP_RENAME_PARTITION: {
-                data = new TableInfo();
-                ((TableInfo) data).readFields(in);
+                data = TableInfo.read(in);
                 isRead = true;
                 break;
             }
@@ -319,8 +317,7 @@ public void readFields(DataInput in) throws IOException {
                 break;
             }
             case OperationType.OP_FINISH_CONSISTENCY_CHECK: {
-                data = new ConsistencyCheckInfo();
-                ((ConsistencyCheckInfo) data).readFields(in);
+                data = ConsistencyCheckInfo.read(in);
                 isRead = true;
                 break;
             }
@@ -373,8 +370,7 @@ public void readFields(DataInput in) throws IOException {
                 break;
             }
             case OperationType.OP_SET_LOAD_ERROR_HUB: {
-                data = new LoadErrorHub.Param();
-                ((LoadErrorHub.Param) data).readFields(in);
+                data = LoadErrorHub.Param.read(in);
                 isRead = true;
                 break;
             }
@@ -410,8 +406,7 @@ public void readFields(DataInput in) throws IOException {
                 break;
             }
             case OperationType.OP_TIMESTAMP: {
-                data = new Timestamp();
-                ((Timestamp) data).readFields(in);
+                data = Timestamp.read(in);
                 isRead = true;
                 break;
             }
@@ -428,8 +423,7 @@ public void readFields(DataInput in) throws IOException {
             }
             case OperationType.OP_ADD_BROKER:
             case OperationType.OP_DROP_BROKER: {
-                data = new BrokerMgr.ModifyBrokerInfo();
-                ((BrokerMgr.ModifyBrokerInfo) data).readFields(in);
+                data = BrokerMgr.ModifyBrokerInfo.read(in);
                 isRead = true;
                 break;
             }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/Timestamp.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/Timestamp.java
@@ -17,14 +17,21 @@
 
 package org.apache.doris.journal.bdbje;
 
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
 
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 
 // Write this class to bdb periodically
 public class Timestamp implements Writable {
+    @SerializedName("ts")
     private long timestamp;
 
     public Timestamp() {
@@ -37,9 +44,20 @@ public long getTimestamp() {
 
     @Override
     public void write(DataOutput out) throws IOException {
-        out.writeLong(timestamp);
+        Text.writeString(out, GsonUtils.GSON.toJson(this));
+    }
+
+    public static Timestamp read(DataInput in) throws IOException {
+        if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_134) {
+            Timestamp timestamp = new Timestamp();
+            timestamp.readFields(in);
+            return timestamp;
+        } else {
+            return GsonUtils.GSON.fromJson(Text.readString(in), Timestamp.class);
+        }
     }
 
+    @Deprecated
     public void readFields(DataInput in) throws IOException {
         timestamp = in.readLong();
     }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/LoadErrorHub.java b/fe/fe-core/src/main/java/org/apache/doris/load/LoadErrorHub.java
@@ -17,12 +17,16 @@
 
 package org.apache.doris.load;
 
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.FeMetaVersion;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.common.util.PrintableMap;
+import org.apache.doris.persist.gson.GsonUtils;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
+import com.google.gson.annotations.SerializedName;
 
 import java.io.DataInput;
 import java.io.DataOutput;
@@ -32,11 +36,17 @@
 public abstract class LoadErrorHub {
 
     public static class MysqlParam implements Writable {
+        @SerializedName("h")
         private String host;
+        @SerializedName("p")
         private int port;
+        @SerializedName("u")
         private String user;
+        @SerializedName("pwd")
         private String passwd;
+        @SerializedName("db")
         private String db;
+        @SerializedName("tb")
         private String table;
 
         public MysqlParam() {
@@ -90,8 +100,11 @@ public void readFields(DataInput in) throws IOException {
     }
 
     public static class BrokerParam implements Writable {
+        @SerializedName("b")
         private String brokerName;
+        @SerializedName("pa")
         private String path;
+        @SerializedName("pr")
         private Map<String, String> prop = Maps.newHashMap();
 
         // for persist
@@ -137,8 +150,11 @@ public static enum HubType {
     }
 
     public static class Param implements Writable {
+        @SerializedName(value = "t")
         private HubType type;
+        @SerializedName(value = "m")
         private MysqlParam mysqlParam;
+        @SerializedName(value = "b")
         private BrokerParam brokerParam;
 
         // for replay
@@ -148,21 +164,20 @@ public Param() {
 
         @Override
         public void write(DataOutput out) throws IOException {
-            Text.writeString(out, type.name());
-            switch (type) {
-                case MYSQL_TYPE:
-                    mysqlParam.write(out);
-                    break;
-                case BROKER_TYPE:
-                    brokerParam.write(out);
-                    break;
-                case NULL_TYPE:
-                    break;
-                default:
-                    Preconditions.checkState(false, "unknown hub type");
+            Text.writeString(out, GsonUtils.GSON.toJson(this));
+        }
+
+        public static Param read(DataInput in) throws IOException {
+            if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_134) {
+                Param param = new Param();
+                param.readFields(in);
+                return param;
+            } else {
+                return GsonUtils.GSON.fromJson(Text.readString(in), Param.class);
             }
         }
 
+        @Deprecated
         public void readFields(DataInput in) throws IOException {
             type = HubType.valueOf(Text.readString(in));
             switch (type) {