From 2e75aadf7b583c5ff3bdf1efc8bcf12ce7bd65ad Mon Sep 17 00:00:00 2001 From: kipshi <48468934+kipshi@users.noreply.github.com> Date: Mon, 25 Apr 2022 11:42:54 +0800 Subject: [PATCH 1/6] [INLONG-3855][Manager] Add lightweight mode for inlong group (#3878) * move OperateType in UpdateGroupProcessForm to public enums package * Add LightGroupWorkflowDefinition * Add transform_field_mapper * Add utils for NodeInfo * Add nodeInfoUtils --- .../inlong/manager/client/api/util/InlongGroupTransfer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java index 2930ae512b7..6df4781b256 100644 --- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java +++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java @@ -34,6 +34,7 @@ import org.apache.inlong.manager.client.api.auth.Authentication.AuthType; import org.apache.inlong.manager.client.api.auth.SecretTokenAuthentication; import org.apache.inlong.manager.client.api.auth.TokenAuthentication; +import org.apache.inlong.manager.common.enums.GroupMode; import org.apache.inlong.manager.common.enums.GlobalConstants; import org.apache.inlong.manager.common.enums.GroupMode; import org.apache.inlong.manager.common.enums.MQType; From 8a7a30f4d361dc0530551d3d9b96cb0f1fb1a3f5 Mon Sep 17 00:00:00 2001 From: kipshi <48468934+kipshi@users.noreply.github.com> Date: Mon, 25 Apr 2022 11:42:54 +0800 Subject: [PATCH 2/6] [INLONG-3855][Manager] Add lightweight mode for inlong group (#3878) * move OperateType in UpdateGroupProcessForm to public enums package * Add LightGroupWorkflowDefinition * Add transform_field_mapper * Add utils for NodeInfo * Add nodeInfoUtils --- .../inlong/manager/client/api/util/InlongGroupTransfer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java index 2930ae512b7..6df4781b256 100644 --- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java +++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java @@ -34,6 +34,7 @@ import org.apache.inlong.manager.client.api.auth.Authentication.AuthType; import org.apache.inlong.manager.client.api.auth.SecretTokenAuthentication; import org.apache.inlong.manager.client.api.auth.TokenAuthentication; +import org.apache.inlong.manager.common.enums.GroupMode; import org.apache.inlong.manager.common.enums.GlobalConstants; import org.apache.inlong.manager.common.enums.GroupMode; import org.apache.inlong.manager.common.enums.MQType; From 7b7790f798f52f8019ce9e1c58a59f22225f01b1 Mon Sep 17 00:00:00 2001 From: kipshi <48468934+kipshi@users.noreply.github.com> Date: Mon, 25 Apr 2022 11:42:54 +0800 Subject: [PATCH 3/6] [INLONG-3855][Manager] Add lightweight mode for inlong group (#3878) * move OperateType in UpdateGroupProcessForm to public enums package * Add LightGroupWorkflowDefinition * Add transform_field_mapper * Add utils for NodeInfo * Add nodeInfoUtils --- .../inlong/manager/client/api/util/InlongGroupTransfer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java index 2930ae512b7..6df4781b256 100644 --- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java +++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java @@ -34,6 +34,7 @@ import org.apache.inlong.manager.client.api.auth.Authentication.AuthType; import org.apache.inlong.manager.client.api.auth.SecretTokenAuthentication; import org.apache.inlong.manager.client.api.auth.TokenAuthentication; +import org.apache.inlong.manager.common.enums.GroupMode; import org.apache.inlong.manager.common.enums.GlobalConstants; import org.apache.inlong.manager.common.enums.GroupMode; import org.apache.inlong.manager.common.enums.MQType; From 733c0289a55c806ad96903a8a525dbac324381a1 Mon Sep 17 00:00:00 2001 From: ganfengtan Date: Sun, 1 May 2022 10:00:02 +0800 Subject: [PATCH 4/6] [INLONG-4050][sort] Fix parameter definition error of hive load --- .../sort/protocol/node/load/HiveLoadNode.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/load/HiveLoadNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/load/HiveLoadNode.java index c38969c5764..d4f114f8962 100644 --- a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/load/HiveLoadNode.java +++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/load/HiveLoadNode.java @@ -32,7 +32,6 @@ import org.apache.inlong.sort.protocol.transformation.FilterFunction; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.io.Serializable; import java.util.List; import java.util.Map; @@ -62,7 +61,6 @@ public class HiveLoadNode extends LoadNode implements Serializable { private String database; @JsonProperty("hiveConfDir") - @Nonnull private String hiveConfDir; @JsonProperty("hiveVersion") @@ -81,20 +79,20 @@ public HiveLoadNode(@JsonProperty("id") String id, @JsonProperty("fields") List fields, @JsonProperty("fieldRelationShips") List fieldRelationShips, @JsonProperty("filters") List filters, - @Nullable @JsonProperty("sinkParallelism") Integer sinkParallelism, + @JsonProperty("sinkParallelism") Integer sinkParallelism, @JsonProperty("properties") Map properties, @JsonProperty("catalogName") String catalogName, - @Nullable @JsonProperty("database") String database, - @Nullable @JsonProperty("tableName") String tableName, - @Nullable @JsonProperty("hiveConfDir") String hiveConfDir, - @Nullable @JsonProperty("hiveVersion") String hiveVersion, + @JsonProperty("database") String database, + @JsonProperty("tableName") String tableName, + @JsonProperty("hiveConfDir") String hiveConfDir, + @JsonProperty("hiveVersion") String hiveVersion, @JsonProperty("hadoopConfDir") String hadoopConfDir, @JsonProperty("parFields") List partitionFields) { super(id, name, fields, fieldRelationShips, filters, sinkParallelism, properties); this.database = Preconditions.checkNotNull(database, "database of hive is null"); this.tableName = Preconditions.checkNotNull(tableName, "table of hive is null"); - this.hiveConfDir = Preconditions.checkNotNull(hiveConfDir, "hive config path is null"); this.hiveVersion = Preconditions.checkNotNull(hiveVersion, "version of hive is null"); + this.hiveConfDir = hiveConfDir; this.catalogName = catalogName; this.hadoopConfDir = hadoopConfDir; this.partitionFields = partitionFields; @@ -121,11 +119,13 @@ public Map tableOptions() { Map map = super.tableOptions(); map.put("connector", "hive"); map.put("default-database", database); - map.put("hive-conf-dir", hiveConfDir); map.put("hive-version", hiveVersion); if (null != hadoopConfDir) { map.put("hadoop-conf-dir", hadoopConfDir); } + if (null != hiveConfDir) { + map.put("hive-conf-dir", hiveConfDir); + } if (null != partitionFields) { Map properties = super.getProperties(); if (null == properties || !properties.containsKey(trigger)) { From 1a8cd94c4d7b4c0748534f5abe08e898f771a3c5 Mon Sep 17 00:00:00 2001 From: kipshi <48468934+kipshi@users.noreply.github.com> Date: Mon, 25 Apr 2022 11:42:54 +0800 Subject: [PATCH 5/6] [INLONG-3855][Manager] Add lightweight mode for inlong group (#3878) * move OperateType in UpdateGroupProcessForm to public enums package * Add LightGroupWorkflowDefinition * Add transform_field_mapper * Add utils for NodeInfo * Add nodeInfoUtils --- .../inlong/manager/client/api/util/InlongGroupTransfer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java index 2930ae512b7..6df4781b256 100644 --- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java +++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java @@ -34,6 +34,7 @@ import org.apache.inlong.manager.client.api.auth.Authentication.AuthType; import org.apache.inlong.manager.client.api.auth.SecretTokenAuthentication; import org.apache.inlong.manager.client.api.auth.TokenAuthentication; +import org.apache.inlong.manager.common.enums.GroupMode; import org.apache.inlong.manager.common.enums.GlobalConstants; import org.apache.inlong.manager.common.enums.GroupMode; import org.apache.inlong.manager.common.enums.MQType; From 325d41fa3e527097c3dc6a8e792cd197070f701b Mon Sep 17 00:00:00 2001 From: kipshi <48468934+kipshi@users.noreply.github.com> Date: Mon, 25 Apr 2022 11:42:54 +0800 Subject: [PATCH 6/6] [INLONG-3855][Manager] Add lightweight mode for inlong group (#3878) * move OperateType in UpdateGroupProcessForm to public enums package * Add LightGroupWorkflowDefinition * Add transform_field_mapper * Add utils for NodeInfo * Add nodeInfoUtils --- .../inlong/manager/client/api/util/InlongGroupTransfer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java index 2930ae512b7..6df4781b256 100644 --- a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java +++ b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/util/InlongGroupTransfer.java @@ -34,6 +34,7 @@ import org.apache.inlong.manager.client.api.auth.Authentication.AuthType; import org.apache.inlong.manager.client.api.auth.SecretTokenAuthentication; import org.apache.inlong.manager.client.api.auth.TokenAuthentication; +import org.apache.inlong.manager.common.enums.GroupMode; import org.apache.inlong.manager.common.enums.GlobalConstants; import org.apache.inlong.manager.common.enums.GroupMode; import org.apache.inlong.manager.common.enums.MQType;