From 7ebce4ae5f3d5eebb08f23039892de070ba8aa77 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Wed, 9 Oct 2024 14:24:20 +0800 Subject: [PATCH] [branch-2.1][chore](dependencies)upgrade fe dependencies (#41142) ## Proposed changes upgrade commons-configuration2 to 2.11.0 upgrade logging-interceptor to 4.12.0 upgrade commons-compress to 1.27.1 upgrade jetty-bom to 9.4.56.v20240826 upgrade azure-sdk to 1.2.27 Iceberg depends on configuration2, and configuration2 relies on a newer version of commons-lang3. However, there were significant breaking changes in commons-lang3, which made it incompatible.https://issues.apache.org/jira/browse/LANG-1705 As a result, I rewrote the clone method. (cherry picked from commit 945edf8dbffaa25c987bcefad59b6cde52772d4f) --- .../doris/common/util/SerializationUtils.java | 71 +++++++++++ .../java/org/apache/doris/qe/VariableMgr.java | 2 +- .../common/util/SerializationUtilsTest.java | 110 ++++++++++++++++++ .../iceberg/IcebergTransactionTest.java | 2 +- fe/pom.xml | 45 ++++--- 5 files changed, 215 insertions(+), 15 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/SerializationUtils.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/util/SerializationUtilsTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/SerializationUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/SerializationUtils.java new file mode 100644 index 00000000000000..cc18bcb0a10a60 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/SerializationUtils.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; + +/** + * A utility class for serializing and deep cloning Serializable objects. + *

+ * This class provides a method to clone an object by serializing it to a byte array + * and then deserializing it back to an object. This approach ensures that a deep copy + * is created, meaning that all objects referenced by the original object are also cloned. + *

+ * Note: The object to be cloned must implement the {@link Serializable} interface. + */ +public class SerializationUtils { + + /** + * Clones a Serializable object by serializing and deserializing it. + * + * @param object the object to be cloned. Must be Serializable. + * @param the type of the object to clone. Must extend Serializable. + * @return a deep copy of the provided object, or null if the input object is null. + * @throws RuntimeException if cloning fails due to I/O or class not found exceptions. + */ + public static T clone(final T object) { + // Check if the object is null; if true, return null + if (object == null) { + return null; // Return null for null input + } + + try { + // Serialize the object to a byte array + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); // Output stream to hold the serialized data + ObjectOutputStream oos = new ObjectOutputStream(baos)) { // ObjectOutputStream for serialization + oos.writeObject(object); // Write the object to the output stream + oos.flush(); // Ensure all data is written + + // Deserialize the byte array back to an object + // Input stream from byte array + try (ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); + ObjectInputStream ois = new ObjectInputStream(bais)) { // ObjectInputStream for deserialization + return (T) ois.readObject(); // Read and return the cloned object + } + } + } catch (IOException | ClassNotFoundException e) { + // Wrap any exceptions thrown during serialization/deserialization + throw new RuntimeException("Cloning failed", e); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java index 3515e9e0861ba0..f8c3a777e1b419 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java @@ -31,6 +31,7 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.PatternMatcher; import org.apache.doris.common.VariableAnnotation; +import org.apache.doris.common.util.SerializationUtils; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.persist.GlobalVarPersistInfo; @@ -40,7 +41,6 @@ import com.google.common.collect.ImmutableSortedMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.similarity.JaroWinklerDistance; import org.apache.logging.log4j.LogManager; diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/SerializationUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/SerializationUtilsTest.java new file mode 100644 index 00000000000000..345fa4c5060a0d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/SerializationUtilsTest.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Objects; + +public class SerializationUtilsTest { + + + private final HashMap sampleMap = new HashMap<>(); + private final String sampleString = "sampleString"; + private final Integer sampleInteger = 456; + + public SerializationUtilsTest() { + sampleMap.put("KEY_ONE", sampleString); + sampleMap.put("KEY_TWO", sampleInteger); + } + + @Test + public void testClone() { + final Object clonedObject = SerializationUtils.clone(sampleMap); + Assertions.assertNotNull(clonedObject); + Assertions.assertTrue(clonedObject instanceof HashMap); + Assertions.assertNotSame(clonedObject, sampleMap); + + final HashMap clonedMap = (HashMap) clonedObject; + Assertions.assertEquals(sampleString, clonedMap.get("KEY_ONE")); + Assertions.assertNotSame(sampleString, clonedMap.get("KEY_ONE")); + Assertions.assertEquals(sampleInteger, clonedMap.get("KEY_TWO")); + Assertions.assertNotSame(sampleInteger, clonedMap.get("KEY_TWO")); + Assertions.assertEquals(sampleMap, clonedMap); + } + + @Test + public void testCloneNull() { + final Object clonedObject = SerializationUtils.clone(null); + Assertions.assertNull(clonedObject); + } + + @Test + public void testCloneWithWriteReplace() { + Child childObject = new Child(true); + Parent clonedParent = SerializationUtils.clone(childObject); + + Assertions.assertNotSame(childObject, clonedParent); + Assertions.assertEquals(new Parent(true), clonedParent); + } + + static class Parent implements Serializable { + private static final long serialVersionUID = 1L; + protected boolean status; + + Parent(boolean status) { + this.status = status; + } + + protected Parent(Parent parent) { + this.status = parent.status; + } + + protected Object writeReplace() { + return new Parent(this); + } + + @Override + public int hashCode() { + return Objects.hash(status); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Parent other = (Parent) obj; + return status == other.status; + } + } + + static class Child extends Parent { + private static final long serialVersionUID = 2L; + + Child(boolean status) { + super(status); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index 432cc47f307821..66c3ea197101d1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.info.SimpleTableInfo; +import org.apache.doris.common.util.SerializationUtils; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.nereids.trees.plans.commands.insert.IcebergInsertCommandContext; import org.apache.doris.thrift.TFileContent; @@ -27,7 +28,6 @@ import com.google.common.collect.Maps; import mockit.Mock; import mockit.MockUp; -import org.apache.commons.lang3.SerializationUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.FileScanTask; diff --git a/fe/pom.xml b/fe/pom.xml index 5e7cc18f46e34b..2a265b1ab56f7a 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -242,9 +242,10 @@ under the License. 2.2 1.4 1.5 + 2.11.0 1.13 2.6 - 3.9 + 3.17.0 2.2 1.5.1 1.10.0 @@ -290,11 +291,11 @@ under the License. 1.11-8 1.0.1 6.7.2 - 4.7.2 + 4.12.0 4.9.3 3.4.0 2.0 - 1.1.0.Final + 2.0.1.Final 0.2.3 3.4.0 6.4.5 @@ -325,7 +326,7 @@ under the License. 1.13.1 3.2.2 - 1.22 + 1.27.1 2.12.10 4.0.2 1.18.24 @@ -336,8 +337,11 @@ under the License. 1.12.669 3.0.9 3.3.6 + 1.2.0 + 1.1.1 2.4.9 - 4.9.3 + 4.1.7 + 4.13.1 2.8.1 github 2.7.18 @@ -350,7 +354,7 @@ under the License. 6.5.1 2.0.3 1.5.4 - 9.4.55.v20240627 + 9.4.56.v20240826 2.9.3 2.5.2 75.1 @@ -411,13 +415,6 @@ under the License. - - com.fasterxml.jackson - jackson-bom - ${jackson.version} - import - pom - io.netty netty-bom @@ -432,6 +429,13 @@ under the License. pom import + + com.fasterxml.jackson + jackson-bom + ${jackson.version} + import + pom + org.apache.ivy ivy @@ -609,6 +613,16 @@ under the License. + + org.apache.hadoop.thirdparty + hadoop-shaded-guava + ${hadoop.thirdparty.guava.version} + + + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_7 + ${hadoop.thirdparty.protobuf_3_7.version} + org.apache.hadoop hadoop-auth @@ -705,6 +719,11 @@ under the License. commons-fileupload ${commons-filerupload.version} + + org.apache.commons + commons-configuration2 + ${commons-configuration2.version} +