[GLUTEN-4675][CH] Support write mergetree to s3 (#4676)

What changes were proposed in this pull request? support write mergetree on s3 and hdfs How was this patch tested? UT --------- Co-authored-by: liuneng1994 <[email protected]> Co-authored-by: liuneng <[email protected]>
apache · Mar 22, 2024 · 4855a4d · 4855a4d
1 parent 4a953a2
commit 4855a4d
Show file tree

Hide file tree

Showing 25 changed files with 2,008 additions and 36 deletions.
diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
@@ -19,6 +19,12 @@
       <artifactId>gluten-core</artifactId>
       <version>${project.version}</version>
       <scope>compile</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>guava</artifactId>
+          <groupId>com.google.guava</groupId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>io.glutenproject</groupId>
@@ -50,6 +56,16 @@
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-yarn_${scala.binary.version}</artifactId>
       <scope>provided</scope>
+      <exclusions>
+        <exclusion>
+          <artifactId>hadoop-client-api</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+        <exclusion>
+          <artifactId>hadoop-client-runtime</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
@@ -138,7 +154,11 @@
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-exec</artifactId>
           </exclusion>
-	    </exclusions>
+          <exclusion>
+            <artifactId>guava</artifactId>
+            <groupId>com.google.guava</groupId>
+          </exclusion>
+        </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -181,6 +201,12 @@
       <version>1.11.901</version>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>io.minio</groupId>
+      <artifactId>minio</artifactId>
+      <version>8.5.9</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>