rename spark module name

apache · Oct 15, 2024 · a337a6a · a337a6a
1 parent 7014c43
commit a337a6a
Show file tree

Hide file tree

Showing 45 changed files with 22 additions and 56 deletions.
diff --git a/.idea/icon.png b/.idea/icon.png
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/client-spark/spark-3/pom.xml → client-spark/spark-3-4/pom.xml b/client-spark/spark-3/pom.xml → client-spark/spark-3-4/pom.xml
@@ -24,9 +24,9 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-client-spark-3_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-client-spark-3-4_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
-  <name>Celeborn Client for Spark 3</name>
+  <name>Celeborn Client for Spark 3 and 4</name>
 
   <dependencies>
     <dependency>

diff --git a/...uffle/celeborn/CelebornShuffleDataIO.java → ...uffle/celeborn/CelebornShuffleDataIO.java b/...uffle/celeborn/CelebornShuffleDataIO.java → ...uffle/celeborn/CelebornShuffleDataIO.java
diff --git a/...ffle/celeborn/HashBasedShuffleWriter.java → ...ffle/celeborn/HashBasedShuffleWriter.java b/...ffle/celeborn/HashBasedShuffleWriter.java → ...ffle/celeborn/HashBasedShuffleWriter.java
diff --git a/...ffle/celeborn/SortBasedShuffleWriter.java → ...ffle/celeborn/SortBasedShuffleWriter.java b/...ffle/celeborn/SortBasedShuffleWriter.java → ...ffle/celeborn/SortBasedShuffleWriter.java
diff --git a/...shuffle/celeborn/SparkShuffleManager.java → ...shuffle/celeborn/SparkShuffleManager.java b/...shuffle/celeborn/SparkShuffleManager.java → ...shuffle/celeborn/SparkShuffleManager.java
diff --git a/...he/spark/shuffle/celeborn/SparkUtils.java → ...he/spark/shuffle/celeborn/SparkUtils.java b/...he/spark/shuffle/celeborn/SparkUtils.java → ...he/spark/shuffle/celeborn/SparkUtils.java
diff --git a/...a/org/apache/spark/SparkVersionUtil.scala → ...a/org/apache/spark/SparkVersionUtil.scala b/...a/org/apache/spark/SparkVersionUtil.scala → ...a/org/apache/spark/SparkVersionUtil.scala
diff --git a/...spark/celeborn/ExceptionMakerHelper.scala → ...spark/celeborn/ExceptionMakerHelper.scala b/...spark/celeborn/ExceptionMakerHelper.scala → ...spark/celeborn/ExceptionMakerHelper.scala
diff --git a/...CelebornShuffleFallbackPolicyRunner.scala → ...CelebornShuffleFallbackPolicyRunner.scala b/...CelebornShuffleFallbackPolicyRunner.scala → ...CelebornShuffleFallbackPolicyRunner.scala
diff --git a/...ffle/celeborn/CelebornShuffleHandle.scala → ...ffle/celeborn/CelebornShuffleHandle.scala b/...ffle/celeborn/CelebornShuffleHandle.scala → ...ffle/celeborn/CelebornShuffleHandle.scala
diff --git a/...ffle/celeborn/CelebornShuffleReader.scala → ...ffle/celeborn/CelebornShuffleReader.scala b/...ffle/celeborn/CelebornShuffleReader.scala → ...ffle/celeborn/CelebornShuffleReader.scala
diff --git a/...eborn/CelebornShuffleWriterSuiteBase.java → ...eborn/CelebornShuffleWriterSuiteBase.java b/...eborn/CelebornShuffleWriterSuiteBase.java → ...eborn/CelebornShuffleWriterSuiteBase.java
diff --git a/...eleborn/HashBasedShuffleWriterSuiteJ.java → ...eleborn/HashBasedShuffleWriterSuiteJ.java b/...eleborn/HashBasedShuffleWriterSuiteJ.java → ...eleborn/HashBasedShuffleWriterSuiteJ.java
diff --git a/.../shuffle/celeborn/ShuffleManagerHook.java → .../shuffle/celeborn/ShuffleManagerHook.java b/.../shuffle/celeborn/ShuffleManagerHook.java → .../shuffle/celeborn/ShuffleManagerHook.java
diff --git a/...eleborn/SortBasedShuffleWriterSuiteJ.java → ...eleborn/SortBasedShuffleWriterSuiteJ.java b/...eleborn/SortBasedShuffleWriterSuiteJ.java → ...eleborn/SortBasedShuffleWriterSuiteJ.java
diff --git a/.../celeborn/TestCelebornShuffleManager.java → .../celeborn/TestCelebornShuffleManager.java b/.../celeborn/TestCelebornShuffleManager.java → .../celeborn/TestCelebornShuffleManager.java
diff --git a/...ark-3/src/test/resources/log4j.properties → ...k-3-4/src/test/resources/log4j.properties b/...ark-3/src/test/resources/log4j.properties → ...k-3-4/src/test/resources/log4j.properties
diff --git a/...park-3/src/test/resources/log4j2-test.xml → ...rk-3-4/src/test/resources/log4j2-test.xml b/...park-3/src/test/resources/log4j2-test.xml → ...rk-3-4/src/test/resources/log4j2-test.xml
diff --git a/...eleborn/CelebornShuffleManagerSuite.scala → ...eleborn/CelebornShuffleManagerSuite.scala b/...eleborn/CelebornShuffleManagerSuite.scala → ...eleborn/CelebornShuffleManagerSuite.scala
diff --git a/client-spark/spark-3-shaded/pom.xml b/client-spark/spark-3-shaded/pom.xml
@@ -31,7 +31,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-3_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-3-4_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
   </dependencies>

diff --git a/...-spark/spark-4.0-columnar-shuffle/pom.xml → ...nt-spark/spark-4-columnar-shuffle/pom.xml b/...-spark/spark-4.0-columnar-shuffle/pom.xml → ...nt-spark/spark-4-columnar-shuffle/pom.xml
@@ -24,9 +24,9 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
 
-  <artifactId>celeborn-spark-4.0-columnar-shuffle_${scala.binary.version}</artifactId>
+  <artifactId>celeborn-spark-4-columnar-shuffle_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
-  <name>Celeborn Client for Spark 4.0 Columnar Shuffle</name>
+  <name>Celeborn Client for Spark 4 Columnar Shuffle</name>
 
   <dependencies>
     <dependency>

diff --git a/...eborn/ColumnarHashBasedShuffleWriter.java → ...eborn/ColumnarHashBasedShuffleWriter.java b/...eborn/ColumnarHashBasedShuffleWriter.java → ...eborn/ColumnarHashBasedShuffleWriter.java
diff --git a/...eborn/CelebornColumnarShuffleReader.scala → ...eborn/CelebornColumnarShuffleReader.scala b/...eborn/CelebornColumnarShuffleReader.scala → ...eborn/CelebornColumnarShuffleReader.scala
diff --git a/...ion/columnar/CelebornColumnAccessor.scala → ...ion/columnar/CelebornColumnAccessor.scala b/...ion/columnar/CelebornColumnAccessor.scala → ...ion/columnar/CelebornColumnAccessor.scala
diff --git a/...tion/columnar/CelebornColumnBuilder.scala → ...tion/columnar/CelebornColumnBuilder.scala b/...tion/columnar/CelebornColumnBuilder.scala → ...tion/columnar/CelebornColumnBuilder.scala
diff --git a/...cution/columnar/CelebornColumnStats.scala → ...cution/columnar/CelebornColumnStats.scala b/...cution/columnar/CelebornColumnStats.scala → ...cution/columnar/CelebornColumnStats.scala
diff --git a/...ecution/columnar/CelebornColumnType.scala → ...ecution/columnar/CelebornColumnType.scala b/...ecution/columnar/CelebornColumnType.scala → ...ecution/columnar/CelebornColumnType.scala
diff --git a/...lumnar/CelebornColumnarBatchBuilder.scala → ...lumnar/CelebornColumnarBatchBuilder.scala b/...lumnar/CelebornColumnarBatchBuilder.scala → ...lumnar/CelebornColumnarBatchBuilder.scala
diff --git a/...r/CelebornColumnarBatchCodeGenBuild.scala → ...r/CelebornColumnarBatchCodeGenBuild.scala b/...r/CelebornColumnarBatchCodeGenBuild.scala → ...r/CelebornColumnarBatchCodeGenBuild.scala
diff --git a/...nar/CelebornColumnarBatchSerializer.scala → ...nar/CelebornColumnarBatchSerializer.scala b/...nar/CelebornColumnarBatchSerializer.scala → ...nar/CelebornColumnarBatchSerializer.scala
diff --git a/.../CelebornCompressibleColumnAccessor.scala → .../CelebornCompressibleColumnAccessor.scala b/.../CelebornCompressibleColumnAccessor.scala → .../CelebornCompressibleColumnAccessor.scala
diff --git a/...r/CelebornCompressibleColumnBuilder.scala → ...r/CelebornCompressibleColumnBuilder.scala b/...r/CelebornCompressibleColumnBuilder.scala → ...r/CelebornCompressibleColumnBuilder.scala
diff --git a/.../columnar/CelebornCompressionScheme.scala → .../columnar/CelebornCompressionScheme.scala b/.../columnar/CelebornCompressionScheme.scala → .../columnar/CelebornCompressionScheme.scala
diff --git a/...columnar/CelebornCompressionSchemes.scala → ...columnar/CelebornCompressionSchemes.scala b/...columnar/CelebornCompressionSchemes.scala → ...columnar/CelebornCompressionSchemes.scala
diff --git a/...mnar/CelebornNullableColumnAccessor.scala → ...mnar/CelebornNullableColumnAccessor.scala b/...mnar/CelebornNullableColumnAccessor.scala → ...mnar/CelebornNullableColumnAccessor.scala
diff --git a/...umnar/CelebornNullableColumnBuilder.scala → ...umnar/CelebornNullableColumnBuilder.scala b/...umnar/CelebornNullableColumnBuilder.scala → ...umnar/CelebornNullableColumnBuilder.scala
diff --git a/...ColumnarHashBasedShuffleWriterSuiteJ.java → ...ColumnarHashBasedShuffleWriterSuiteJ.java b/...ColumnarHashBasedShuffleWriterSuiteJ.java → ...ColumnarHashBasedShuffleWriterSuiteJ.java
diff --git a/...uffle/src/test/resources/log4j.properties → ...uffle/src/test/resources/log4j.properties b/...uffle/src/test/resources/log4j.properties → ...uffle/src/test/resources/log4j.properties
diff --git a/...huffle/src/test/resources/log4j2-test.xml → ...huffle/src/test/resources/log4j2-test.xml b/...huffle/src/test/resources/log4j2-test.xml → ...huffle/src/test/resources/log4j2-test.xml
diff --git a/.../CelebornColumnarShuffleReaderSuite.scala → .../CelebornColumnarShuffleReaderSuite.scala b/.../CelebornColumnarShuffleReaderSuite.scala → .../CelebornColumnarShuffleReaderSuite.scala
diff --git a/client-spark/spark-4-shaded/pom.xml b/client-spark/spark-4-shaded/pom.xml
@@ -31,7 +31,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.celeborn</groupId>
-      <artifactId>celeborn-client-spark-3_${scala.binary.version}</artifactId>
+      <artifactId>celeborn-client-spark-3-4_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
   </dependencies>

diff --git a/pom.xml b/pom.xml
@@ -1378,7 +1378,7 @@
       <id>spark-3.0</id>
       <modules>
         <module>client-spark/common</module>
-        <module>client-spark/spark-3</module>
+        <module>client-spark/spark-3-4</module>
         <module>client-spark/spark-3-columnar-common</module>
         <module>client-spark/spark-3-columnar-shuffle</module>
         <module>client-spark/spark-3-shaded</module>
@@ -1398,7 +1398,7 @@
       <id>spark-3.1</id>
       <modules>
         <module>client-spark/common</module>
-        <module>client-spark/spark-3</module>
+        <module>client-spark/spark-3-4</module>
         <module>client-spark/spark-3-columnar-common</module>
         <module>client-spark/spark-3-columnar-shuffle</module>
         <module>client-spark/spark-3-shaded</module>
@@ -1418,7 +1418,7 @@
       <id>spark-3.2</id>
       <modules>
         <module>client-spark/common</module>
-        <module>client-spark/spark-3</module>
+        <module>client-spark/spark-3-4</module>
         <module>client-spark/spark-3-columnar-common</module>
         <module>client-spark/spark-3-columnar-shuffle</module>
         <module>client-spark/spark-3-shaded</module>
@@ -1437,7 +1437,7 @@
       <id>spark-3.3</id>
       <modules>
         <module>client-spark/common</module>
-        <module>client-spark/spark-3</module>
+        <module>client-spark/spark-3-4</module>
         <module>client-spark/spark-3-columnar-common</module>
         <module>client-spark/spark-3-columnar-shuffle</module>
         <module>client-spark/spark-3-shaded</module>
@@ -1475,7 +1475,7 @@
       <id>spark-3.5</id>
       <modules>
         <module>client-spark/common</module>
-        <module>client-spark/spark-3</module>
+        <module>client-spark/spark-3-4</module>
         <module>client-spark/spark-3-columnar-common</module>
         <module>client-spark/spark-3.5-columnar-shuffle</module>
         <module>client-spark/spark-3-shaded</module>
@@ -1494,9 +1494,9 @@
       <id>spark-4.0</id>
       <modules>
         <module>client-spark/common</module>
-        <module>client-spark/spark-3</module>
+        <module>client-spark/spark-3-4</module>
         <module>client-spark/spark-3-columnar-common</module>
-        <module>client-spark/spark-4.0-columnar-shuffle</module>
+        <module>client-spark/spark-4-columnar-shuffle</module>
         <module>client-spark/spark-4-shaded</module>
         <module>tests/spark-it</module>
       </modules>

diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
@@ -691,7 +691,7 @@ object Spark24 extends SparkClientProjects {
 
 object Spark30 extends SparkClientProjects {
 
-  val sparkClientProjectPath = "client-spark/spark-3"
+  val sparkClientProjectPath = "client-spark/spark-3-4"
   val sparkClientProjectName = "celeborn-client-spark-3"
   val sparkClientShadedProjectPath = "client-spark/spark-3-shaded"
   val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded"
@@ -705,7 +705,7 @@ object Spark30 extends SparkClientProjects {
 
 object Spark31 extends SparkClientProjects {
 
-  val sparkClientProjectPath = "client-spark/spark-3"
+  val sparkClientProjectPath = "client-spark/spark-3-4"
   val sparkClientProjectName = "celeborn-client-spark-3"
   val sparkClientShadedProjectPath = "client-spark/spark-3-shaded"
   val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded"
@@ -719,7 +719,7 @@ object Spark31 extends SparkClientProjects {
 
 object Spark32 extends SparkClientProjects {
 
-  val sparkClientProjectPath = "client-spark/spark-3"
+  val sparkClientProjectPath = "client-spark/spark-3-4"
   val sparkClientProjectName = "celeborn-client-spark-3"
   val sparkClientShadedProjectPath = "client-spark/spark-3-shaded"
   val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded"
@@ -733,7 +733,7 @@ object Spark32 extends SparkClientProjects {
 
 object Spark33 extends SparkClientProjects {
 
-  val sparkClientProjectPath = "client-spark/spark-3"
+  val sparkClientProjectPath = "client-spark/spark-3-4"
   val sparkClientProjectName = "celeborn-client-spark-3"
   val sparkClientShadedProjectPath = "client-spark/spark-3-shaded"
   val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded"
@@ -750,7 +750,7 @@ object Spark33 extends SparkClientProjects {
 
 object Spark34 extends SparkClientProjects {
 
-  val sparkClientProjectPath = "client-spark/spark-3"
+  val sparkClientProjectPath = "client-spark/spark-3-4"
   val sparkClientProjectName = "celeborn-client-spark-3"
   val sparkClientShadedProjectPath = "client-spark/spark-3-shaded"
   val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded"
@@ -764,7 +764,7 @@ object Spark34 extends SparkClientProjects {
 
 object Spark35 extends SparkClientProjects {
 
-  val sparkClientProjectPath = "client-spark/spark-3"
+  val sparkClientProjectPath = "client-spark/spark-3-4"
   val sparkClientProjectName = "celeborn-client-spark-3"
   val sparkClientShadedProjectPath = "client-spark/spark-3-shaded"
   val sparkClientShadedProjectName = "celeborn-client-spark-3-shaded"
@@ -780,7 +780,7 @@ object Spark35 extends SparkClientProjects {
 
 object Spark40 extends SparkClientProjects {
 
-  val sparkClientProjectPath = "client-spark/spark-3"
+  val sparkClientProjectPath = "client-spark/spark-3-4"
   val sparkClientProjectName = "celeborn-client-spark-3"
   val sparkClientShadedProjectPath = "client-spark/spark-4-shaded"
   val sparkClientShadedProjectName = "celeborn-client-spark-4-shaded"
@@ -792,7 +792,7 @@ object Spark40 extends SparkClientProjects {
   val zstdJniVersion = "1.5.5-6"
   val scalaBinaryVersion = "2.13"
 
-  override val sparkColumnarShuffleVersion: String = "4.0"
+  override val sparkColumnarShuffleVersion: String = "4"
 }
 
 trait SparkClientProjects {