From 0dc5c833125c3d8a60a2d161c3ae1cc85e992c66 Mon Sep 17 00:00:00 2001 From: Fu Chen Date: Tue, 25 Jul 2023 11:58:46 +0800 Subject: [PATCH] spark-3.4 support --- project/CelebornBuild.scala | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index ce7f8157361..10555a723a1 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -36,8 +36,9 @@ object CelebornCommonSettings { val SCALA_2_11_12 = "2.11.12" val SCALA_2_12_10 = "2.12.10" val SCALA_2_12_15 = "2.12.15" + val SCALA_2_12_17 = "2.12.17" val scala213 = "2.13.5" - val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15, scala213) + val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15, SCALA_2_12_17, scala213) val DEFAULT_SCALA_VERSION = SCALA_2_12_15 @@ -173,6 +174,7 @@ object Utils { case Some("spark-3.1") => Some(Spark31) case Some("spark-3.2") => Some(Spark32) case Some("spark-3.3") => Some(Spark33) + case Some("spark-3.4") => Some(Spark34) case _ => None } @@ -409,6 +411,20 @@ object Spark33 extends SparkClientProjects { val zstdJniVersion = "1.5.2-1" } +object Spark34 extends SparkClientProjects { + + val sparkClientProjectPath = "client-spark/spark-3" + val sparkClientProjectName = "celeborn-client-spark-3" + val sparkClientShadeProjectPath = "client-spark/spark-3-shade" + val sparkClientShadeProjectName = "celeborn-client-spark-3-shaded" + + val lz4JavaVersion = "1.8.0" + val sparkProjectScalaVersion = "2.12.17" + + val sparkVersion = "3.4.1" + val zstdJniVersion = "1.5.2-5" +} + trait SparkClientProjects { val sparkClientProjectPath: String