test by group

apache · Jul 31, 2023 · dc16e50 · dc16e50
1 parent 134cb3e
commit dc16e50
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 6 deletions.
diff --git a/.github/workflows/sbt.yml b/.github/workflows/sbt.yml
@@ -78,7 +78,7 @@ jobs:
           check-latest: false
       - name: Test with SBT
         run: |
-          build/sbt -Pspark-${{ matrix.spark }} "clean; test"
+          build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test"
       - name: Upload test log
         if: failure()
         uses: actions/upload-artifact@v3
@@ -121,7 +121,7 @@ jobs:
         check-latest: false
     - name: Test with SBT
       run: |
-        build/sbt -Pspark-${{ matrix.spark }} "clean; test"
+        build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test"
     - name: Upload test log
       if: failure()
       uses: actions/upload-artifact@v3
@@ -153,7 +153,7 @@ jobs:
           check-latest: false
       - name: Test with SBT
         run: |
-          build/sbt -Pflink-${{ matrix.flink }} "clean; test"
+          build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-flink-group/test"
       - name: Upload test log
         if: failure()
         uses: actions/upload-artifact@v3

diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
@@ -505,7 +505,11 @@ trait SparkClientProjects {
   val sparkVersion: String
   val zstdJniVersion: String
 
-  def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt, sparkClientShade)
+  def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt, sparkGroup, sparkClientShade)
+
+  // for test only, don't use this group for any other projects
+  lazy val sparkGroup = (project withId "celeborn-spark-group")
+    .aggregate(sparkCommon, sparkClient, sparkIt)
 
   def sparkCommon: Project = {
     Project("celeborn-spark-common", file("client-spark/common"))
@@ -569,7 +573,7 @@ trait SparkClientProjects {
         ) ++ commonUnitTestDependencies
       )
   }
-  
+
   def sparkClientShade: Project = {
     Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
       .dependsOn(sparkClient)
@@ -673,7 +677,11 @@ trait FlinkClientProjects {
   lazy val flinkClientsDependency: ModuleID = "org.apache.flink" % "flink-clients" % flinkVersion % "test"
   lazy val flinkRuntimeWebDependency: ModuleID = "org.apache.flink" % "flink-runtime-web" % flinkVersion % "test"
 
-  def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt, flinkClientShade)
+  def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt, flinkGroup, flinkClientShade)
+
+  // for test only, don't use this group for any other projects
+  lazy val flinkGroup = (project withId "celeborn-flink-group")
+    .aggregate(flinkCommon, flinkClient, flinkIt)
 
   // get flink major version. e.g:
   //   1.17.0 -> 1.17