[CELEBORN-850][INFRA] Add SBT CI

.github/workflows/sbt.yml

@@ -0,0 +1,163 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+name: Celeborn SBT CI
+
+on:
+  push:
+    branches:
+      - main
+      - branch-*
+  pull_request:
+    branches:
+      - main
+      - branch-*
+
+jobs:
+  service:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - 8
+          - 11
+          - 17
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup JDK ${{ matrix.java }}
+      uses: actions/setup-java@v2
+      with:
+        distribution: zulu
+        java-version: ${{ matrix.java }}
+        cache: maven
+        check-latest: false
+    - name: Test Service with SBT
+      run: |
+        build/sbt "clean; test"
+    - name: Upload test log
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+          name: service-${{ matrix.java }}-unit-test-log
+          path: |
+              **/target/test-reports/**
+
+  spark2:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - 8
+        spark:
+          - '2.4'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup JDK ${{ matrix.java }}
+        uses: actions/setup-java@v2
+        with:
+          distribution: zulu
+          java-version: ${{ matrix.java }}
+          cache: maven
+          check-latest: false
+      - name: Test with SBT
+        run: |
+          build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test"
+      - name: Upload test log
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+            name: spark-${{ matrix.spark }}-unit-test-log
+            path: |
+                **/target/test-reports/**
+
+  spark3:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - 8
+          - 11
+          - 17
+        spark:
+          - '3.0'
+          - '3.1'
+          - '3.2'
+          - '3.3'
+          - '3.4'
+        exclude:
+          # SPARK-33772: Spark supports JDK 17 since 3.3.0
+          - java: 17
+            spark: '3.0'
+          - java: 17
+            spark: '3.1'
+          - java: 17
+            spark: '3.2'
+    steps:
+    - uses: actions/checkout@v2
+    - name: Setup JDK ${{ matrix.java }}
+      uses: actions/setup-java@v2
+      with:
+        distribution: zulu
+        java-version: ${{ matrix.java }}
+        cache: maven
+        check-latest: false
+    - name: Test with SBT
+      run: |
+        build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test"
+    - name: Upload test log
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+          name: spark-${{ matrix.spark }}-unit-test-log
+          path: |
+              **/target/test-reports/**
+
+  flink:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        java:
+          - 8
+          - 11
+        flink:
+          - '1.14'
+          - '1.15'
+          - '1.17'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup JDK ${{ matrix.java }}
+        uses: actions/setup-java@v2
+        with:
+          distribution: zulu
+          java-version: ${{ matrix.java }}
+          cache: maven
+          check-latest: false
+      - name: Test with SBT
+        run: |
+          build/sbt -Pflink-${{ matrix.flink }} "clean; celeborn-flink-group/test"
+      - name: Upload test log
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+            name: flink-${{ matrix.flink }}-unit-test-log
+            path: |
+                **/target/test-reports/**

project/CelebornBuild.scala

@@ -89,7 +89,23 @@ object CelebornCommonSettings {
 
     // Make sure any tests in any project that uses Spark is configured for running well locally
     Test / javaOptions ++= Seq(
-      "-Xmx4g"
+      "-Xmx4g",
+      "-XX:+IgnoreUnrecognizedVMOptions",
+      "--add-opens=java.base/java.lang=ALL-UNNAMED",
+      "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",
+      "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED",
+      "--add-opens=java.base/java.io=ALL-UNNAMED",
+      "--add-opens=java.base/java.net=ALL-UNNAMED",
+      "--add-opens=java.base/java.nio=ALL-UNNAMED",
+      "--add-opens=java.base/java.util=ALL-UNNAMED",
+      "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
+      "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED",
+      "--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED",
+      "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
+      "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
+      "--add-opens=java.base/sun.security.action=ALL-UNNAMED",
+      "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
+      "-Dio.netty.tryReflectionSetAccessible=true"
     ),
 
     testOptions += Tests.Argument("-oF"),
@@ -125,7 +141,7 @@ object CelebornCommonSettings {
   lazy val commonUnitTestDependencies = Seq(
     "org.mockito" % "mockito-core" % "4.11.0" % "test",
     "org.scalatest" %% "scalatest" % "3.2.16" % "test",
-    "junit" % "junit" % "4.12" % "test",
+    "junit" % "junit" % "4.13.2" % "test",
     // https://www.scala-sbt.org/1.x/docs/Testing.html
     "com.github.sbt" % "junit-interface" % "0.13.3" % "test")
 }
@@ -489,7 +505,11 @@ trait SparkClientProjects {
   val sparkVersion: String
   val zstdJniVersion: String
 
-  def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt, sparkClientShade)
+  def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt, sparkGroup, sparkClientShade)
+
+  // for test only, don't use this group for any other projects
+  lazy val sparkGroup = (project withId "celeborn-spark-group")
+    .aggregate(sparkCommon, sparkClient, sparkIt)
 
   def sparkCommon: Project = {
     Project("celeborn-spark-common", file("client-spark/common"))
@@ -553,7 +573,7 @@ trait SparkClientProjects {
         ) ++ commonUnitTestDependencies
       )
   }
-  
+
   def sparkClientShade: Project = {
     Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
       .dependsOn(sparkClient)
@@ -657,7 +677,11 @@ trait FlinkClientProjects {
   lazy val flinkClientsDependency: ModuleID = "org.apache.flink" % "flink-clients" % flinkVersion % "test"
   lazy val flinkRuntimeWebDependency: ModuleID = "org.apache.flink" % "flink-runtime-web" % flinkVersion % "test"
 
-  def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt, flinkClientShade)
+  def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt, flinkGroup, flinkClientShade)
+
+  // for test only, don't use this group for any other projects
+  lazy val flinkGroup = (project withId "celeborn-flink-group")
+    .aggregate(flinkCommon, flinkClient, flinkIt)
 
   // get flink major version. e.g:
   //   1.17.0 -> 1.17