diff --git a/.github/workflows/sbt.yml b/.github/workflows/sbt.yml new file mode 100644 index 00000000000..c50ff05c6cc --- /dev/null +++ b/.github/workflows/sbt.yml @@ -0,0 +1,163 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +name: Celeborn SBT CI + +on: + push: + branches: + - main + - branch-* + pull_request: + branches: + - main + - branch-* + +jobs: + service: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + java: + - 8 + - 11 + - 17 + steps: + - uses: actions/checkout@v2 + - name: Setup JDK ${{ matrix.java }} + uses: actions/setup-java@v2 + with: + distribution: temurin + java-version: ${{ matrix.java }} + cache: maven + check-latest: false + - name: Test Service with SBT + run: | + build/sbt "clean; test" + - name: Upload test log + if: failure() + uses: actions/upload-artifact@v3 + with: + name: service-${{ matrix.java }}-unit-test-log + path: | + **/target/test-reports/** + + spark2: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + java: + - 8 + spark: + - '2.4' + steps: + - uses: actions/checkout@v2 + - name: Setup JDK ${{ matrix.java }} + uses: actions/setup-java@v2 + with: + distribution: temurin + java-version: ${{ matrix.java }} + cache: maven + check-latest: false + - name: Test with SBT + run: | + build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test" + - name: Upload test log + if: failure() + uses: actions/upload-artifact@v3 + with: + name: spark-${{ matrix.spark }}-unit-test-log + path: | + **/target/test-reports/** + + spark3: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + java: + - 8 + - 11 + - 17 + spark: + - '3.0' + - '3.1' + - '3.2' + - '3.3' + - '3.4' + exclude: + # SPARK-33772: Spark supports JDK 17 since 3.3.0 + - java: 17 + spark: '3.0' + - java: 17 + spark: '3.1' + - java: 17 + spark: '3.2' + steps: + - uses: actions/checkout@v2 + - name: Setup JDK ${{ matrix.java }} + uses: actions/setup-java@v2 + with: + distribution: temurin + java-version: ${{ matrix.java }} + cache: maven + check-latest: false + - name: Test with SBT + run: | + build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test" + - name: Upload test log + if: failure() + uses: actions/upload-artifact@v3 + with: + name: spark-${{ matrix.spark }}-unit-test-log + path: | + **/target/test-reports/** + + flink: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + java: + - 8 + - 11 + flink: + - '1.14' + - '1.15' + - '1.17' + steps: + - uses: actions/checkout@v2 + - name: Setup JDK ${{ matrix.java }} + uses: actions/setup-java@v2 + with: + distribution: temurin + java-version: ${{ matrix.java }} + cache: maven + check-latest: false + - name: Test with SBT + run: | + build/sbt -Pflink-${{ matrix.flink }} "clean; celeborn-flink-group/test" + - name: Upload test log + if: failure() + uses: actions/upload-artifact@v3 + with: + name: flink-${{ matrix.flink }}-unit-test-log + path: | + **/target/test-reports/** diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala index f55aa26f6ec..fbcbbf3a26b 100644 --- a/project/CelebornBuild.scala +++ b/project/CelebornBuild.scala @@ -89,7 +89,23 @@ object CelebornCommonSettings { // Make sure any tests in any project that uses Spark is configured for running well locally Test / javaOptions ++= Seq( - "-Xmx4g" + "-Xmx4g", + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", + "-Dio.netty.tryReflectionSetAccessible=true" ), testOptions += Tests.Argument("-oF"), @@ -125,7 +141,7 @@ object CelebornCommonSettings { lazy val commonUnitTestDependencies = Seq( "org.mockito" % "mockito-core" % "4.11.0" % "test", "org.scalatest" %% "scalatest" % "3.2.16" % "test", - "junit" % "junit" % "4.12" % "test", + "junit" % "junit" % "4.13.2" % "test", // https://www.scala-sbt.org/1.x/docs/Testing.html "com.github.sbt" % "junit-interface" % "0.13.3" % "test") } @@ -489,7 +505,11 @@ trait SparkClientProjects { val sparkVersion: String val zstdJniVersion: String - def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt, sparkClientShade) + def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt, sparkGroup, sparkClientShade) + + // for test only, don't use this group for any other projects + lazy val sparkGroup = (project withId "celeborn-spark-group") + .aggregate(sparkCommon, sparkClient, sparkIt) def sparkCommon: Project = { Project("celeborn-spark-common", file("client-spark/common")) @@ -553,7 +573,7 @@ trait SparkClientProjects { ) ++ commonUnitTestDependencies ) } - + def sparkClientShade: Project = { Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath)) .dependsOn(sparkClient) @@ -657,7 +677,11 @@ trait FlinkClientProjects { lazy val flinkClientsDependency: ModuleID = "org.apache.flink" % "flink-clients" % flinkVersion % "test" lazy val flinkRuntimeWebDependency: ModuleID = "org.apache.flink" % "flink-runtime-web" % flinkVersion % "test" - def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt, flinkClientShade) + def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt, flinkGroup, flinkClientShade) + + // for test only, don't use this group for any other projects + lazy val flinkGroup = (project withId "celeborn-flink-group") + .aggregate(flinkCommon, flinkClient, flinkIt) // get flink major version. e.g: // 1.17.0 -> 1.17