Skip to content

Commit

Permalink
enable ci
Browse files Browse the repository at this point in the history
add flink 1.15/1.17 support

add flink-it/flink-client-shaded support

rebase branch pure-sbt

align final jar name with maven

add prefix and update project name

enabled ci

bump junit

jdk 17

try

set java.io.tmpdir

Revert "try"

This reverts commit 9e78f0d.

try

try

disable parallelExecution

try

upload log

try

try

try

debug info

print debug info

try

fix

remove celeborn-client/test

fail-fast: false

test by group

fix
  • Loading branch information
cfmcgrady committed Aug 1, 2023
1 parent 40e416c commit 0a2018d
Show file tree
Hide file tree
Showing 2 changed files with 192 additions and 5 deletions.
163 changes: 163 additions & 0 deletions .github/workflows/sbt.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


name: Celeborn SBT CI

on:
push:
branches:
- main
- branch-*
pull_request:
branches:
- main
- branch-*

jobs:
service:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
java:
- 8
- 11
- 17
steps:
- uses: actions/checkout@v2
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v2
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: maven
check-latest: false
- name: Test Service with SBT
run: |
build/sbt "clean; test"
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v3
with:
name: service-${{ matrix.java }}-unit-test-log
path: |
**/target/test-reports/**
spark2:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
java:
- 8
spark:
- '2.4'
steps:
- uses: actions/checkout@v2
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v2
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: maven
check-latest: false
- name: Test with SBT
run: |
build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test"
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v3
with:
name: spark-${{ matrix.spark }}-unit-test-log
path: |
**/target/test-reports/**
spark3:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
java:
- 8
- 11
- 17
spark:
- '3.0'
- '3.1'
- '3.2'
- '3.3'
- '3.4'
exclude:
# SPARK-33772: Spark supports JDK 17 since 3.3.0
- java: 17
spark: '3.0'
- java: 17
spark: '3.1'
- java: 17
spark: '3.2'
steps:
- uses: actions/checkout@v2
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v2
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: maven
check-latest: false
- name: Test with SBT
run: |
build/sbt -Pspark-${{ matrix.spark }} "clean; celeborn-spark-group/test"
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v3
with:
name: spark-${{ matrix.spark }}-unit-test-log
path: |
**/target/test-reports/**
flink:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
java:
- 8
- 11
flink:
- '1.14'
- '1.15'
- '1.17'
steps:
- uses: actions/checkout@v2
- name: Setup JDK ${{ matrix.java }}
uses: actions/setup-java@v2
with:
distribution: temurin
java-version: ${{ matrix.java }}
cache: maven
check-latest: false
- name: Test with SBT
run: |
build/sbt -Pflink-${{ matrix.flink }} "clean; celeborn-flink-group/test"
- name: Upload test log
if: failure()
uses: actions/upload-artifact@v3
with:
name: flink-${{ matrix.flink }}-unit-test-log
path: |
**/target/test-reports/**
34 changes: 29 additions & 5 deletions project/CelebornBuild.scala
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,23 @@ object CelebornCommonSettings {

// Make sure any tests in any project that uses Spark is configured for running well locally
Test / javaOptions ++= Seq(
"-Xmx4g"
"-Xmx4g",
"-XX:+IgnoreUnrecognizedVMOptions",
"--add-opens=java.base/java.lang=ALL-UNNAMED",
"--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",
"--add-opens=java.base/java.lang.reflect=ALL-UNNAMED",
"--add-opens=java.base/java.io=ALL-UNNAMED",
"--add-opens=java.base/java.net=ALL-UNNAMED",
"--add-opens=java.base/java.nio=ALL-UNNAMED",
"--add-opens=java.base/java.util=ALL-UNNAMED",
"--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
"--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED",
"--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED",
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
"--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
"--add-opens=java.base/sun.security.action=ALL-UNNAMED",
"--add-opens=java.base/sun.util.calendar=ALL-UNNAMED",
"-Dio.netty.tryReflectionSetAccessible=true"
),

testOptions += Tests.Argument("-oF"),
Expand Down Expand Up @@ -125,7 +141,7 @@ object CelebornCommonSettings {
lazy val commonUnitTestDependencies = Seq(
"org.mockito" % "mockito-core" % "4.11.0" % "test",
"org.scalatest" %% "scalatest" % "3.2.16" % "test",
"junit" % "junit" % "4.12" % "test",
"junit" % "junit" % "4.13.2" % "test",
// https://www.scala-sbt.org/1.x/docs/Testing.html
"com.github.sbt" % "junit-interface" % "0.13.3" % "test")
}
Expand Down Expand Up @@ -489,7 +505,11 @@ trait SparkClientProjects {
val sparkVersion: String
val zstdJniVersion: String

def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt, sparkClientShade)
def modules: Seq[Project] = Seq(sparkCommon, sparkClient, sparkIt, sparkGroup, sparkClientShade)

// for test only, don't use this group for any other projects
lazy val sparkGroup = (project withId "celeborn-spark-group")
.aggregate(sparkCommon, sparkClient, sparkIt)

def sparkCommon: Project = {
Project("celeborn-spark-common", file("client-spark/common"))
Expand Down Expand Up @@ -553,7 +573,7 @@ trait SparkClientProjects {
) ++ commonUnitTestDependencies
)
}

def sparkClientShade: Project = {
Project(sparkClientShadedProjectName, file(sparkClientShadedProjectPath))
.dependsOn(sparkClient)
Expand Down Expand Up @@ -657,7 +677,11 @@ trait FlinkClientProjects {
lazy val flinkClientsDependency: ModuleID = "org.apache.flink" % "flink-clients" % flinkVersion % "test"
lazy val flinkRuntimeWebDependency: ModuleID = "org.apache.flink" % "flink-runtime-web" % flinkVersion % "test"

def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt, flinkClientShade)
def modules: Seq[Project] = Seq(flinkCommon, flinkClient, flinkIt, flinkGroup, flinkClientShade)

// for test only, don't use this group for any other projects
lazy val flinkGroup = (project withId "celeborn-flink-group")
.aggregate(flinkCommon, flinkClient, flinkIt)

// get flink major version. e.g:
// 1.17.0 -> 1.17
Expand Down

0 comments on commit 0a2018d

Please sign in to comment.