From bec8ea9343cd057da303ab86dc08704c09421c37 Mon Sep 17 00:00:00 2001 From: davidyuan Date: Sun, 26 Nov 2023 16:20:40 +0800 Subject: [PATCH 1/7] support sbt --- build/sbt | 149 +++++++++++++++++++ build/sbt-launch-lib.bash | 193 +++++++++++++++++++++++++ build/util.sh | 13 +- project/Build.scala | 295 ++++++++++++++++++++++++++++++++++++++ project/build.properties | 1 + project/plugin.sbt | 9 ++ 6 files changed, 653 insertions(+), 7 deletions(-) create mode 100755 build/sbt create mode 100644 build/sbt-launch-lib.bash mode change 100755 => 100644 build/util.sh create mode 100644 project/Build.scala create mode 100644 project/build.properties create mode 100644 project/plugin.sbt diff --git a/build/sbt b/build/sbt new file mode 100755 index 00000000000..db9d3b345ff --- /dev/null +++ b/build/sbt @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SELF=$(cd $(dirname $0) && pwd) +. "$SELF/util.sh" + +# When creating new tests for Spark SQL Hive, the HADOOP_CLASSPATH must contain the hive jars so +# that we can run Hive to generate the golden answer. This is not required for normal development +# or testing. +if [ -n "$HIVE_HOME" ]; then + for i in "$HIVE_HOME"/lib/* + do HADOOP_CLASSPATH="$HADOOP_CLASSPATH:$i" + done + export HADOOP_CLASSPATH +fi + +. "$(dirname "$(realpath "$0")")"/sbt-launch-lib.bash + + +declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" +declare -r sbt_opts_file=".sbtopts" +declare -r etc_sbt_opts_file="/etc/sbt/sbtopts" +declare -r default_sbt_opts="-Xss64m" + +usage() { + cat < path to global settings/plugins directory (default: ~/.sbt) + -sbt-boot path to shared boot directory (default: ~/.sbt/boot in 0.11 series) + -ivy path to local Ivy repository (default: ~/.ivy2) + -mem set memory options (default: $sbt_default_mem, which is $(get_mem_opts $sbt_default_mem)) + -no-share use all local caches; no sharing + -no-global uses global caches, but does not use global ~/.sbt directory. + -jvm-debug Turn on JVM debugging, open at the given port. + -batch Disable interactive mode + + # sbt version (default: from project/build.properties if present, else latest release) + -sbt-version use the specified version of sbt + -sbt-jar use the specified jar as the sbt launcher + -sbt-rc use an RC version of sbt + -sbt-snapshot use a snapshot version of sbt + + # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) + -java-home alternate JAVA_HOME + + # jvm options and output control + JAVA_OPTS environment variable, if unset uses "$java_opts" + SBT_OPTS environment variable, if unset uses "$default_sbt_opts" + .sbtopts if this file exists in the current directory, it is + prepended to the runner args + /etc/sbt/sbtopts if this file exists, it is prepended to the runner args + -Dkey=val pass -Dkey=val directly to the java runtime + -J-X pass option -X directly to the java runtime + (-J is stripped) + -S-X add -X to sbt's scalacOptions (-S is stripped) + -PmavenProfiles Enable a maven profile for the build. + +In the case of duplicated or conflicting options, the order above +shows precedence: JAVA_OPTS lowest, command line options highest. +EOM +} + +process_my_args () { + while [[ $# -gt 0 ]]; do + case "$1" in + -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; + -no-share) addJava "$noshare_opts" && shift ;; + -no-global) addJava "-Dsbt.global.base=$(pwd)/project/.sbtboot" && shift ;; + -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; + -sbt-dir) require_arg path "$1" "$2" && addJava "-Dsbt.global.base=$2" && shift 2 ;; + -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; + -batch) exec /dev/null) + if [[ ! $? ]]; then + saved_stty="" + fi +} + +if [ ! -z "${SPARK_LOCAL_HOSTNAME}" ]; then + echo "Using SPARK_LOCAL_HOSTNAME=$SPARK_LOCAL_HOSTNAME" 1>&2 +fi +if [ ! -z "${SPARK_LOCAL_IP}" ]; then + echo "Using SPARK_LOCAL_IP=$SPARK_LOCAL_IP" 1>&2 +fi + +saveSttySettings +trap onExit INT + +run "$@" + +exit_status=$? +onExit diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash new file mode 100644 index 00000000000..5f269c212d6 --- /dev/null +++ b/build/sbt-launch-lib.bash @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# + +# A library to simplify using the SBT launcher from other packages. +# Note: This should be used by tools like giter8/conscript etc. + + +if test -z "$HOME"; then + declare -r script_dir="$(dirname "$script_path")" +else + declare -r script_dir="$HOME/.sbt" +fi + +declare -a residual_args +declare -a java_args +declare -a scalac_args +declare -a sbt_commands +declare -a maven_profiles +declare sbt_default_mem=4096 + +if test -x "$JAVA_HOME/bin/java"; then + echo -e "Using $JAVA_HOME as default JAVA_HOME." + echo "Note, this will be overridden by -java-home if it is set." + declare java_cmd="$JAVA_HOME/bin/java" +else + declare java_cmd=java +fi + +echoerr () { + echo 1>&2 "$@" +} +vlog () { + [[ $verbose || $debug ]] && echoerr "$@" +} +dlog () { + [[ $debug ]] && echoerr "$@" +} + +acquire_sbt_jar () { + SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties` + # DEFAULT_ARTIFACT_REPOSITORY env variable can be used to only fetch + # artifacts from internal repos only. + # Ex: + # DEFAULT_ARTIFACT_REPOSITORY=https://artifacts.internal.com/libs-release/ + URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar + JAR=sbt-launch-${SBT_VERSION}.jar + + sbt_jar=$JAR + echo $sbt_jar + + if [[ ! -f "$sbt_jar" ]]; then + # Download sbt launch jar if it hasn't been downloaded yet + if [ ! -f "${JAR}" ]; then + # Download + printf "Attempting to fetch sbt\n" + JAR_DL="${JAR}.part" + if [ $(command -v curl) ]; then + curl --fail --location --silent ${URL1} > "${JAR_DL}" &&\ + mv "${JAR_DL}" "${JAR}" + elif [ $(command -v wget) ]; then + wget --quiet ${URL1} -O "${JAR_DL}" &&\ + mv "${JAR_DL}" "${JAR}" + else + printf "You do not have curl or wget installed, please install sbt manually from https://www.scala-sbt.org/\n" + exit -1 + fi + fi + if [ ! -f "${JAR}" ]; then + # We failed to download + printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from https://www.scala-sbt.org/\n" + exit -1 + fi + printf "Launching sbt from ${JAR}\n" + fi +} + +execRunner () { + # print the arguments one to a line, quoting any containing spaces + [[ $verbose || $debug ]] && echo "# Executing command line:" && { + for arg; do + if printf "%s\n" "$arg" | grep -q ' '; then + printf "\"%s\"\n" "$arg" + else + printf "%s\n" "$arg" + fi + done + echo "" + } + + "$@" +} + +addJava () { + dlog "[addJava] arg = '$1'" + java_args=( "${java_args[@]}" "$1" ) +} + +enableProfile () { + dlog "[enableProfile] arg = '$1'" + maven_profiles=( "${maven_profiles[@]}" "$1" ) + export SBT_MAVEN_PROFILES="${maven_profiles[@]}" +} + +addSbt () { + dlog "[addSbt] arg = '$1'" + sbt_commands=( "${sbt_commands[@]}" "$1" ) +} +addResidual () { + dlog "[residual] arg = '$1'" + residual_args=( "${residual_args[@]}" "$1" ) +} +addDebugger () { + addJava "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$1" +} + +# a ham-fisted attempt to move some memory settings in concert +# so they need not be dicked around with individually. +get_mem_opts () { + local mem=${1:-$sbt_default_mem} + local codecache=$(( $mem / 8 )) + (( $codecache > 128 )) || codecache=128 + (( $codecache < 2048 )) || codecache=2048 + + echo "-Xms${mem}m -Xmx${mem}m -XX:ReservedCodeCacheSize=${codecache}m" +} + +require_arg () { + local type="$1" + local opt="$2" + local arg="$3" + if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then + echo "$opt requires <$type> argument" 1>&2 + exit 1 + fi +} + +is_function_defined() { + declare -f "$1" > /dev/null +} + +process_args () { + while [[ $# -gt 0 ]]; do + case "$1" in + -h|-help) usage; exit 1 ;; + -v|-verbose) verbose=1 && shift ;; + -d|-debug) debug=1 && shift ;; + + -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; + -mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;; + -jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;; + -batch) exec Seq("sbt") + case Some(v) => + v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq + } + if (profiles.contains("jwdp-test-debug")) { + sys.props.put("test.jwdp.enabled", "true") + } + profiles + } + + lazy val scalaStyleRules = Project("scalaStyleRules", file("scalastyle")) + .settings( + libraryDependencies += "org.scalastyle" %% "scalastyle" % "1.0.0") + + lazy val scalaStyleOnCompile = taskKey[Unit]("scalaStyleOnCompile") + + lazy val scalaStyleOnTest = taskKey[Unit]("scalaStyleOnTest") + + val scalaStyleOnCompileConfig: String = { + val in = "scalastyle-config.xml" + val out = "scalastyle-on-compile.generated.xml" + val replacements = Map( + """customId="println" level="error"""" -> """customId="println" level="warn"""") + var contents = Source.fromFile(in).getLines.mkString("\n") + for ((k, v) <- replacements) { + require(contents.contains(k), s"Could not rewrite '$k' in original scalastyle config.") + contents = contents.replace(k, v) + } + new PrintWriter(out) { + write(contents) + close() + } + out + } + + // Return a cached scalastyle task for a given configuration (usually Compile or Test) + private def cachedScalaStyle(config: Configuration) = Def.task { + val logger = streams.value.log + // We need a different cache dir per Configuration, otherwise they collide + val cacheDir = target.value / s"scalastyle-cache-${config.name}" + val cachedFun = FileFunction.cached(cacheDir, FilesInfo.lastModified, FilesInfo.exists) { + (inFiles: Set[File]) => + { + val args: Seq[String] = Seq.empty + val scalaSourceV = Seq(file((config / scalaSource).value.getAbsolutePath)) + val configV = (ThisBuild / baseDirectory).value / scalaStyleOnCompileConfig + val configUrlV = (config / scalastyleConfigUrl).value + val streamsV = ((config / streams).value: @sbtUnchecked) + val failOnErrorV = true + val failOnWarningV = false + val scalastyleTargetV = (config / scalastyleTarget).value + val configRefreshHoursV = (config / scalastyleConfigRefreshHours).value + val targetV = (config / target).value + val configCacheFileV = (config / scalastyleConfigUrlCacheFile).value + + logger.info(s"Running scalastyle on ${name.value} in ${config.name}") + Tasks.doScalastyle( + args, + configV, + configUrlV, + failOnErrorV, + failOnWarningV, + scalaSourceV, + scalastyleTargetV, + streamsV, + configRefreshHoursV, + targetV, + configCacheFileV) + + Set.empty + } + } + + cachedFun(findFiles((config / scalaSource).value)) + } + + private def findFiles(file: File): Set[File] = if (file.isDirectory) { + file.listFiles().toSet.flatMap(findFiles) + file + } else { + Set(file) + } + + def enableScalaStyle: Seq[sbt.Def.Setting[_]] = Seq( + scalaStyleOnCompile := cachedScalaStyle(Compile).value, + scalaStyleOnTest := cachedScalaStyle(Test).value, + (Compile / compile) := { + scalaStyleOnCompile.value + (Compile / compile).value + }, + (Test / compile) := { + scalaStyleOnTest.value + (Test / compile).value + }) + + // Silencer: Scala compiler plugin for warning suppression + // Aim: enable fatal warnings, but suppress ones related to using of deprecated APIs + // depends on scala version: + // <2.13.2 - silencer 1.7.12 and compiler settings to enable fatal warnings + // 2.13.2+ - no silencer and configured warnings to achieve the same + lazy val compilerWarningSettings: Seq[sbt.Def.Setting[_]] = Seq( + libraryDependencies ++= { + if (VersionNumber(scalaVersion.value).matchesSemVer(SemanticSelector("<2.13.2"))) { + val silencerVersion = "1.7.13" + Seq( + "org.scala-lang.modules" %% "scala-collection-compat" % "2.2.0", + compilerPlugin( + "com.github.ghik" % "silencer-plugin" % silencerVersion cross CrossVersion.full), + "com.github.ghik" % "silencer-lib" % silencerVersion % Provided cross CrossVersion.full) + } else { + Seq.empty + } + }) + + lazy val sharedSettings = compilerWarningSettings ++ + (if (sys.env.contains("NOLINT_ON_COMPILE")) Nil else enableScalaStyle) ++ Seq( + (Compile / exportJars) := true, + (Test / exportJars) := false, + javaHome := sys.env.get("JAVA_HOME") + .orElse(sys.props.get("java.home").map { p => + new File(p).getParentFile().getAbsolutePath() + }) + .map(file), + publishMavenStyle := true, + (Compile / javacOptions) ++= Seq( + "-encoding", + UTF_8.name()), + SbtPomKeys.profiles := profiles) + + def enable(settings: Seq[Setting[_]])(projectRef: ProjectRef): Unit = { + val existingSettings = projectsMap.getOrElse(projectRef.project, Seq[Setting[_]]()) + projectsMap += (projectRef.project -> (existingSettings ++ settings)) + } + + (allProjects).foreach(enable(sharedSettings)) + enable(KyuubiExtensionSparkCommon.settings)(kyuubiExtensionSparkCommon) + + // TODO: move this to its upstream project. + override def projectDefinitions(baseDirectory: File): Seq[Project] = { + super.projectDefinitions(baseDirectory).map { x => + if (projectsMap.exists(_._1 == x.id)) x.settings(projectsMap(x.id): _*) + else x.settings(Seq[Setting[_]](): _*) + } + } +} + +object KyuubiExtensionSparkCommon { + import com.simplytyped.Antlr4Plugin + import com.simplytyped.Antlr4Plugin.autoImport._ + + lazy val settings = Antlr4Plugin.projectSettings ++ Seq( + (Antlr4 / antlr4Version) := SbtPomKeys.effectivePom.value.getProperties.get("antlr4.version").asInstanceOf[String], + (Antlr4 / antlr4PackageName) := Some("org.apache.kyuubi.sql"), + (Antlr4 / antlr4GenListener) := true, + (Antlr4 / antlr4GenVisitor) := true, + (Antlr4 / antlr4TreatWarningsAsErrors) := true + ) +} diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 00000000000..40b3b8e7b65 --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.9.0 diff --git a/project/plugin.sbt b/project/plugin.sbt new file mode 100644 index 00000000000..64ec1d4bdbb --- /dev/null +++ b/project/plugin.sbt @@ -0,0 +1,9 @@ +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.1") + +addSbtPlugin("com.github.sbt" % "sbt-pom-reader" % "2.4.0") + +addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.6") + +addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.3") + +addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") From f4b9a12185babf91582ddf3f5fd23fd4222a6752 Mon Sep 17 00:00:00 2001 From: davidyuan Date: Sun, 26 Nov 2023 16:32:36 +0800 Subject: [PATCH 2/7] support sbt --- build/sbt-launch-lib.bash | 19 ++++++++++++++++--- project/Build.scala | 1 + 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash index 5f269c212d6..1fd1a323591 100644 --- a/build/sbt-launch-lib.bash +++ b/build/sbt-launch-lib.bash @@ -1,8 +1,21 @@ #!/usr/bin/env bash -# -# A library to simplify using the SBT launcher from other packages. -# Note: This should be used by tools like giter8/conscript etc. +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# if test -z "$HOME"; then diff --git a/project/Build.scala b/project/Build.scala index ade6b748ccb..a76774c3cd0 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import java.io.{File, PrintWriter} import java.nio.charset.StandardCharsets.UTF_8 From 231bb3b7e44a2e694960c4d71da47059b6e18a11 Mon Sep 17 00:00:00 2001 From: davidyuan Date: Sun, 26 Nov 2023 16:35:02 +0800 Subject: [PATCH 3/7] support sbt --- project/build.properties | 16 ++++++++++++++++ project/plugin.sbt | 17 +++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/project/build.properties b/project/build.properties index 40b3b8e7b65..6a3bb887716 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# sbt.version=1.9.0 diff --git a/project/plugin.sbt b/project/plugin.sbt index 64ec1d4bdbb..5b1dbbef4fe 100644 --- a/project/plugin.sbt +++ b/project/plugin.sbt @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.1") addSbtPlugin("com.github.sbt" % "sbt-pom-reader" % "2.4.0") From a2a3f0c805c6205fca4d8c92b84f8e35a0139fbb Mon Sep 17 00:00:00 2001 From: davidyuan Date: Sun, 26 Nov 2023 16:51:03 +0800 Subject: [PATCH 4/7] support sbt --- build/sbt-launch-lib.bash | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash index 1fd1a323591..53046db943b 100644 --- a/build/sbt-launch-lib.bash +++ b/build/sbt-launch-lib.bash @@ -1,23 +1,3 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - if test -z "$HOME"; then declare -r script_dir="$(dirname "$script_path")" else @@ -56,10 +36,9 @@ acquire_sbt_jar () { # Ex: # DEFAULT_ARTIFACT_REPOSITORY=https://artifacts.internal.com/libs-release/ URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar - JAR=sbt-launch-${SBT_VERSION}.jar + JAR=build/sbt-launch-${SBT_VERSION}.jar sbt_jar=$JAR - echo $sbt_jar if [[ ! -f "$sbt_jar" ]]; then # Download sbt launch jar if it hasn't been downloaded yet From 536f8e364e892ac88cb000f294914f3781269243 Mon Sep 17 00:00:00 2001 From: davidyuan Date: Sun, 26 Nov 2023 16:54:36 +0800 Subject: [PATCH 5/7] support sbt --- build/sbt-launch-lib.bash | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash index 53046db943b..7a47d871027 100644 --- a/build/sbt-launch-lib.bash +++ b/build/sbt-launch-lib.bash @@ -1,3 +1,22 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + if test -z "$HOME"; then declare -r script_dir="$(dirname "$script_path")" else From d2b6779e1a304f1f8a27a4b3b00c7a8522922fb7 Mon Sep 17 00:00:00 2001 From: davidyuan Date: Sun, 26 Nov 2023 18:30:49 +0800 Subject: [PATCH 6/7] support sbt --- build/sbt-launch-lib.bash | 0 build/util.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build/sbt-launch-lib.bash mode change 100644 => 100755 build/util.sh diff --git a/build/sbt-launch-lib.bash b/build/sbt-launch-lib.bash old mode 100644 new mode 100755 diff --git a/build/util.sh b/build/util.sh old mode 100644 new mode 100755 From c833c68aa3cd6b2a1ae964e4a140b44f94c8a68a Mon Sep 17 00:00:00 2001 From: davidyuan Date: Mon, 27 Nov 2023 18:30:08 +0800 Subject: [PATCH 7/7] try support sbt test in github ci --- .github/workflows/master.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) mode change 100644 => 100755 .github/workflows/master.yml diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml old mode 100644 new mode 100755 index fe92f6b6bf1..a165aec666f --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -218,13 +218,16 @@ jobs: if: matrix.flink-archive == '' run: | TEST_MODULES="externals/kyuubi-flink-sql-engine,integration-tests/kyuubi-flink-it" - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -Pflink-${{ matrix.flink }} ${{ matrix.flink-archive }} test + # ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -Pflink-${{ matrix.flink }} ${{ matrix.flink-archive }} test + ./build/sbt ${MVN_OPT} -Pflink-${{ matrix.flink }} ${{ matrix.flink-archive }} kyuubi-flink-sql-engine/test + ./build/sbt ${MVN_OPT} -Pflink-${{ matrix.flink }} ${{ matrix.flink-archive }} kyuubi-flink-it/test - name: Cross-version test Flink if: matrix.flink-archive != '' run: | IT_FLINK=`echo "${{ matrix.flink-archive }}" | grep -E 'flink\-([0-9]+\.[0-9]+.[0-9]+)\-bin' -o | grep -E '[0-9]+\.[0-9]+' -o` IT_MODULE="integration-tests/kyuubi-flink-it" - ./build/mvn ${MVN_OPT} -pl ${IT_MODULE} -Pflink-${IT_FLINK} ${{ matrix.flink-archive }} test + # ./build/mvn ${MVN_OPT} -pl ${IT_MODULE} -Pflink-${IT_FLINK} ${{ matrix.flink-archive }} test + ./build/sbt ${MVN_OPT} -Pflink-${IT_FLINK} ${{ matrix.flink-archive }} kyuubi-flink-it/test - name: Upload test logs if: failure() uses: actions/upload-artifact@v3 @@ -274,7 +277,9 @@ jobs: rm $HIVE_239_LIB/derby-* wget https://repo1.maven.org/maven2/org/apache/derby/derby/10.14.2.0/derby-10.14.2.0.jar -P $HIVE_239_LIB fi - ./build/mvn ${MVN_OPT} ${{ matrix.hive-archive }} -pl ${TEST_MODULES} test + # ./build/mvn ${MVN_OPT} ${{ matrix.hive-archive }} -pl ${TEST_MODULES} test + ./build/sbt ${MVN_OPT} ${{ matrix.hive-archive }} kyuubi-hive-sql-engine/test + ./build/sbt ${MVN_OPT} ${{ matrix.hive-archive }} kyuubi-hive-it/test - name: Upload test logs if: failure() uses: actions/upload-artifact@v3 @@ -313,7 +318,9 @@ jobs: run: | TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it" ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test + # ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test + ./build/sbt ${MVN_OPT} kyuubi-jdbc-engine/test + ./build/sbt ${MVN_OPT} kyuubi-jdbc-it/test - name: Upload test logs if: failure() uses: actions/upload-artifact@v3 @@ -404,7 +411,7 @@ jobs: with: # passthrough CI into build container build-args: | - CI=${CI} + CI=${CI} MVN_ARG=--flink-provided --hive-provided -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -DskipTests context: . file: build/Dockerfile