From 7e1aa2a44488828a997195eaa5a64824b32edf60 Mon Sep 17 00:00:00 2001 From: timl Date: Mon, 14 Oct 2024 21:05:10 +0800 Subject: [PATCH] Use mvn -f scala2.13/ in the build scripts to build the 2.13 jars To fix: https://github.com/NVIDIA/spark-rapids/issues/11444 Signed-off-by: timl --- .github/workflows/mvn-verify-check.yml | 8 ++------ build/buildall | 7 ++----- jenkins/spark-nightly-build.sh | 9 ++++----- jenkins/spark-premerge-build.sh | 11 ++++------- scala2.13/README.md | 6 ++---- 5 files changed, 14 insertions(+), 27 deletions(-) diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index 86e12a4a32b..0aca7bc3655 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -246,12 +246,10 @@ jobs: echo "Generated Scala 2.13 build files don't match what's in repository" exit 1 fi - # change to Scala 2.13 Directory - cd scala2.13 # test command, will retry for 3 times if failed. max_retry=3; delay=30; i=1 while true; do - mvn package \ + mvn package -f scala2.13/ \ -pl integration_tests,tests,tools -am -P 'individual,pre-merge' \ -Dbuildver=${{ matrix.spark-version }} -Dmaven.scalastyle.skip=true \ -Drat.skip=true ${{ env.COMMON_MVN_FLAGS }} && break || { @@ -303,12 +301,10 @@ jobs: echo "Generated Scala 2.13 build files don't match what's in repository" exit 1 fi - # change to Scala 2.13 Directory - cd scala2.13 # test command, will retry for 3 times if failed. max_retry=3; delay=30; i=1 while true; do - mvn verify \ + mvn verify -f scala2.13/ \ -P "individual,pre-merge,source-javadoc" -Dbuildver=${{ matrix.spark-version }} \ ${{ env.COMMON_MVN_FLAGS }} && break || { if [[ $i -le $max_retry ]]; then diff --git a/build/buildall b/build/buildall index 0599d080054..2da61fe6451 100755 --- a/build/buildall +++ b/build/buildall @@ -86,7 +86,7 @@ function bloopInstall() { function versionsFromDistProfile() { [[ "$BUILD_ALL_DEBUG" == "1" ]] && set -x - versionRawStr=$(mvn -B help:evaluate -q -pl dist -P"$1" -Dexpression=included_buildvers -DforceStdout) + versionRawStr=$($MVN -B help:evaluate -q -pl dist -P"$1" -Dexpression=included_buildvers -DforceStdout) versionStr=${versionRawStr//[$'\n',]/} echo -n $versionStr } @@ -171,6 +171,7 @@ fi export MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 ${MVN_OPT}" if [[ "$SCALA213" == "1" ]]; then + MVN="$MVN -f scala2.13/" DIST_PROFILE=${DIST_PROFILE:-"noSnapshotsScala213"} $(dirname $0)/make-scala-version-build-files.sh 2.13 else @@ -234,10 +235,6 @@ if [[ "$SKIP_CLEAN" != "1" ]]; then $MVN -q clean fi -if [[ "$SCALA213" == "1" ]]; then - cd scala2.13 -fi - echo "Building a combined dist jar with Shims for ${SPARK_SHIM_VERSIONS[@]} ..." function build_single_shim() { diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh index 7f7ba8d65a9..cdda90f2c0c 100755 --- a/jenkins/spark-nightly-build.sh +++ b/jenkins/spark-nightly-build.sh @@ -19,6 +19,9 @@ set -ex . jenkins/version-def.sh +## MVN_OPT : maven options environment, e.g. MVN_OPT='-Dspark-rapids-jni.version=xxx' to specify spark-rapids-jni dependency's version. +MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 ${MVN_OPT} -Psource-javadoc" + SCALA_BINARY_VER=${SCALA_BINARY_VER:-"2.12"} if [ $SCALA_BINARY_VER == "2.13" ]; then # Run scala2.13 build and test against JDK17 @@ -26,17 +29,13 @@ if [ $SCALA_BINARY_VER == "2.13" ]; then update-java-alternatives --set $JAVA_HOME java -version - cd scala2.13 - ln -sf ../jenkins jenkins + MVN="$MVN -f scala2.13/" fi WORKSPACE=${WORKSPACE:-$(pwd)} ## export 'M2DIR' so that shims can get the correct Spark dependency info export M2DIR=${M2DIR:-"$WORKSPACE/.m2"} -## MVN_OPT : maven options environment, e.g. MVN_OPT='-Dspark-rapids-jni.version=xxx' to specify spark-rapids-jni dependency's version. -MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 ${MVN_OPT} -Psource-javadoc" - DIST_PL="dist" function mvnEval { $MVN help:evaluate -q -pl $DIST_PL $MVN_URM_MIRROR -Prelease320 -Dmaven.repo.local=$M2DIR -DforceStdout -Dexpression=$1 diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh index e09558425e3..150de339e09 100755 --- a/jenkins/spark-premerge-build.sh +++ b/jenkins/spark-premerge-build.sh @@ -191,9 +191,6 @@ ci_scala213() { update-java-alternatives --set $JAVA_HOME java -version - cd scala2.13 - ln -sf ../jenkins jenkins - # Download a Scala 2.13 version of Spark prepare_spark 3.3.0 2.13 @@ -202,15 +199,15 @@ ci_scala213() { do echo "Spark version (Scala 2.13): $version" env -u SPARK_HOME \ - $MVN_CMD -U -B $MVN_URM_MIRROR -Dbuildver=$version clean install $MVN_BUILD_ARGS -Dpytest.TEST_TAGS='' + $MVN_CMD -f scala2.13/ -U -B $MVN_URM_MIRROR -Dbuildver=$version clean install $MVN_BUILD_ARGS -Dpytest.TEST_TAGS='' # Run filecache tests env -u SPARK_HOME SPARK_CONF=spark.rapids.filecache.enabled=true \ - $MVN_CMD -B $MVN_URM_MIRROR -Dbuildver=$version test -rf tests $MVN_BUILD_ARGS -Dpytest.TEST_TAGS='' \ + $MVN_CMD -f scala2.13/ -B $MVN_URM_MIRROR -Dbuildver=$version test -rf tests $MVN_BUILD_ARGS -Dpytest.TEST_TAGS='' \ -DwildcardSuites=org.apache.spark.sql.rapids.filecache.FileCacheIntegrationSuite done - $MVN_CMD -U -B $MVN_URM_MIRROR clean package $MVN_BUILD_ARGS -DskipTests=true - cd .. # Run integration tests in the project root dir to leverage test cases and resource files + $MVN_CMD -f scala2.13/ -U -B $MVN_URM_MIRROR clean package $MVN_BUILD_ARGS -DskipTests=true + export TEST_TAGS="not premerge_ci_1" export TEST_TYPE="pre-commit" # SPARK_HOME (and related) must be set to a Spark built with Scala 2.13 diff --git a/scala2.13/README.md b/scala2.13/README.md index 4096363cf52..f2e5200c9c7 100644 --- a/scala2.13/README.md +++ b/scala2.13/README.md @@ -25,8 +25,7 @@ You can use Maven to build the plugin. Like with Scala 2.12, we recommend buildi phase. ```shell script -cd scala2.13 -mvn verify +mvn verify -f scala2.13/ ``` After a successful build, the RAPIDS Accelerator jar will be in the `scala2.13/dist/target/` directory. @@ -45,7 +44,6 @@ You can also use the `buildall` script in the parent directory to build against of Apache Spark. ```shell script -cd .. ./build/buildall --profile=noSnapshotsScala213 ``` @@ -72,4 +70,4 @@ That way any new dependencies or other changes will be picked up in the Scala 2. You should be able to open the `scala2.13` directory directly in IntelliJ as a separate project. You can build and debug as normal, although there are slight differences in how to navigate the source. In particular, when you select a particular build profile, you will only be able to navigate the source used by modules that are included for that -spark version. \ No newline at end of file +spark version.