[SPARK-37103][INFRA] Switch from Maven to SBT to build Spark on AppVeyor

### What changes were proposed in this pull request? This PR proposes to switch from Maven to SBT to build Spark on AppVeyor. ### Why are the changes needed? Recently, building Spark on AppVeyor almost always fails due to StackOverflowError at compile time. We can't identify the reason so far but one workaround would be building with SBT. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? AppVeyor itself. Closes apache#34373 from sarutak/appveyor-sbt. Lead-authored-by: Kousuke Saruta <[email protected]> Co-authored-by: Kousuke Saruta <[email protected]> Signed-off-by: Hyukjin Kwon <[email protected]>
masseyke · Oct 25, 2021 · 10080fa · 10080fa
1 parent 3608971
commit 10080fa
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 10 deletions.
diff --git a/appveyor.yml b/appveyor.yml
@@ -39,7 +39,7 @@ cache:
   - C:\Users\appveyor\.m2
 
 install:
-  # Install maven and dependencies
+  # Install SBT and dependencies
   - ps: .\dev\appveyor-install-dependencies.ps1
   # Required package for R unit tests. xml2 is required to use jUnit reporter in testthat.
   - cmd: Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', 'xml2'), repos='https://cloud.r-project.org/')"
@@ -49,7 +49,11 @@ build_script:
   # '-Djna.nosys=true' is required to avoid kernel32.dll load failure.
   # See SPARK-28759.
   # Ideally we should check the tests related to Hive in SparkR as well (SPARK-31745).
-  - cmd: mvn -DskipTests -Psparkr -Djna.nosys=true package
+  - cmd: set SBT_MAVEN_PROFILES=-Psparkr
+  - cmd: set SBT_OPTS=-Djna.nosys=true -Dfile.encoding=UTF-8 -Xms4096m -Xms4096m -XX:ReservedCodeCacheSize=128m
+  - cmd: sbt package
+  - cmd: set SBT_MAVEN_PROFILES=
+  - cmd: set SBT_OPTS=
 
 environment:
   NOT_CRAN: true

diff --git a/dev/appveyor-install-dependencies.ps1 b/dev/appveyor-install-dependencies.ps1
@@ -79,18 +79,32 @@ if (!(Test-Path $tools)) {
 }
 
 # ========================== Maven
+# Push-Location $tools
+#
+# $mavenVer = "3.6.3"
+# Start-FileDownload "https://archive.apache.org/dist/maven/maven-3/$mavenVer/binaries/apache-maven-$mavenVer-bin.zip" "maven.zip"
+#
+# # extract
+# Invoke-Expression "7z.exe x maven.zip"
+#
+# # add maven to environment variables
+# $env:PATH = "$tools\apache-maven-$mavenVer\bin;" + $env:PATH
+# $env:M2_HOME = "$tools\apache-maven-$mavenVer"
+# $env:MAVEN_OPTS = "-Xmx2g -XX:ReservedCodeCacheSize=1g"
+#
+# Pop-Location
+
+# ========================== SBT
 Push-Location $tools
 
-$mavenVer = "3.6.3"
-Start-FileDownload "https://archive.apache.org/dist/maven/maven-3/$mavenVer/binaries/apache-maven-$mavenVer-bin.zip" "maven.zip"
+$sbtVer = "1.5.5"
+Start-FileDownload "https://github.com/sbt/sbt/releases/download/v$sbtVer/sbt-$sbtVer.zip" "sbt.zip"
 
 # extract
-Invoke-Expression "7z.exe x maven.zip"
+Invoke-Expression "7z.exe x sbt.zip"
 
-# add maven to environment variables
-$env:PATH = "$tools\apache-maven-$mavenVer\bin;" + $env:PATH
-$env:M2_HOME = "$tools\apache-maven-$mavenVer"
-$env:MAVEN_OPTS = "-Xmx2g -XX:ReservedCodeCacheSize=1g"
+# add sbt to environment variables
+$env:PATH = "$tools\sbt\bin;" + $env:PATH
 
 Pop-Location
 

diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
@@ -917,7 +917,8 @@ object SparkR {
   val buildRPackage = taskKey[Unit]("Build the R package")
   lazy val settings = Seq(
     buildRPackage := {
-      val command = baseDirectory.value / ".." / "R" / "install-dev.sh"
+      val postfix = if (File.separator == "\\") ".bat" else ".sh"
+      val command = baseDirectory.value / ".." / "R" / s"install-dev$postfix"
       Process(command.toString).!!
     },
     (Compile / compile) := (Def.taskDyn {

diff --git a/project/build.properties b/project/build.properties
@@ -14,4 +14,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+# Please update the version in appveyor-install-dependencies.ps1 together.
 sbt.version=1.5.5