Skip to content

Commit

Permalink
use RegexFileFilter for sparkCoreScalaVersion
Browse files Browse the repository at this point in the history
  • Loading branch information
bowenliang123 committed Sep 21, 2023
1 parent deb2010 commit a7f387f
Showing 1 changed file with 7 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,13 @@ class SparkProcessBuilder(
}
}

private lazy val sparkCoreScalaVersion: String = {
Paths.get(sparkHome, "jars").toFile
.list(new RegexFileFilter("^spark-core_.*\\.jar$"))
.map { p => p.substring(p.indexOf("_") + 1, p.lastIndexOf("-")) }
.head
}

override protected def engineScalaBinaryVersion: String =
StringUtils.defaultIfBlank(System.getenv("SPARK_SCALA_VERSION"), sparkCoreScalaVersion)

Expand All @@ -115,14 +122,6 @@ class SparkProcessBuilder(
new RegexFileFilter(pattern)
}

private def sparkCoreScalaVersion: String = {
val sparkCoreJarPattern = Pattern.compile("^spark-core_.*\\.jar$")
Paths.get(sparkHome, "jars").toFile
.list((_, fileName) => sparkCoreJarPattern.matcher(fileName).matches())
.map { p => p.substring(p.indexOf("_") + 1, p.lastIndexOf("-")) }
.head
}

override protected lazy val commands: Array[String] = {
// complete `spark.master` if absent on kubernetes
completeMasterUrl(conf)
Expand Down

0 comments on commit a7f387f

Please sign in to comment.