Merge remote-tracking branch 'origin/master' into uncheckedIO

build.sbt

@@ -65,7 +65,9 @@ crossScalaVersions := Nil
 // For Java 11 use the following on command line
 // sbt 'set targetJvm := "11"' [commands]
 val targetJvm = settingKey[String]("Target JVM version")
-Global / targetJvm := "1.8"
+Global / targetJvm := "8"
+
+lazy val javaVersion = sys.props.getOrElse("java.version", "Unknown")
 
 /**
  * Returns the current spark version, which is the same value as `sparkVersion.value`.
@@ -103,10 +105,14 @@ lazy val commonSettings = Seq(
   scalaVersion := default_scala_version.value,
   crossScalaVersions := all_scala_versions,
   fork := true,
-  scalacOptions ++= Seq(s"-target:jvm-${targetJvm.value}", "-Ywarn-unused:imports"),
-  javacOptions ++= Seq("-source", targetJvm.value),
-  // -target cannot be passed as a parameter to javadoc. See https://github.com/sbt/sbt/issues/355
-  Compile / compile / javacOptions ++= Seq("-target", targetJvm.value),
+  scalacOptions ++= Seq("-Ywarn-unused:imports"),
+  javacOptions ++= {
+    if (javaVersion.startsWith("1.8")) {
+      Seq.empty // `--release` is supported since JDK 9 and the minimum supported JDK is 8
+    } else {
+      Seq("--release", targetJvm.value) // generated bytecode should be usable with JVM 1.8
+    }
+  },
 
   // Make sure any tests in any project that uses Spark is configured for running well locally
   Test / javaOptions ++= Seq(
@@ -134,7 +140,7 @@ def crossSparkSettings(): Seq[Setting[_]] = getSparkVersion() match {
   case LATEST_RELEASED_SPARK_VERSION => Seq(
     scalaVersion := default_scala_version.value,
     crossScalaVersions := all_scala_versions,
-    targetJvm := "1.8",
+    targetJvm := "8",
     // For adding staged Spark RC versions, e.g.:
     // resolvers += "Apache Spark 3.5.0 (RC1) Staging" at "https://repository.apache.org/content/repositories/orgapachespark-1444/",
     Compile / unmanagedSourceDirectories += (Compile / baseDirectory).value / "src" / "main" / "scala-spark-3.5",

spark/src/test/scala/org/apache/spark/sql/delta/managedcommit/CommitOwnerClientImplSuiteBase.scala

@@ -70,10 +70,10 @@ trait CommitOwnerClientImplSuiteBase extends QueryTest
    * where maxVersion is the current latest version of the table.
    */
   protected def validateGetCommitsResult(
-      startVersion: Option[Long],
-      endVersion: Option[Long],
-      maxVersion: Long,
-      commitVersions: Seq[Long]): Unit
+    response: GetCommitsResponse,
+    startVersion: Option[Long],
+    endVersion: Option[Long],
+    maxVersion: Long): Unit
 
   /**
    * Checks that the commit owner state is correct in terms of
@@ -230,8 +230,8 @@ trait CommitOwnerClientImplSuiteBase extends QueryTest
         startVersion: Option[Long],
         endVersion: Option[Long],
         maxVersion: Long): Unit = {
-      val result = client.getCommits(startVersion, endVersion).getCommits.map(_.getVersion)
-      validateGetCommitsResult(startVersion, endVersion, maxVersion, result)
+      val result = client.getCommits(startVersion, endVersion)
+      validateGetCommitsResult(result, startVersion, endVersion, maxVersion)
     }
 
     withTempTableDir { tempDir =>
@@ -299,7 +299,7 @@ trait CommitOwnerClientImplSuiteBase extends QueryTest
       assertCommitFail(4, 5, retryable = true, commit(4, 6, tableCommitOwnerClient))
 
       commit(5, 5, tableCommitOwnerClient)
-      assert(tableCommitOwnerClient.getCommits() == GetCommitsResponse(Seq.empty, 5))
+      validateGetCommitsResult(tableCommitOwnerClient.getCommits(), None, None, 5)
       assertCommitFail(5, 6, retryable = true, commit(5, 5, tableCommitOwnerClient))
       assertCommitFail(7, 6, retryable = false, commit(7, 7, tableCommitOwnerClient))
 

spark/src/test/scala/org/apache/spark/sql/delta/managedcommit/InMemoryCommitOwnerSuite.scala

@@ -52,13 +52,15 @@ abstract class InMemoryCommitOwnerSuite(batchSize: Int) extends CommitOwnerClien
   }
 
   protected def validateGetCommitsResult(
+      result: GetCommitsResponse,
       startVersion: Option[Long],
       endVersion: Option[Long],
-      maxVersion: Long,
-      commitVersions: Seq[Long]): Unit = {
+      maxVersion: Long): Unit = {
+    val commitVersions = result.getCommits.map(_.getVersion)
     val lastExpectedBackfilledVersion = (maxVersion - (maxVersion % batchSize)).toInt
     val expectedVersions = lastExpectedBackfilledVersion + 1 to maxVersion.toInt
     assert(commitVersions == expectedVersions)
+    assert(result.getLatestTableVersion == maxVersion)
   }
 
   test("InMemoryCommitOwnerBuilder works as expected") {

version.sbt

@@ -1 +1 @@
-ThisBuild / version := "3.2.1-SNAPSHOT"
+ThisBuild / version := "3.3.0-SNAPSHOT"