From c3fd887977292a9fec2d20e81c3884d927717720 Mon Sep 17 00:00:00 2001 From: Christopher Lambert Date: Thu, 12 Sep 2024 11:05:47 +0200 Subject: [PATCH] remove NessieApiV1 support from spark extensions --- .../sql/execution/datasources/v2/CatalogBridge.java | 4 ++-- .../spark/sql/execution/datasources/v2/CatalogUtils.java | 9 +++------ .../execution/datasources/v2/NessieCatalogBridge.java | 6 +++--- .../sql/execution/datasources/v2/RestCatalogBridge.java | 6 +++--- .../spark/sql/execution/datasources/v2/NessieUtils.scala | 8 ++++---- 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/CatalogBridge.java b/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/CatalogBridge.java index a1dc05a1350..7f4f6c7ee98 100644 --- a/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/CatalogBridge.java +++ b/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/CatalogBridge.java @@ -16,14 +16,14 @@ package org.apache.spark.sql.execution.datasources.v2; import org.apache.spark.sql.connector.catalog.CatalogPlugin; -import org.projectnessie.client.api.NessieApiV1; +import org.projectnessie.client.api.NessieApiV2; import org.projectnessie.error.NessieReferenceNotFoundException; import org.projectnessie.model.Reference; public interface CatalogBridge extends AutoCloseable { CatalogPlugin currentCatalog(); - NessieApiV1 api(); + NessieApiV2 api(); void setCurrentRefForSpark(Reference ref, boolean configureRefAtHash); diff --git a/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/CatalogUtils.java b/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/CatalogUtils.java index 156a9230431..ae03bfb93a2 100644 --- a/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/CatalogUtils.java +++ b/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/CatalogUtils.java @@ -30,7 +30,6 @@ import org.projectnessie.api.v2.params.ParsedReference; import org.projectnessie.client.NessieClientBuilder; import org.projectnessie.client.NessieConfigConstants; -import org.projectnessie.client.api.NessieApiV1; import org.projectnessie.client.api.NessieApiV2; import org.projectnessie.client.config.NessieClientConfigSource; @@ -76,22 +75,20 @@ public static CatalogBridge buildBridge(CatalogPlugin currentCatalog, String cat } } - static NessieApiV1 buildApi(NessieClientConfigSource nessieClientConfigMapper) { + static NessieApiV2 buildApi(NessieClientConfigSource nessieClientConfigMapper) { NessieClientBuilder nessieClientBuilder = NessieClientBuilder.createClientBuilderFromSystemSettings(nessieClientConfigMapper); String clientApiVer = nessieClientConfigMapper.getValue("nessie.client-api-version"); if (clientApiVer == null) { - clientApiVer = "1"; + clientApiVer = "2"; } switch (clientApiVer) { - case "1": - return nessieClientBuilder.build(NessieApiV1.class); case "2": return nessieClientBuilder.build(NessieApiV2.class); default: throw new IllegalArgumentException( String.format( - "Unsupported client-api-version value: %s. Can only be 1 or 2", clientApiVer)); + "Unsupported client-api-version value: %s. Can only be '2'", clientApiVer)); } } diff --git a/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/NessieCatalogBridge.java b/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/NessieCatalogBridge.java index 96a62873a94..8630ea16d25 100644 --- a/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/NessieCatalogBridge.java +++ b/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/NessieCatalogBridge.java @@ -23,7 +23,7 @@ import org.apache.spark.SparkContext; import org.apache.spark.sql.connector.catalog.CatalogPlugin; import org.apache.spark.sql.util.CaseInsensitiveStringMap; -import org.projectnessie.client.api.NessieApiV1; +import org.projectnessie.client.api.NessieApiV2; import org.projectnessie.error.NessieNotFoundException; import org.projectnessie.error.NessieReferenceNotFoundException; import org.projectnessie.model.Branch; @@ -33,7 +33,7 @@ final class NessieCatalogBridge implements CatalogBridge { private final SparkContext sparkContext; - private final NessieApiV1 api; + private final NessieApiV2 api; private final CatalogPlugin currentCatalog; private final String catalogName; private final String confPrefix; @@ -106,7 +106,7 @@ public void setCurrentRefForSpark(Reference ref, boolean configureRefAtHash) { } @Override - public NessieApiV1 api() { + public NessieApiV2 api() { return api; } diff --git a/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/RestCatalogBridge.java b/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/RestCatalogBridge.java index 90227fabcdd..9c4977b348f 100644 --- a/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/RestCatalogBridge.java +++ b/integrations/spark-extensions-base/src/main/java/org/apache/spark/sql/execution/datasources/v2/RestCatalogBridge.java @@ -30,7 +30,7 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.projectnessie.api.v2.params.ParsedReference; import org.projectnessie.client.NessieConfigConstants; -import org.projectnessie.client.api.NessieApiV1; +import org.projectnessie.client.api.NessieApiV2; import org.projectnessie.client.config.NessieClientConfigSource; import org.projectnessie.error.NessieNotFoundException; import org.projectnessie.error.NessieReferenceNotFoundException; @@ -41,7 +41,7 @@ final class RestCatalogBridge implements CatalogBridge { private final SparkContext sparkContext; - private final NessieApiV1 api; + private final NessieApiV2 api; private final CatalogPlugin currentCatalog; private final String catalogName; private final String confPrefix; @@ -188,7 +188,7 @@ private static String encode(String s) { } @Override - public NessieApiV1 api() { + public NessieApiV2 api() { return api; } diff --git a/integrations/spark-extensions-base/src/main/scala/org/apache/spark/sql/execution/datasources/v2/NessieUtils.scala b/integrations/spark-extensions-base/src/main/scala/org/apache/spark/sql/execution/datasources/v2/NessieUtils.scala index e8982f841c2..9215e5f0442 100644 --- a/integrations/spark-extensions-base/src/main/scala/org/apache/spark/sql/execution/datasources/v2/NessieUtils.scala +++ b/integrations/spark-extensions-base/src/main/scala/org/apache/spark/sql/execution/datasources/v2/NessieUtils.scala @@ -15,7 +15,7 @@ */ package org.apache.spark.sql.execution.datasources.v2 -import org.projectnessie.client.api.NessieApiV1 +import org.projectnessie.client.api.NessieApiV2 import org.projectnessie.error.NessieReferenceNotFoundException import org.projectnessie.model._ @@ -36,7 +36,7 @@ object NessieUtils { def calculateRef( branch: String, tsOrHash: Option[String], - api: NessieApiV1 + api: NessieApiV2 ): Reference = { val refName = unquoteRefName(branch) @@ -76,7 +76,7 @@ object NessieUtils { private def findReferenceFromHash( branch: String, requestedHash: String, - api: NessieApiV1 + api: NessieApiV2 ) = { val commit = Option( api.getCommitLog @@ -105,7 +105,7 @@ object NessieUtils { private def findReferenceFromTimestamp( branch: String, - api: NessieApiV1, + api: NessieApiV2, timestamp: Instant ) = { val commit = Option(