From 0d1aaedfe641983834aa2c9326ae1b3fe86375d7 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Tue, 22 Oct 2024 15:57:08 -0500 Subject: [PATCH 1/7] Refactored dependencies and imports for snowflake-jdbc-thin - Replaced shaded Apache HTTP Client with standard packages - Replaced shaded Google Guava with standard packages - Replaced shaded Apache commons-io with standard packages - Replaced shaded Jackson JSON with standard packages - Replaced shaded Gson with Jackson ObjectMapper in OAuthClient class - Replaced shaded HttpStatusCodes with HttpURLConnection status codes - Added UnmodifiableMap class to remove commons-collections dependency from Hadoop - Added Google HTTP Client 1.43.3 - Added JCL over SLF4J to replace Commons Logging - Removed unused direct dependency on Jettison - Removed usage of netty-common PlatformDependent memory value from Netty property - Removed unused direct dependency on Commons Configuration 2 - Removed unused direct dependency on Commons Compress - Excluded unused dependencies from Hadoop Common - Excluded several libraries from Dependency Convergence checking - Updated allowed license formats - Updated Apache Avro from 1.11.3 to 1.11.4 - Updated Apache Commons Codec from 1.15 to 1.17.1 - Updated Apache Commons Compress from 1.26.0 to 1.27.1 - Updated Apache Commons Lang3 from 3.14.0 to 3.17.0 - Updated Guava from 32.0.1 to 32.1.1 - Updated Snowflake JDBC from 3.18.0 to 3.19.0 - Updated ThreeTenBP from 1.6.8 to 1.6.9 - Updated Netty from 4.1.113 to 4.1.115 - Updated Nimbus from 9.37.1 to 9.47 --- pom.xml | 185 +++++++++++++++--- scripts/process_licenses.py | 65 +++++- .../snowflake/ingest/SimpleIngestManager.java | 8 +- .../ingest/connection/OAuthClient.java | 32 +-- .../ingest/connection/OAuthManager.java | 2 +- .../ingest/connection/RequestBuilder.java | 16 +- .../connection/ServiceResponseHandler.java | 18 +- .../ingest/connection/TelemetryService.java | 6 +- .../internal/DataValidationUtil.java | 2 +- .../streaming/internal/FlushService.java | 2 +- .../streaming/internal/InternalStage.java | 13 +- .../streaming/internal/ParquetRowBuffer.java | 2 +- .../internal/PresignedUrlExternalVolume.java | 0 .../internal/SnowflakeServiceClient.java | 2 +- ...nowflakeStreamingIngestClientInternal.java | 4 +- .../internal/StreamingIngestUtils.java | 6 +- .../fileTransferAgent/IcebergAzureClient.java | 20 +- .../fileTransferAgent/IcebergGCSClient.java | 14 +- .../fileTransferAgent/IcebergS3Client.java | 46 ++--- .../IcebergS3ObjectMetadata.java | 2 +- .../IcebergStorageClientFactory.java | 2 +- .../net/snowflake/ingest/utils/HttpUtil.java | 46 ++--- .../net/snowflake/ingest/utils/Utils.java | 4 +- .../collections/map/UnmodifiableMap.java | 17 ++ .../net/snowflake/ingest/SimpleIngestIT.java | 6 +- .../java/net/snowflake/ingest/TestUtils.java | 6 +- .../ingest/connection/MockOAuthClient.java | 2 +- .../connection/TelemetryServiceTest.java | 2 +- .../streaming/internal/ChannelCacheTest.java | 2 +- .../internal/InsertRowsBenchmarkTest.java | 2 +- .../streaming/internal/InternalStageTest.java | 29 +-- .../internal/MockSnowflakeServiceClient.java | 18 +- .../internal/RegisterServiceTest.java | 2 +- .../SnowflakeStreamingIngestChannelTest.java | 16 +- .../SnowflakeStreamingIngestClientTest.java | 18 +- .../internal/StreamingIngestUtilsIT.java | 2 +- .../internal/StreamingIngestUtilsTest.java | 10 +- .../snowflake/ingest/utils/HttpUtilTest.java | 10 +- 38 files changed, 423 insertions(+), 216 deletions(-) create mode 100644 src/main/java/net/snowflake/ingest/streaming/internal/PresignedUrlExternalVolume.java create mode 100644 src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java diff --git a/pom.xml b/pom.xml index b4610c9ac..4285d3ebb 100644 --- a/pom.xml +++ b/pom.xml @@ -38,19 +38,23 @@ + 4.5.14 + 4.4.16 1.11.4 1.78.1 1.9.13 1.17.1 3.2.2 - 1.26.2 + 1.27.1 2.10.1 - 2.16.1 - 3.14.0 + 2.17.0 + 3.17.0 1.3.3 1.11.0 2.17.2 - 32.0.1-jre + 1.43.3 + 2.10.1 + 32.1.1-jre 3.4.0 1.6.1 true @@ -63,9 +67,9 @@ 1.8 2.4.9 4.1.115.Final - 9.37.3 + 9.47 3.1 - 1.14.1 + 1.14.4 2.0.9 UTF-8 4.27.5 @@ -73,6 +77,7 @@ 1.7.36 1.1.10.5 3.20.0 + 1.6.9 0.13.0 @@ -85,11 +90,26 @@ pom import + + com.google.code.gson + gson + ${gson.version} + com.google.guava guava ${guava.version} + + com.google.http-client + google-http-client + ${google.httpclient.version} + + + com.google.http-client + google-http-client-gson + ${google.httpclient.version} + com.google.protobuf protobuf-java @@ -137,8 +157,30 @@ net.snowflake - snowflake-jdbc + snowflake-jdbc-thin ${snowjdbc.version} + + + com.google.errorprone + error_prone_annotations + + + commons-logging + commons-logging + + + io.grpc + grpc-context + + + javax.annotation + javax.annotation-api + + + javax.servlet + javax.servlet-api + + org.apache.avro @@ -173,6 +215,14 @@ com.github.pjfanning jersey-json + + com.google.code.gson + gson + + + com.google.re2j + re2j + com.jcraft jsch @@ -189,10 +239,26 @@ com.sun.jersey jersey-servlet + + commons-beanutils + commons-beanutils + + + commons-cli + commons-cli + + + commons-collections + commons-collections + commons-logging commons-logging + + commons-net + commons-net + dnsjava dnsjava @@ -209,6 +275,10 @@ io.netty netty-transport-native-epoll + + jakarta.activation + jakarta.activation-api + javax.activation activation @@ -225,6 +295,22 @@ org.apache.avro avro + + org.apache.commons + commons-compress + + + org.apache.commons + commons-configuration2 + + + org.apache.commons + commons-math3 + + + org.apache.commons + commons-text + org.apache.curator curator-client @@ -233,10 +319,18 @@ org.apache.curator curator-recipes + + org.apache.hadoop + hadoop-annotations + org.apache.hadoop hadoop-auth + + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_21 + org.apache.httpcomponents httpclient @@ -253,6 +347,10 @@ org.bouncycastle bcprov-jdk15on + + org.codehaus.jettison + jettison + org.eclipse.jetty jetty-server @@ -279,6 +377,16 @@ + + org.apache.httpcomponents + httpclient + ${apache.httpclient.version} + + + org.apache.httpcomponents + httpcore + ${apache.httpcomponents.version} + org.apache.iceberg iceberg-api @@ -308,6 +416,12 @@ org.apache.parquet parquet-hadoop ${parquet.version} + + + com.github.luben + zstd-jni + + org.apache.yetus @@ -339,11 +453,6 @@ jackson-xc ${codehaus.version} - - org.codehaus.jettison - jettison - 1.5.4 - org.codehaus.woodstox stax2-api @@ -354,11 +463,21 @@ objenesis ${objenesis.version} + + org.slf4j + jcl-over-slf4j + ${slf4j.version} + org.slf4j slf4j-api ${slf4j.version} + + org.threeten + threetenbp + ${threetenbp.version} + org.xerial.snappy snappy-java @@ -527,10 +646,6 @@ metrics-jvm 4.1.22 - - io.netty - netty-common - net.bytebuddy byte-buddy-agent @@ -538,18 +653,10 @@ net.snowflake - snowflake-jdbc - - - org.apache.commons - commons-compress + snowflake-jdbc-thin - - org.apache.commons - commons-configuration2 - org.apache.commons commons-text @@ -562,6 +669,14 @@ org.apache.hadoop hadoop-mapreduce-client-core + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + org.apache.iceberg iceberg-api @@ -626,7 +741,7 @@ com.github.luben zstd-jni - 1.5.6-2 + 1.5.6-5 runtime @@ -908,7 +1023,15 @@ - + + + commons-io:commons-io + commons-codec:commons-codec + org.apache.avro:avro + org.apache.commons:commons-compress + org.apache.commons:commons-lang3 + + @@ -983,8 +1106,9 @@ 3-Clause BSD License The MIT License EDL 1.0 - The Go license + The Go License Bouncy Castle Licence + BSD New license test,provided,system true @@ -999,8 +1123,9 @@ |Apache 2 BSD 2-Clause License |The BSD License |BSD - The MIT License|MIT License + The MIT License|MIT License|MIT license 3-Clause BSD License|BSD-3-Clause + The Go License|Go License @@ -1137,7 +1262,7 @@ - net.snowflake:snowflake-jdbc + net.snowflake:snowflake-jdbc-thin org.slf4j:slf4j-api com.github.luben:zstd-jni diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py index 9f715abd6..54470fdf7 100644 --- a/scripts/process_licenses.py +++ b/scripts/process_licenses.py @@ -27,13 +27,16 @@ APACHE_LICENSE = "Apache License 2.0" BSD_2_CLAUSE_LICENSE = "2-Clause BSD License" BSD_3_CLAUSE_LICENSE = "3-Clause BSD License" +BSD_NEW_LICENSE = "BSD New license" EDL_10_LICENSE = "EDL 1.0" MIT_LICENSE = "The MIT License" +MIT_LICENSE_ALT = "MIT License" GO_LICENSE = "The Go license" +GO_LICENSE_ALT = "Go License" BOUNCY_CASTLE_LICENSE = "Bouncy Castle Licence " # The SDK does not need to include licenses of dependencies, which aren't shaded -IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc", "org.slf4j:slf4j-api"} +IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc-thin", "org.slf4j:slf4j-api"} # List of dependencies, which don't ship with a license file. # Only add a new record here after verifying that the dependency JAR does not contain a license! @@ -50,7 +53,7 @@ "com.nimbusds:nimbus-jose-jwt": APACHE_LICENSE, "com.github.stephenc.jcip:jcip-annotations": APACHE_LICENSE, "io.netty:netty-common": APACHE_LICENSE, - "com.google.re2j:re2j": GO_LICENSE, + "com.google.re2j:re2j": GO_LICENSE_ALT, "com.google.protobuf:protobuf-java": BSD_3_CLAUSE_LICENSE, "com.google.code.gson:gson": APACHE_LICENSE, "org.xerial.snappy:snappy-java": APACHE_LICENSE, @@ -64,6 +67,64 @@ "com.thoughtworks.paranamer:paranamer": BSD_2_CLAUSE_LICENSE, "org.roaringbitmap:RoaringBitmap": APACHE_LICENSE, "org.roaringbitmap:shims": APACHE_LICENSE, + "com.amazonaws:aws-java-sdk-core": APACHE_LICENSE, + "software.amazon.ion:ion-java": APACHE_LICENSE, + "com.amazonaws:aws-java-sdk-kms": APACHE_LICENSE, + "com.amazonaws:jmespath-java": APACHE_LICENSE, + "com.amazonaws:aws-java-sdk-s3": APACHE_LICENSE, + "com.google.api:gax": BSD_3_CLAUSE_LICENSE, + "com.google.api:api-common": BSD_3_CLAUSE_LICENSE, + "com.google.auth:google-auth-library-credentials": BSD_NEW_LICENSE, + "com.google.api.grpc:proto-google-common-protos": APACHE_LICENSE, + "io.opencensus:opencensus-api": APACHE_LICENSE, + "com.google.auth:google-auth-library-oauth2-http": BSD_NEW_LICENSE, + "com.google.cloud:google-cloud-core": APACHE_LICENSE, + "com.google.auto.value:auto-value-annotations": APACHE_LICENSE, + "com.google.protobuf:protobuf-java-util": BSD_3_CLAUSE_LICENSE, + "com.google.api.grpc:proto-google-iam-v1": APACHE_LICENSE, + "com.google.cloud:google-cloud-storage": APACHE_LICENSE, + "io.opencensus:opencensus-contrib-http-util": APACHE_LICENSE, + "com.google.http-client:google-http-client-jackson2": APACHE_LICENSE, + "com.google.api-client:google-api-client": APACHE_LICENSE, + "com.google.oauth-client:google-oauth-client": APACHE_LICENSE, + "com.google.http-client:google-http-client-apache-v2": APACHE_LICENSE, + "com.google.apis:google-api-services-storage": APACHE_LICENSE, + "com.google.cloud:google-cloud-core-http": APACHE_LICENSE, + "com.google.http-client:google-http-client-appengine": APACHE_LICENSE, + "com.google.api:gax-httpjson": BSD_3_CLAUSE_LICENSE, + "com.google.cloud:google-cloud-core-grpc": APACHE_LICENSE, + "com.google.api:gax-grpc": BSD_3_CLAUSE_LICENSE, + "io.grpc:grpc-alts": APACHE_LICENSE, + "io.grpc:grpc-grpclb": APACHE_LICENSE, + "org.conscrypt:conscrypt-openjdk-uber": APACHE_LICENSE, + "io.grpc:grpc-auth": APACHE_LICENSE, + "io.grpc:grpc-protobuf": APACHE_LICENSE, + "io.grpc:grpc-protobuf-lite": APACHE_LICENSE, + "com.google.api.grpc:proto-google-cloud-storage-v2": APACHE_LICENSE, + "com.google.api.grpc:grpc-google-cloud-storage-v2": APACHE_LICENSE, + "com.google.api.grpc:gapic-google-cloud-storage-v2": APACHE_LICENSE, + "io.grpc:grpc-api": APACHE_LICENSE, + "io.perfmark:perfmark-api": APACHE_LICENSE, + "io.grpc:grpc-core": APACHE_LICENSE, + "com.google.android:annotations": APACHE_LICENSE, + "org.codehaus.mojo:animal-sniffer-annotations": MIT_LICENSE_ALT, + "io.grpc:grpc-stub": APACHE_LICENSE, + "io.grpc:grpc-googleapis": APACHE_LICENSE, + "io.grpc:grpc-xds": APACHE_LICENSE, + "io.opencensus:opencensus-proto": APACHE_LICENSE, + "io.grpc:grpc-services": APACHE_LICENSE, + "io.grpc:grpc-rls": APACHE_LICENSE, + "com.microsoft.azure:azure-storage": APACHE_LICENSE, + "com.microsoft.azure:azure-keyvault-core": APACHE_LICENSE, + "com.yammer.metrics:metrics-core": APACHE_LICENSE, + "com.yammer.metrics:metrics-servlet": APACHE_LICENSE, + "net.minidev:json-smart": APACHE_LICENSE, + "net.minidev:accessors-smart": APACHE_LICENSE, + "org.ow2.asm:asm": BSD_3_CLAUSE_LICENSE, + "com.google.flatbuffers:flatbuffers-java": APACHE_LICENSE, + "com.google.http-client:google-http-client": APACHE_LICENSE, + "com.google.http-client:google-http-client-gson": APACHE_LICENSE, + "io.grpc:grpc-context": APACHE_LICENSE, } diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java index 17caa2b9b..f9ba34128 100644 --- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java +++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java @@ -18,10 +18,10 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.HistoryRangeResponse; import net.snowflake.ingest.connection.HistoryResponse; import net.snowflake.ingest.connection.IngestResponse; diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java index a592899ea..d1f8384bd 100644 --- a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java +++ b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java @@ -6,23 +6,23 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; import java.net.URLEncoder; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.apache.http.entity.ContentType; -import net.snowflake.client.jdbc.internal.apache.http.entity.StringEntity; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.util.EntityUtils; -import net.snowflake.client.jdbc.internal.google.api.client.http.HttpStatusCodes; -import net.snowflake.client.jdbc.internal.google.gson.JsonObject; -import net.snowflake.client.jdbc.internal.google.gson.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.HttpUtil; import net.snowflake.ingest.utils.SFException; @@ -44,6 +44,8 @@ public class OAuthClient { private static final String REFRESH_TOKEN = "refresh_token"; private static final String EXPIRES_IN = "expires_in"; + private static final ObjectMapper objectMapper = new ObjectMapper(); + // OAuth credential private final AtomicReference oAuthCredential; @@ -75,14 +77,14 @@ public void refreshToken() throws IOException { CloseableHttpResponse httpResponse = httpClient.execute(makeRefreshTokenRequest()); String respBodyString = EntityUtils.toString(httpResponse.getEntity()); - if (httpResponse.getStatusLine().getStatusCode() == HttpStatusCodes.STATUS_CODE_OK) { - JsonObject respBody = JsonParser.parseString(respBodyString).getAsJsonObject(); + if (httpResponse.getStatusLine().getStatusCode() == HttpURLConnection.HTTP_OK) { + JsonNode respBody = objectMapper.readTree(respBodyString); if (respBody.has(ACCESS_TOKEN) && respBody.has(EXPIRES_IN)) { // Trim surrounding quotation marks String newAccessToken = respBody.get(ACCESS_TOKEN).toString().replaceAll("^\"|\"$", ""); oAuthCredential.get().setAccessToken(newAccessToken); - oAuthCredential.get().setExpiresIn(respBody.get(EXPIRES_IN).getAsInt()); + oAuthCredential.get().setExpiresIn(respBody.get(EXPIRES_IN).asInt()); return; } } diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java index f9e614a27..a33d18da3 100644 --- a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java +++ b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java @@ -6,7 +6,7 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; +import org.apache.http.client.utils.URIBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java index ec275e118..acfa42b0d 100644 --- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java +++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java @@ -15,14 +15,14 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.apache.http.entity.ContentType; -import net.snowflake.client.jdbc.internal.apache.http.entity.StringEntity; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; diff --git a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java index c9968bcf3..50d7015f3 100644 --- a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java +++ b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java @@ -8,15 +8,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.UUID; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.util.EntityUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; import net.snowflake.ingest.utils.BackOffException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java index 16688f290..1f3ccf4c5 100644 --- a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java +++ b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java @@ -14,9 +14,9 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.impl.client.CloseableHttpClient; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.client.jdbc.telemetry.TelemetryClient; import net.snowflake.client.jdbc.telemetry.TelemetryUtil; import net.snowflake.ingest.utils.Logging; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java index 3ccecfdb5..a1efd3e16 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java @@ -37,7 +37,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import net.snowflake.client.jdbc.internal.google.common.collect.Sets; +import com.google.common.collect.Sets; import net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat; import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10; import net.snowflake.ingest.streaming.internal.serialization.ByteArraySerializer; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java index f0f87e889..8d685dc20 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java @@ -38,7 +38,7 @@ import javax.crypto.BadPaddingException; import javax.crypto.IllegalBlockSizeException; import javax.crypto.NoSuchPaddingException; -import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java index c7df36d65..53da0771b 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java @@ -28,12 +28,12 @@ import net.snowflake.client.jdbc.SnowflakeFileTransferMetadataV1; import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; -import net.snowflake.client.jdbc.internal.apache.commons.io.FileUtils; import net.snowflake.ingest.streaming.internal.fileTransferAgent.IcebergFileTransferAgent; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.Utils; +import org.apache.commons.io.FileUtils; /** Handles uploading files to the Snowflake Streaming Ingest Storage */ class InternalStage implements IStorage { @@ -42,13 +42,13 @@ class InternalStage implements IStorage { /** * Object mapper for parsing the client/configure response to Jackson version the same as * jdbc.internal.fasterxml.jackson. We need two different versions of ObjectMapper because {@link - * SnowflakeFileTransferAgent#getFileTransferMetadatas(net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode)} + * SnowflakeFileTransferAgent#getFileTransferMetadatas(com.fasterxml.jackson.databind.JsonNode)} * expects a different version of json object than {@link StreamingIngestResponse}. TODO: * SNOW-1493470 Align Jackson version */ - private static final net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper + private static final com.fasterxml.jackson.databind.ObjectMapper parseConfigureResponseMapper = - new net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper(); + new com.fasterxml.jackson.databind.ObjectMapper(); private static final long REFRESH_THRESHOLD_IN_MS = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES); @@ -260,7 +260,6 @@ private synchronized void setFileLocationInfo(FileLocationInfo fileLocationInfo) static SnowflakeFileTransferMetadataWithAge createFileTransferMetadataWithAge( FileLocationInfo fileLocationInfo) throws JsonProcessingException, - net.snowflake.client.jdbc.internal.fasterxml.jackson.core.JsonProcessingException, SnowflakeSQLException { final SnowflakeFileTransferMetadataWithAge fileTransferMetadataWithAge; @@ -324,10 +323,10 @@ SnowflakeFileTransferMetadataV1 fetchSignedURL(String fileName) return metadata; } - static net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode + static com.fasterxml.jackson.databind.JsonNode parseFileLocationInfo(FileLocationInfo fileLocationInfo) throws JsonProcessingException, - net.snowflake.client.jdbc.internal.fasterxml.jackson.core.JsonProcessingException { + com.fasterxml.jackson.core.JsonProcessingException { JsonNode fileLocationInfoNode = mapper.valueToTree(fileLocationInfo); // Currently there are a few mismatches between the client/configure response and what diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java index 95ec2a2a3..014f1fd6a 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java @@ -19,7 +19,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import net.snowflake.client.jdbc.internal.google.common.collect.Sets; +import com.google.common.collect.Sets; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.TelemetryService; import net.snowflake.ingest.streaming.InsertValidationResponse; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/PresignedUrlExternalVolume.java b/src/main/java/net/snowflake/ingest/streaming/internal/PresignedUrlExternalVolume.java new file mode 100644 index 000000000..e69de29bb diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java index 918c0a7f4..ae7578b8d 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java @@ -22,7 +22,7 @@ import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS; import java.io.IOException; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.ServiceResponseHandler; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java index 8eb5cc066..5f07b1f37 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java @@ -53,8 +53,8 @@ import javax.management.MalformedObjectNameException; import javax.management.ObjectName; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.OAuthCredential; import net.snowflake.ingest.connection.RequestBuilder; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java index 771d97894..666ddb18a 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java @@ -11,9 +11,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.function.Function; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.ServiceResponseHandler; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java index f4dd51daf..061a1a23d 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java @@ -24,16 +24,16 @@ import net.snowflake.client.jdbc.SnowflakeUtil; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.OperationContext; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.StorageCredentials; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.StorageCredentialsAnonymous; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.StorageException; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.StorageExtendedErrorInformation; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.blob.BlobRequestOptions; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.blob.CloudBlobClient; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.blob.CloudBlobContainer; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAnonymous; +import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.StorageExtendedErrorInformation; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java index 0bd3a6cc1..cee2d9437 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java @@ -22,13 +22,13 @@ import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import net.snowflake.client.jdbc.internal.google.api.gax.rpc.FixedHeaderProvider; -import net.snowflake.client.jdbc.internal.google.cloud.storage.Blob; -import net.snowflake.client.jdbc.internal.google.cloud.storage.BlobId; -import net.snowflake.client.jdbc.internal.google.cloud.storage.BlobInfo; -import net.snowflake.client.jdbc.internal.google.cloud.storage.Storage; -import net.snowflake.client.jdbc.internal.google.cloud.storage.StorageException; -import net.snowflake.client.jdbc.internal.google.cloud.storage.StorageOptions; +import com.google.api.gax.rpc.FixedHeaderProvider; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java index 024ef7a1a..7a8f1849b 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java @@ -27,29 +27,29 @@ import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; import net.snowflake.client.jdbc.cloud.storage.S3HttpUtil; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import net.snowflake.client.jdbc.internal.amazonaws.AmazonClientException; -import net.snowflake.client.jdbc.internal.amazonaws.AmazonServiceException; -import net.snowflake.client.jdbc.internal.amazonaws.ClientConfiguration; -import net.snowflake.client.jdbc.internal.amazonaws.auth.AWSCredentials; -import net.snowflake.client.jdbc.internal.amazonaws.auth.AWSStaticCredentialsProvider; -import net.snowflake.client.jdbc.internal.amazonaws.auth.BasicAWSCredentials; -import net.snowflake.client.jdbc.internal.amazonaws.auth.BasicSessionCredentials; -import net.snowflake.client.jdbc.internal.amazonaws.client.builder.AwsClientBuilder; -import net.snowflake.client.jdbc.internal.amazonaws.client.builder.ExecutorFactory; -import net.snowflake.client.jdbc.internal.amazonaws.regions.Region; -import net.snowflake.client.jdbc.internal.amazonaws.regions.RegionUtils; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.AmazonS3; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.AmazonS3Builder; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.AmazonS3Client; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.model.AmazonS3Exception; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.model.ObjectMetadata; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.model.PutObjectRequest; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.transfer.TransferManager; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.transfer.TransferManagerBuilder; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.transfer.Upload; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.SSLConnectionSocketFactory; -import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.SSLInitializationException; +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.client.builder.ExecutorFactory; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.RegionUtils; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Builder; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.amazonaws.services.s3.transfer.Upload; +import org.apache.http.HttpStatus; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLInitializationException; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java index d4fa9128e..69a65e9ab 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java @@ -7,7 +7,7 @@ import java.util.Map; import net.snowflake.client.jdbc.SnowflakeUtil; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.ObjectMetadata; /** * s3 implementation of platform independent StorageObjectMetadata interface, wraps an S3 diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java index a06337e38..b50658857 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java @@ -6,7 +6,7 @@ import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import net.snowflake.client.jdbc.internal.amazonaws.ClientConfiguration; +import com.amazonaws.ClientConfiguration; import net.snowflake.ingest.utils.Logging; /** diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java index 0da6370f7..0a24230f3 100644 --- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java +++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java @@ -17,29 +17,29 @@ import java.util.regex.Pattern; import javax.net.ssl.SSLContext; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.http.HttpHost; -import net.snowflake.client.jdbc.internal.apache.http.HttpRequest; -import net.snowflake.client.jdbc.internal.apache.http.HttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.NoHttpResponseException; -import net.snowflake.client.jdbc.internal.apache.http.auth.AuthScope; -import net.snowflake.client.jdbc.internal.apache.http.auth.Credentials; -import net.snowflake.client.jdbc.internal.apache.http.auth.UsernamePasswordCredentials; -import net.snowflake.client.jdbc.internal.apache.http.client.CredentialsProvider; -import net.snowflake.client.jdbc.internal.apache.http.client.HttpRequestRetryHandler; -import net.snowflake.client.jdbc.internal.apache.http.client.ServiceUnavailableRetryStrategy; -import net.snowflake.client.jdbc.internal.apache.http.client.config.RequestConfig; -import net.snowflake.client.jdbc.internal.apache.http.client.protocol.HttpClientContext; -import net.snowflake.client.jdbc.internal.apache.http.conn.routing.HttpRoute; -import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.DefaultHostnameVerifier; -import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.SSLConnectionSocketFactory; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.BasicCredentialsProvider; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.HttpClientBuilder; -import net.snowflake.client.jdbc.internal.apache.http.impl.conn.DefaultProxyRoutePlanner; -import net.snowflake.client.jdbc.internal.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import net.snowflake.client.jdbc.internal.apache.http.pool.PoolStats; -import net.snowflake.client.jdbc.internal.apache.http.protocol.HttpContext; -import net.snowflake.client.jdbc.internal.apache.http.ssl.SSLContexts; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.NoHttpResponseException; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.ServiceUnavailableRetryStrategy; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.pool.PoolStats; +import org.apache.http.protocol.HttpContext; +import org.apache.http.ssl.SSLContexts; import net.snowflake.ingest.streaming.internal.StreamingIngestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java index 1759a9d8f..2c3f913a7 100644 --- a/src/main/java/net/snowflake/ingest/utils/Utils.java +++ b/src/main/java/net/snowflake/ingest/utils/Utils.java @@ -7,7 +7,6 @@ import static net.snowflake.ingest.utils.Constants.USER; import com.codahale.metrics.Timer; -import io.netty.util.internal.PlatformDependent; import java.io.IOException; import java.io.StringReader; import java.lang.management.BufferPoolMXBean; @@ -354,9 +353,8 @@ public static void showMemory() { Runtime runtime = Runtime.getRuntime(); logger.logInfo( - "Max direct memory={}, max runtime memory={}, total runtime memory={}, free runtime" + "Max max runtime memory={}, total runtime memory={}, free runtime" + " memory={}", - PlatformDependent.maxDirectMemory(), runtime.maxMemory(), runtime.totalMemory(), runtime.freeMemory()); diff --git a/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java b/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java new file mode 100644 index 000000000..54df85840 --- /dev/null +++ b/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2022-2024 Snowflake Computing Inc. All rights reserved. + */ +package org.apache.commons.collections.map; + +import java.util.Collections; +import java.util.Map; + +/** + * Minimum viable implementation for Hadoop Configuration to avoid commons-collections dependency + */ +public class UnmodifiableMap { + + public static Map decorate(final Map map) { + return Collections.unmodifiableMap(map); + } +} diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java index 20a27eafb..1e1a9036a 100644 --- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java +++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java @@ -19,9 +19,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.Header; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; import net.snowflake.ingest.connection.HistoryRangeResponse; import net.snowflake.ingest.connection.HistoryResponse; import net.snowflake.ingest.connection.IngestResponse; diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java index 008eb8712..99a5f42c0 100644 --- a/src/test/java/net/snowflake/ingest/TestUtils.java +++ b/src/test/java/net/snowflake/ingest/TestUtils.java @@ -46,9 +46,9 @@ import java.util.Properties; import java.util.Random; import java.util.function.Supplier; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.client.utils.URIBuilder; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient; diff --git a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java index ee98bd04d..2b1ba91c8 100644 --- a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java +++ b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java @@ -2,7 +2,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; +import org.apache.http.client.utils.URIBuilder; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; diff --git a/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java b/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java index 94f4e0614..a2b659101 100644 --- a/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java +++ b/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java @@ -4,7 +4,7 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java index 1ce9407f5..2862870b6 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java @@ -11,7 +11,7 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.OpenChannelRequest; import net.snowflake.ingest.utils.ParameterProvider; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java index 3ea364d4c..c116b6ad7 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java @@ -11,7 +11,7 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.OpenChannelRequest; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java index ee07a4cb1..da92c8b98 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java @@ -29,6 +29,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import net.snowflake.client.core.HttpUtil; import net.snowflake.client.core.OCSPMode; import net.snowflake.client.core.SFSessionProperty; @@ -37,15 +38,13 @@ import net.snowflake.client.jdbc.SnowflakeFileTransferMetadataV1; import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; -import net.snowflake.client.jdbc.internal.amazonaws.util.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.entity.BasicHttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.ErrorCode; @@ -172,7 +171,9 @@ public void testPutRemote() throws Exception { capturedMetadata.getStageInfo().getStageType()); InputStream capturedInput = capturedConfig.getUploadStream(); - Assert.assertEquals("Hello Upload", IOUtils.toString(capturedInput)); + final byte[] captured = new byte[12]; + capturedInput.read(captured); + Assert.assertArrayEquals("Hello Upload".getBytes(), captured); } @Test @@ -259,7 +260,9 @@ public void doTestPutRemoteRefreshes() throws Exception { capturedMetadata.getStageInfo().getStageType()); InputStream capturedInput = capturedConfig.getUploadStream(); - Assert.assertEquals("Hello Upload", IOUtils.toString(capturedInput)); + final byte[] captured = new byte[12]; + capturedInput.read(captured); + Assert.assertArrayEquals("Hello Upload".getBytes(), captured); } @Test @@ -655,7 +658,9 @@ public Object answer(org.mockito.invocation.InvocationOnMock invocation) capturedMetadata.getStageInfo().getStageType()); InputStream capturedInput = capturedConfig.getUploadStream(); - Assert.assertEquals("Hello Upload", IOUtils.toString(capturedInput)); + final byte[] captured = new byte[12]; + capturedInput.read(captured); + Assert.assertArrayEquals("Hello Upload".getBytes(), captured); } private HttpEntity createHttpEntity(String content) { diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java b/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java index 8f7f98e39..8505e0e6e 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java @@ -17,15 +17,15 @@ import java.util.List; import java.util.Map; import java.util.function.Function; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.HttpVersion; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.message.BasicStatusLine; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpStatus; +import org.apache.http.HttpVersion; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicStatusLine; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.Logging; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java index 5b1a8aa16..4ce4f3efa 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java @@ -16,7 +16,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.Pair; import net.snowflake.ingest.utils.ParameterProvider; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java index 44949c458..b7ec55d91 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java @@ -29,14 +29,14 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.InsertValidationResponse; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java index 17afada70..e295166a1 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java @@ -38,16 +38,16 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import com.google.common.collect.Sets; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.google.common.collect.Sets; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.DropChannelRequest; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java index be0bf8d20..22a8a1ea4 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java @@ -9,7 +9,7 @@ import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT; import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.JWTManager; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java index 6aadfbcae..711b13091 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java @@ -10,11 +10,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.InputStream; import java.util.ArrayList; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import org.junit.Assert; diff --git a/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java b/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java index a905848e6..99707cda5 100644 --- a/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java +++ b/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java @@ -8,11 +8,11 @@ import java.net.SocketException; import java.net.UnknownHostException; import javax.net.ssl.SSLException; -import net.snowflake.client.jdbc.internal.apache.http.HttpRequest; -import net.snowflake.client.jdbc.internal.apache.http.NoHttpResponseException; -import net.snowflake.client.jdbc.internal.apache.http.RequestLine; -import net.snowflake.client.jdbc.internal.apache.http.client.HttpRequestRetryHandler; -import net.snowflake.client.jdbc.internal.apache.http.client.protocol.HttpClientContext; +import org.apache.http.HttpRequest; +import org.apache.http.NoHttpResponseException; +import org.apache.http.RequestLine; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.protocol.HttpClientContext; import org.junit.Test; import org.mockito.Mockito; From df178a7ba390a02184b235a1ab34c12c79786ac9 Mon Sep 17 00:00:00 2001 From: Dariusz Seweryn Date: Fri, 13 Dec 2024 14:20:16 +0100 Subject: [PATCH 2/7] Fixed formatting. --- .../snowflake/ingest/SimpleIngestManager.java | 8 ++-- .../ingest/connection/OAuthClient.java | 10 ++-- .../ingest/connection/OAuthManager.java | 2 +- .../ingest/connection/RequestBuilder.java | 8 ++-- .../connection/ServiceResponseHandler.java | 2 +- .../ingest/connection/TelemetryService.java | 6 +-- .../internal/DataValidationUtil.java | 2 +- .../streaming/internal/FlushService.java | 2 +- .../streaming/internal/InternalStage.java | 16 +++---- .../streaming/internal/ParquetRowBuffer.java | 2 +- .../internal/SnowflakeServiceClient.java | 2 +- ...nowflakeStreamingIngestClientInternal.java | 4 +- .../internal/StreamingIngestUtils.java | 6 +-- .../fileTransferAgent/IcebergAzureClient.java | 20 ++++---- .../fileTransferAgent/IcebergGCSClient.java | 14 +++--- .../fileTransferAgent/IcebergS3Client.java | 46 +++++++++---------- .../IcebergS3ObjectMetadata.java | 2 +- .../IcebergStorageClientFactory.java | 2 +- .../net/snowflake/ingest/utils/HttpUtil.java | 2 +- .../net/snowflake/ingest/utils/Utils.java | 3 +- .../collections/map/UnmodifiableMap.java | 6 +-- .../net/snowflake/ingest/SimpleIngestIT.java | 6 +-- .../java/net/snowflake/ingest/TestUtils.java | 6 +-- .../ingest/connection/MockOAuthClient.java | 2 +- .../streaming/internal/ChannelCacheTest.java | 2 +- .../internal/InsertRowsBenchmarkTest.java | 2 +- .../streaming/internal/InternalStageTest.java | 14 +++--- .../internal/MockSnowflakeServiceClient.java | 8 ++-- .../internal/RegisterServiceTest.java | 2 +- .../SnowflakeStreamingIngestChannelTest.java | 16 +++---- .../SnowflakeStreamingIngestClientTest.java | 18 ++++---- .../internal/StreamingIngestUtilsIT.java | 2 +- .../internal/StreamingIngestUtilsTest.java | 4 +- 33 files changed, 121 insertions(+), 126 deletions(-) diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java index f9ba34128..441cd4b81 100644 --- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java +++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java @@ -18,10 +18,6 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.HistoryRangeResponse; import net.snowflake.ingest.connection.HistoryResponse; import net.snowflake.ingest.connection.IngestResponse; @@ -32,6 +28,10 @@ import net.snowflake.ingest.utils.HttpUtil; import net.snowflake.ingest.utils.StagedFileWrapper; import net.snowflake.ingest.utils.Utils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java index d1f8384bd..f008b43ce 100644 --- a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java +++ b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java @@ -4,6 +4,8 @@ package net.snowflake.ingest.connection; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; @@ -12,8 +14,9 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; +import net.snowflake.ingest.utils.ErrorCode; +import net.snowflake.ingest.utils.HttpUtil; +import net.snowflake.ingest.utils.SFException; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; @@ -23,9 +26,6 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; -import net.snowflake.ingest.utils.ErrorCode; -import net.snowflake.ingest.utils.HttpUtil; -import net.snowflake.ingest.utils.SFException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java index a33d18da3..01cbcd846 100644 --- a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java +++ b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java @@ -6,10 +6,10 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; -import org.apache.http.client.utils.URIBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; +import org.apache.http.client.utils.URIBuilder; /** This class manages creating and automatically refresh the OAuth token */ public final class OAuthManager extends SecurityManager { diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java index acfa42b0d..74bc0b90f 100644 --- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java +++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java @@ -15,6 +15,10 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import net.snowflake.ingest.utils.ErrorCode; +import net.snowflake.ingest.utils.SFException; +import net.snowflake.ingest.utils.SnowflakeURL; +import net.snowflake.ingest.utils.StagedFileWrapper; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; @@ -23,10 +27,6 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.ingest.utils.ErrorCode; -import net.snowflake.ingest.utils.SFException; -import net.snowflake.ingest.utils.SnowflakeURL; -import net.snowflake.ingest.utils.StagedFileWrapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java index 50d7015f3..a3763615f 100644 --- a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java +++ b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.UUID; +import net.snowflake.ingest.utils.BackOffException; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -17,7 +18,6 @@ import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; -import net.snowflake.ingest.utils.BackOffException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java index 1f3ccf4c5..8821b6a24 100644 --- a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java +++ b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java @@ -10,16 +10,16 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Snapshot; import com.codahale.metrics.Timer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.RateLimiter; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -import org.apache.http.impl.client.CloseableHttpClient; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.client.jdbc.telemetry.TelemetryClient; import net.snowflake.client.jdbc.telemetry.TelemetryUtil; import net.snowflake.ingest.utils.Logging; +import org.apache.http.impl.client.CloseableHttpClient; /** * Telemetry service to collect logs in the SDK and send them to Snowflake through the JDBC client diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java index a1efd3e16..0582b5315 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; +import com.google.common.collect.Sets; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; @@ -37,7 +38,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import com.google.common.collect.Sets; import net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat; import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10; import net.snowflake.ingest.streaming.internal.serialization.ByteArraySerializer; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java index 8d685dc20..b83274adc 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java @@ -12,6 +12,7 @@ import com.codahale.metrics.Timer; import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; import java.lang.management.ManagementFactory; import java.security.InvalidAlgorithmParameterException; @@ -38,7 +39,6 @@ import javax.crypto.BadPaddingException; import javax.crypto.IllegalBlockSizeException; import javax.crypto.NoSuchPaddingException; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java index 53da0771b..21aa1c906 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java @@ -46,9 +46,8 @@ class InternalStage implements IStorage { * expects a different version of json object than {@link StreamingIngestResponse}. TODO: * SNOW-1493470 Align Jackson version */ - private static final com.fasterxml.jackson.databind.ObjectMapper - parseConfigureResponseMapper = - new com.fasterxml.jackson.databind.ObjectMapper(); + private static final com.fasterxml.jackson.databind.ObjectMapper parseConfigureResponseMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); private static final long REFRESH_THRESHOLD_IN_MS = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES); @@ -258,9 +257,7 @@ private synchronized void setFileLocationInfo(FileLocationInfo fileLocationInfo) } static SnowflakeFileTransferMetadataWithAge createFileTransferMetadataWithAge( - FileLocationInfo fileLocationInfo) - throws JsonProcessingException, - SnowflakeSQLException { + FileLocationInfo fileLocationInfo) throws JsonProcessingException, SnowflakeSQLException { final SnowflakeFileTransferMetadataWithAge fileTransferMetadataWithAge; if (fileLocationInfo @@ -323,10 +320,9 @@ SnowflakeFileTransferMetadataV1 fetchSignedURL(String fileName) return metadata; } - static com.fasterxml.jackson.databind.JsonNode - parseFileLocationInfo(FileLocationInfo fileLocationInfo) - throws JsonProcessingException, - com.fasterxml.jackson.core.JsonProcessingException { + static com.fasterxml.jackson.databind.JsonNode parseFileLocationInfo( + FileLocationInfo fileLocationInfo) + throws JsonProcessingException, com.fasterxml.jackson.core.JsonProcessingException { JsonNode fileLocationInfoNode = mapper.valueToTree(fileLocationInfo); // Currently there are a few mismatches between the client/configure response and what diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java index 014f1fd6a..f1f0af47d 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java @@ -6,6 +6,7 @@ import static net.snowflake.ingest.utils.Utils.concatDotPath; +import com.google.common.collect.Sets; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; @@ -19,7 +20,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import com.google.common.collect.Sets; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.TelemetryService; import net.snowflake.ingest.streaming.InsertValidationResponse; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java index ae7578b8d..d3d423dbb 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java @@ -22,13 +22,13 @@ import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS; import java.io.IOException; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.ServiceResponseHandler; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; import net.snowflake.ingest.utils.SFException; +import org.apache.http.impl.client.CloseableHttpClient; /** * The SnowflakeServiceClient class is responsible for making API requests to the Snowflake service. diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java index 5f07b1f37..0d7dd75b6 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java @@ -53,8 +53,6 @@ import javax.management.MalformedObjectNameException; import javax.management.ObjectName; import net.snowflake.client.core.SFSessionProperty; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.OAuthCredential; import net.snowflake.ingest.connection.RequestBuilder; @@ -72,6 +70,8 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.parquet.column.ParquetProperties; /** diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java index 666ddb18a..3c5234a6c 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java @@ -11,15 +11,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.function.Function; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.ServiceResponseHandler; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; import net.snowflake.ingest.utils.SFException; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.CloseableHttpClient; public class StreamingIngestUtils { diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java index 061a1a23d..09ebf284e 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java @@ -3,6 +3,16 @@ import static net.snowflake.client.core.Constants.CLOUD_STORAGE_CREDENTIALS_EXPIRED; import static net.snowflake.client.core.HttpUtil.setSessionlessProxyForAzure; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAnonymous; +import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.StorageExtendedErrorInformation; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -24,16 +34,6 @@ import net.snowflake.client.jdbc.SnowflakeUtil; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import com.microsoft.azure.storage.OperationContext; -import com.microsoft.azure.storage.StorageCredentials; -import com.microsoft.azure.storage.StorageCredentialsAnonymous; -import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; -import com.microsoft.azure.storage.StorageException; -import com.microsoft.azure.storage.StorageExtendedErrorInformation; -import com.microsoft.azure.storage.blob.BlobRequestOptions; -import com.microsoft.azure.storage.blob.CloudBlobClient; -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlockBlob; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java index cee2d9437..d94a728f1 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java @@ -3,6 +3,13 @@ import static net.snowflake.client.jdbc.SnowflakeUtil.getRootCause; import static net.snowflake.client.jdbc.SnowflakeUtil.isBlank; +import com.google.api.gax.rpc.FixedHeaderProvider; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; import com.google.common.base.Strings; import java.io.File; import java.io.FileInputStream; @@ -22,13 +29,6 @@ import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import com.google.api.gax.rpc.FixedHeaderProvider; -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.BlobId; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageException; -import com.google.cloud.storage.StorageOptions; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java index 7a8f1849b..2cb93470a 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java @@ -4,6 +4,26 @@ import static net.snowflake.client.jdbc.SnowflakeUtil.createDefaultExecutorService; import static net.snowflake.client.jdbc.SnowflakeUtil.getRootCause; +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.client.builder.ExecutorFactory; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.RegionUtils; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Builder; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.amazonaws.services.s3.transfer.Upload; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -27,34 +47,14 @@ import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; import net.snowflake.client.jdbc.cloud.storage.S3HttpUtil; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.auth.BasicSessionCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.client.builder.ExecutorFactory; -import com.amazonaws.regions.Region; -import com.amazonaws.regions.RegionUtils; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Builder; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; -import com.amazonaws.services.s3.transfer.Upload; -import org.apache.http.HttpStatus; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLInitializationException; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; import net.snowflake.ingest.utils.Logging; import org.apache.commons.io.IOUtils; +import org.apache.http.HttpStatus; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLInitializationException; class IcebergS3Client implements IcebergStorageClient { private static final Logging logger = new Logging(IcebergS3Client.class); diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java index 69a65e9ab..03f77c4d5 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java @@ -4,10 +4,10 @@ package net.snowflake.ingest.streaming.internal.fileTransferAgent; +import com.amazonaws.services.s3.model.ObjectMetadata; import java.util.Map; import net.snowflake.client.jdbc.SnowflakeUtil; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import com.amazonaws.services.s3.model.ObjectMetadata; /** * s3 implementation of platform independent StorageObjectMetadata interface, wraps an S3 diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java index b50658857..c53439dea 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java @@ -1,12 +1,12 @@ package net.snowflake.ingest.streaming.internal.fileTransferAgent; +import com.amazonaws.ClientConfiguration; import java.util.Map; import java.util.Properties; import net.snowflake.client.core.HttpUtil; import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import com.amazonaws.ClientConfiguration; import net.snowflake.ingest.utils.Logging; /** diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java index 0a24230f3..601ca7a0a 100644 --- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java +++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java @@ -17,6 +17,7 @@ import java.util.regex.Pattern; import javax.net.ssl.SSLContext; import net.snowflake.client.core.SFSessionProperty; +import net.snowflake.ingest.streaming.internal.StreamingIngestUtils; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; @@ -40,7 +41,6 @@ import org.apache.http.pool.PoolStats; import org.apache.http.protocol.HttpContext; import org.apache.http.ssl.SSLContexts; -import net.snowflake.ingest.streaming.internal.StreamingIngestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java index 2c3f913a7..9b584d011 100644 --- a/src/main/java/net/snowflake/ingest/utils/Utils.java +++ b/src/main/java/net/snowflake/ingest/utils/Utils.java @@ -353,8 +353,7 @@ public static void showMemory() { Runtime runtime = Runtime.getRuntime(); logger.logInfo( - "Max max runtime memory={}, total runtime memory={}, free runtime" - + " memory={}", + "Max max runtime memory={}, total runtime memory={}, free runtime" + " memory={}", runtime.maxMemory(), runtime.totalMemory(), runtime.freeMemory()); diff --git a/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java b/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java index 54df85840..d53167738 100644 --- a/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java +++ b/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java @@ -11,7 +11,7 @@ */ public class UnmodifiableMap { - public static Map decorate(final Map map) { - return Collections.unmodifiableMap(map); - } + public static Map decorate(final Map map) { + return Collections.unmodifiableMap(map); + } } diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java index 1e1a9036a..189f0be76 100644 --- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java +++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java @@ -19,13 +19,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.apache.http.Header; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.HttpPost; import net.snowflake.ingest.connection.HistoryRangeResponse; import net.snowflake.ingest.connection.HistoryResponse; import net.snowflake.ingest.connection.IngestResponse; import net.snowflake.ingest.utils.StagedFileWrapper; +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; import org.junit.After; import org.junit.Before; import org.junit.Ignore; diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java index 99a5f42c0..c121f80bf 100644 --- a/src/test/java/net/snowflake/ingest/TestUtils.java +++ b/src/test/java/net/snowflake/ingest/TestUtils.java @@ -21,6 +21,8 @@ import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION; import static net.snowflake.ingest.utils.ParameterProvider.ENABLE_ICEBERG_STREAMING; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.math.BigDecimal; import java.net.URI; @@ -46,9 +48,6 @@ import java.util.Properties; import java.util.Random; import java.util.function.Supplier; -import org.apache.http.client.utils.URIBuilder; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient; @@ -59,6 +58,7 @@ import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; import org.apache.commons.codec.binary.Base64; +import org.apache.http.client.utils.URIBuilder; import org.junit.Assert; public class TestUtils { diff --git a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java index 2b1ba91c8..4188735a9 100644 --- a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java +++ b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java @@ -2,9 +2,9 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; -import org.apache.http.client.utils.URIBuilder; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; +import org.apache.http.client.utils.URIBuilder; /** Mock implementation of {@link OAuthClient}, only use for test */ public class MockOAuthClient extends OAuthClient { diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java index 2862870b6..b59c628f7 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java @@ -11,10 +11,10 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.OpenChannelRequest; import net.snowflake.ingest.utils.ParameterProvider; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.parquet.column.ParquetProperties; import org.junit.Assert; import org.junit.Before; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java index c116b6ad7..ab65c72bb 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java @@ -11,12 +11,12 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.OpenChannelRequest; import net.snowflake.ingest.utils.ParameterProvider; import net.snowflake.ingest.utils.Utils; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.parquet.column.ParquetProperties; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java index da92c8b98..93ce0d6d6 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java @@ -16,6 +16,9 @@ import static net.snowflake.ingest.utils.HttpUtil.shouldBypassProxy; import static org.mockito.Mockito.times; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -29,7 +32,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import net.snowflake.client.core.HttpUtil; import net.snowflake.client.core.OCSPMode; import net.snowflake.client.core.SFSessionProperty; @@ -38,17 +40,15 @@ import net.snowflake.client.jdbc.SnowflakeFileTransferMetadataV1; import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; +import net.snowflake.ingest.TestUtils; +import net.snowflake.ingest.connection.RequestBuilder; +import net.snowflake.ingest.utils.ErrorCode; +import net.snowflake.ingest.utils.SFException; import org.apache.http.HttpEntity; import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.entity.BasicHttpEntity; import org.apache.http.impl.client.CloseableHttpClient; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.ingest.TestUtils; -import net.snowflake.ingest.connection.RequestBuilder; -import net.snowflake.ingest.utils.ErrorCode; -import net.snowflake.ingest.utils.SFException; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java b/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java index 8505e0e6e..d67d3d3a8 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java @@ -17,7 +17,11 @@ import java.util.List; import java.util.Map; import java.util.function.Function; +import net.snowflake.ingest.TestUtils; +import net.snowflake.ingest.connection.RequestBuilder; +import net.snowflake.ingest.utils.Logging; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.HttpVersion; @@ -26,10 +30,6 @@ import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.message.BasicStatusLine; -import net.snowflake.ingest.TestUtils; -import net.snowflake.ingest.connection.RequestBuilder; -import net.snowflake.ingest.utils.Logging; -import org.apache.commons.lang3.tuple.Pair; import org.mockito.Mockito; import org.mockito.stubbing.Answer; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java index 4ce4f3efa..fe15253aa 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java @@ -16,10 +16,10 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.Pair; import net.snowflake.ingest.utils.ParameterProvider; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java index b7ec55d91..192727d3c 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java @@ -29,14 +29,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import net.snowflake.client.core.SFSessionProperty; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.InsertValidationResponse; @@ -48,7 +40,15 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.parquet.column.ParquetProperties; import org.junit.After; import org.junit.Assert; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java index e295166a1..d887b92f2 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import java.io.IOException; import java.io.StringWriter; import java.security.KeyPair; @@ -38,16 +39,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; -import com.google.common.collect.Sets; import net.snowflake.client.core.SFSessionProperty; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpStatus; -import org.apache.http.StatusLine; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.DropChannelRequest; @@ -60,7 +52,15 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Types; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java index 22a8a1ea4..bf79e39fa 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java @@ -9,13 +9,13 @@ import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT; import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS; -import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.JWTManager; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.HttpUtil; import net.snowflake.ingest.utils.SnowflakeURL; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java index 711b13091..854ac66ef 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java @@ -10,13 +10,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.InputStream; import java.util.ArrayList; +import net.snowflake.ingest.TestUtils; +import net.snowflake.ingest.connection.RequestBuilder; import org.apache.commons.io.IOUtils; import org.apache.http.HttpEntity; import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.ingest.TestUtils; -import net.snowflake.ingest.connection.RequestBuilder; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; From fa48359fffad31f3029a25a8ff35a2f6ff4fb2f7 Mon Sep 17 00:00:00 2001 From: Dariusz Seweryn Date: Fri, 13 Dec 2024 14:35:45 +0100 Subject: [PATCH 3/7] Test undeclared dep. no.0 --- pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pom.xml b/pom.xml index 4285d3ebb..54f95e886 100644 --- a/pom.xml +++ b/pom.xml @@ -90,6 +90,11 @@ pom import + + com.amazonaws + aws-java-sdk-core + ${aws.version} + com.google.code.gson gson @@ -180,6 +185,10 @@ javax.servlet javax.servlet-api + + joda-time + joda-time + From 6f6406d698db9350200b199b91cc87ba017aad60 Mon Sep 17 00:00:00 2001 From: Dariusz Seweryn Date: Fri, 13 Dec 2024 15:22:17 +0100 Subject: [PATCH 4/7] Test undeclared dep. no.1 --- pom.xml | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/pom.xml b/pom.xml index 54f95e886..73766f2f9 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,7 @@ 4.5.14 4.4.16 1.11.4 + 1.12.655 1.78.1 1.9.13 1.17.1 @@ -95,6 +96,26 @@ aws-java-sdk-core ${aws.version} + + com.amazonaws + aws-java-sdk-s3 + ${aws.version} + + + com.google.api + gax + 2.31.0 + + + com.google.cloud + google-cloud-core + 2.21.0 + + + com.google.cloud + google-cloud-storage + 2.22.6 + com.google.code.gson gson @@ -120,6 +141,11 @@ protobuf-java ${protobuf.version} + + com.microsoft.azure + azure-storage + 5.0.0 + com.nimbusds nimbus-jose-jwt @@ -579,6 +605,14 @@ + + com.amazonaws + aws-java-sdk-core + + + com.amazonaws + aws-java-sdk-s3 + com.fasterxml.jackson.core jackson-annotations @@ -611,6 +645,48 @@ + + com.google.api + gax + + + + javax.annotation + javax.annotation-api + + + + + com.google.cloud + google-cloud-core + + + + com.google.errorprone + error_prone_annotations + + + + + com.google.cloud + google-cloud-storage + + + + com.google.errorprone + error_prone_annotations + + + io.grpc + grpc-context + + + + javax.annotation + javax.annotation-api + + + com.google.code.findbugs jsr305 @@ -620,6 +696,10 @@ com.google.guava guava + + com.microsoft.azure + azure-storage + @@ -1277,6 +1357,50 @@ + + io.perfmark + ${shadeBase}.io.perfmark + + + io.grpc + ${shadeBase}.io.grpc + + + io.opencensus + ${shadeBase}.io.opencensus + + + org.conscrypt + ${shadeBase}.org.conscrypt + + + com.microsoft + ${shadeBase}.com.microsoft + + + com.yammer + ${shadeBase}.com.yammer + + + org.jsoup + ${shadeBase}.org.jsoup + + + org.joda + ${shadeBase}.org.joda + + + org.threeten + ${shadeBase}.org.threeten + + + javax.servlet + ${shadeBase}.javax.servlet + + + android.annotation + ${shadeBase}.android.annotation + com.nimbusds ${shadeBase}.com.nimbusds @@ -1379,6 +1503,22 @@ org.roaringbitmap ${shadeBase}.org.roaringbitmap + + net.snowflake.client + ${shadeBase}.net.snowflake.client + + + com.snowflake.client + ${shadeBase}.com.snowflake.client + + + com.amazonaws + ${shadeBase}.com.amazonaws + + + software.amazon.ion + ${shadeBase}.software.amazon.ion + From 6206285475d2df7ea3d4bdd61f3f9aad4f6dfe65 Mon Sep 17 00:00:00 2001 From: Dariusz Seweryn Date: Fri, 13 Dec 2024 15:27:10 +0100 Subject: [PATCH 5/7] check_content.sh update with #733 --- scripts/check_content.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/check_content.sh b/scripts/check_content.sh index 7608c23ec..ddad4d642 100755 --- a/scripts/check_content.sh +++ b/scripts/check_content.sh @@ -29,6 +29,7 @@ if jar tvf $DIR/../target/snowflake-ingest-sdk.jar | awk '{print $8}' | \ grep -v properties.dtd | \ grep -v parquet.thrift | \ grep -v assets/org/apache/commons/math3/random/new-joe-kuo-6.1000 | \ + grep -v dependencies.properties | \ # Native zstd libraries are allowed grep -v -E '^darwin' | \ From f75f7374a0e760f92b85cd04b2bb962d02af0c29 Mon Sep 17 00:00:00 2001 From: Dariusz Seweryn Date: Fri, 13 Dec 2024 16:48:20 +0100 Subject: [PATCH 6/7] Fix unshaded .proto with #733 --- pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 73766f2f9..f23153ee6 100644 --- a/pom.xml +++ b/pom.xml @@ -1544,7 +1544,9 @@ LICENSE NOTICE iceberg-build.properties - google/protobuf/**/*.proto + google/**/**/*.proto + grpc/**/**/*.proto + opencensus/proto/**/v1/*.proto THIRD-PARTY.txt From dfe5d545084e0d7149ac7c0f70e9eb87d223bb37 Mon Sep 17 00:00:00 2001 From: Dariusz Seweryn Date: Fri, 13 Dec 2024 17:21:02 +0100 Subject: [PATCH 7/7] Try fix shaded client sdk classes (from source branch) --- .../core/src/main/java/net/snowflake/IngestTestUtils.java | 6 +++++- pom.xml | 8 -------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java b/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java index 5b347a72e..195d13b5e 100644 --- a/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java +++ b/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java @@ -110,7 +110,11 @@ private Properties loadProperties() throws IOException { private Connection getConnection() throws IOException, ClassNotFoundException, SQLException, NoSuchAlgorithmException, InvalidKeySpecException { - Class.forName("net.snowflake.client.jdbc.SnowflakeDriver"); + try { + Class.forName("net.snowflake.client.jdbc.SnowflakeDriver"); + } catch (ClassNotFoundException e) { + Class.forName("net.snowflake.ingest.internal.net.snowflake.client.jdbc.SnowflakeDriver"); + } Properties loadedProps = loadProperties(); diff --git a/pom.xml b/pom.xml index f23153ee6..0a3191b50 100644 --- a/pom.xml +++ b/pom.xml @@ -1503,14 +1503,6 @@ org.roaringbitmap ${shadeBase}.org.roaringbitmap - - net.snowflake.client - ${shadeBase}.net.snowflake.client - - - com.snowflake.client - ${shadeBase}.com.snowflake.client - com.amazonaws ${shadeBase}.com.amazonaws