diff --git a/pom.xml b/pom.xml index b4610c9ac..4285d3ebb 100644 --- a/pom.xml +++ b/pom.xml @@ -38,19 +38,23 @@ + 4.5.14 + 4.4.16 1.11.4 1.78.1 1.9.13 1.17.1 3.2.2 - 1.26.2 + 1.27.1 2.10.1 - 2.16.1 - 3.14.0 + 2.17.0 + 3.17.0 1.3.3 1.11.0 2.17.2 - 32.0.1-jre + 1.43.3 + 2.10.1 + 32.1.1-jre 3.4.0 1.6.1 true @@ -63,9 +67,9 @@ 1.8 2.4.9 4.1.115.Final - 9.37.3 + 9.47 3.1 - 1.14.1 + 1.14.4 2.0.9 UTF-8 4.27.5 @@ -73,6 +77,7 @@ 1.7.36 1.1.10.5 3.20.0 + 1.6.9 0.13.0 @@ -85,11 +90,26 @@ pom import + + com.google.code.gson + gson + ${gson.version} + com.google.guava guava ${guava.version} + + com.google.http-client + google-http-client + ${google.httpclient.version} + + + com.google.http-client + google-http-client-gson + ${google.httpclient.version} + com.google.protobuf protobuf-java @@ -137,8 +157,30 @@ net.snowflake - snowflake-jdbc + snowflake-jdbc-thin ${snowjdbc.version} + + + com.google.errorprone + error_prone_annotations + + + commons-logging + commons-logging + + + io.grpc + grpc-context + + + javax.annotation + javax.annotation-api + + + javax.servlet + javax.servlet-api + + org.apache.avro @@ -173,6 +215,14 @@ com.github.pjfanning jersey-json + + com.google.code.gson + gson + + + com.google.re2j + re2j + com.jcraft jsch @@ -189,10 +239,26 @@ com.sun.jersey jersey-servlet + + commons-beanutils + commons-beanutils + + + commons-cli + commons-cli + + + commons-collections + commons-collections + commons-logging commons-logging + + commons-net + commons-net + dnsjava dnsjava @@ -209,6 +275,10 @@ io.netty netty-transport-native-epoll + + jakarta.activation + jakarta.activation-api + javax.activation activation @@ -225,6 +295,22 @@ org.apache.avro avro + + org.apache.commons + commons-compress + + + org.apache.commons + commons-configuration2 + + + org.apache.commons + commons-math3 + + + org.apache.commons + commons-text + org.apache.curator curator-client @@ -233,10 +319,18 @@ org.apache.curator curator-recipes + + org.apache.hadoop + hadoop-annotations + org.apache.hadoop hadoop-auth + + org.apache.hadoop.thirdparty + hadoop-shaded-protobuf_3_21 + org.apache.httpcomponents httpclient @@ -253,6 +347,10 @@ org.bouncycastle bcprov-jdk15on + + org.codehaus.jettison + jettison + org.eclipse.jetty jetty-server @@ -279,6 +377,16 @@ + + org.apache.httpcomponents + httpclient + ${apache.httpclient.version} + + + org.apache.httpcomponents + httpcore + ${apache.httpcomponents.version} + org.apache.iceberg iceberg-api @@ -308,6 +416,12 @@ org.apache.parquet parquet-hadoop ${parquet.version} + + + com.github.luben + zstd-jni + + org.apache.yetus @@ -339,11 +453,6 @@ jackson-xc ${codehaus.version} - - org.codehaus.jettison - jettison - 1.5.4 - org.codehaus.woodstox stax2-api @@ -354,11 +463,21 @@ objenesis ${objenesis.version} + + org.slf4j + jcl-over-slf4j + ${slf4j.version} + org.slf4j slf4j-api ${slf4j.version} + + org.threeten + threetenbp + ${threetenbp.version} + org.xerial.snappy snappy-java @@ -527,10 +646,6 @@ metrics-jvm 4.1.22 - - io.netty - netty-common - net.bytebuddy byte-buddy-agent @@ -538,18 +653,10 @@ net.snowflake - snowflake-jdbc - - - org.apache.commons - commons-compress + snowflake-jdbc-thin - - org.apache.commons - commons-configuration2 - org.apache.commons commons-text @@ -562,6 +669,14 @@ org.apache.hadoop hadoop-mapreduce-client-core + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + org.apache.iceberg iceberg-api @@ -626,7 +741,7 @@ com.github.luben zstd-jni - 1.5.6-2 + 1.5.6-5 runtime @@ -908,7 +1023,15 @@ - + + + commons-io:commons-io + commons-codec:commons-codec + org.apache.avro:avro + org.apache.commons:commons-compress + org.apache.commons:commons-lang3 + + @@ -983,8 +1106,9 @@ 3-Clause BSD License The MIT License EDL 1.0 - The Go license + The Go License Bouncy Castle Licence + BSD New license test,provided,system true @@ -999,8 +1123,9 @@ |Apache 2 BSD 2-Clause License |The BSD License |BSD - The MIT License|MIT License + The MIT License|MIT License|MIT license 3-Clause BSD License|BSD-3-Clause + The Go License|Go License @@ -1137,7 +1262,7 @@ - net.snowflake:snowflake-jdbc + net.snowflake:snowflake-jdbc-thin org.slf4j:slf4j-api com.github.luben:zstd-jni diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py index 9f715abd6..54470fdf7 100644 --- a/scripts/process_licenses.py +++ b/scripts/process_licenses.py @@ -27,13 +27,16 @@ APACHE_LICENSE = "Apache License 2.0" BSD_2_CLAUSE_LICENSE = "2-Clause BSD License" BSD_3_CLAUSE_LICENSE = "3-Clause BSD License" +BSD_NEW_LICENSE = "BSD New license" EDL_10_LICENSE = "EDL 1.0" MIT_LICENSE = "The MIT License" +MIT_LICENSE_ALT = "MIT License" GO_LICENSE = "The Go license" +GO_LICENSE_ALT = "Go License" BOUNCY_CASTLE_LICENSE = "Bouncy Castle Licence " # The SDK does not need to include licenses of dependencies, which aren't shaded -IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc", "org.slf4j:slf4j-api"} +IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc-thin", "org.slf4j:slf4j-api"} # List of dependencies, which don't ship with a license file. # Only add a new record here after verifying that the dependency JAR does not contain a license! @@ -50,7 +53,7 @@ "com.nimbusds:nimbus-jose-jwt": APACHE_LICENSE, "com.github.stephenc.jcip:jcip-annotations": APACHE_LICENSE, "io.netty:netty-common": APACHE_LICENSE, - "com.google.re2j:re2j": GO_LICENSE, + "com.google.re2j:re2j": GO_LICENSE_ALT, "com.google.protobuf:protobuf-java": BSD_3_CLAUSE_LICENSE, "com.google.code.gson:gson": APACHE_LICENSE, "org.xerial.snappy:snappy-java": APACHE_LICENSE, @@ -64,6 +67,64 @@ "com.thoughtworks.paranamer:paranamer": BSD_2_CLAUSE_LICENSE, "org.roaringbitmap:RoaringBitmap": APACHE_LICENSE, "org.roaringbitmap:shims": APACHE_LICENSE, + "com.amazonaws:aws-java-sdk-core": APACHE_LICENSE, + "software.amazon.ion:ion-java": APACHE_LICENSE, + "com.amazonaws:aws-java-sdk-kms": APACHE_LICENSE, + "com.amazonaws:jmespath-java": APACHE_LICENSE, + "com.amazonaws:aws-java-sdk-s3": APACHE_LICENSE, + "com.google.api:gax": BSD_3_CLAUSE_LICENSE, + "com.google.api:api-common": BSD_3_CLAUSE_LICENSE, + "com.google.auth:google-auth-library-credentials": BSD_NEW_LICENSE, + "com.google.api.grpc:proto-google-common-protos": APACHE_LICENSE, + "io.opencensus:opencensus-api": APACHE_LICENSE, + "com.google.auth:google-auth-library-oauth2-http": BSD_NEW_LICENSE, + "com.google.cloud:google-cloud-core": APACHE_LICENSE, + "com.google.auto.value:auto-value-annotations": APACHE_LICENSE, + "com.google.protobuf:protobuf-java-util": BSD_3_CLAUSE_LICENSE, + "com.google.api.grpc:proto-google-iam-v1": APACHE_LICENSE, + "com.google.cloud:google-cloud-storage": APACHE_LICENSE, + "io.opencensus:opencensus-contrib-http-util": APACHE_LICENSE, + "com.google.http-client:google-http-client-jackson2": APACHE_LICENSE, + "com.google.api-client:google-api-client": APACHE_LICENSE, + "com.google.oauth-client:google-oauth-client": APACHE_LICENSE, + "com.google.http-client:google-http-client-apache-v2": APACHE_LICENSE, + "com.google.apis:google-api-services-storage": APACHE_LICENSE, + "com.google.cloud:google-cloud-core-http": APACHE_LICENSE, + "com.google.http-client:google-http-client-appengine": APACHE_LICENSE, + "com.google.api:gax-httpjson": BSD_3_CLAUSE_LICENSE, + "com.google.cloud:google-cloud-core-grpc": APACHE_LICENSE, + "com.google.api:gax-grpc": BSD_3_CLAUSE_LICENSE, + "io.grpc:grpc-alts": APACHE_LICENSE, + "io.grpc:grpc-grpclb": APACHE_LICENSE, + "org.conscrypt:conscrypt-openjdk-uber": APACHE_LICENSE, + "io.grpc:grpc-auth": APACHE_LICENSE, + "io.grpc:grpc-protobuf": APACHE_LICENSE, + "io.grpc:grpc-protobuf-lite": APACHE_LICENSE, + "com.google.api.grpc:proto-google-cloud-storage-v2": APACHE_LICENSE, + "com.google.api.grpc:grpc-google-cloud-storage-v2": APACHE_LICENSE, + "com.google.api.grpc:gapic-google-cloud-storage-v2": APACHE_LICENSE, + "io.grpc:grpc-api": APACHE_LICENSE, + "io.perfmark:perfmark-api": APACHE_LICENSE, + "io.grpc:grpc-core": APACHE_LICENSE, + "com.google.android:annotations": APACHE_LICENSE, + "org.codehaus.mojo:animal-sniffer-annotations": MIT_LICENSE_ALT, + "io.grpc:grpc-stub": APACHE_LICENSE, + "io.grpc:grpc-googleapis": APACHE_LICENSE, + "io.grpc:grpc-xds": APACHE_LICENSE, + "io.opencensus:opencensus-proto": APACHE_LICENSE, + "io.grpc:grpc-services": APACHE_LICENSE, + "io.grpc:grpc-rls": APACHE_LICENSE, + "com.microsoft.azure:azure-storage": APACHE_LICENSE, + "com.microsoft.azure:azure-keyvault-core": APACHE_LICENSE, + "com.yammer.metrics:metrics-core": APACHE_LICENSE, + "com.yammer.metrics:metrics-servlet": APACHE_LICENSE, + "net.minidev:json-smart": APACHE_LICENSE, + "net.minidev:accessors-smart": APACHE_LICENSE, + "org.ow2.asm:asm": BSD_3_CLAUSE_LICENSE, + "com.google.flatbuffers:flatbuffers-java": APACHE_LICENSE, + "com.google.http-client:google-http-client": APACHE_LICENSE, + "com.google.http-client:google-http-client-gson": APACHE_LICENSE, + "io.grpc:grpc-context": APACHE_LICENSE, } diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java index 17caa2b9b..f9ba34128 100644 --- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java +++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java @@ -18,10 +18,10 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.HistoryRangeResponse; import net.snowflake.ingest.connection.HistoryResponse; import net.snowflake.ingest.connection.IngestResponse; diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java index a592899ea..d1f8384bd 100644 --- a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java +++ b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java @@ -6,23 +6,23 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; import java.net.URLEncoder; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.apache.http.entity.ContentType; -import net.snowflake.client.jdbc.internal.apache.http.entity.StringEntity; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.util.EntityUtils; -import net.snowflake.client.jdbc.internal.google.api.client.http.HttpStatusCodes; -import net.snowflake.client.jdbc.internal.google.gson.JsonObject; -import net.snowflake.client.jdbc.internal.google.gson.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.HttpUtil; import net.snowflake.ingest.utils.SFException; @@ -44,6 +44,8 @@ public class OAuthClient { private static final String REFRESH_TOKEN = "refresh_token"; private static final String EXPIRES_IN = "expires_in"; + private static final ObjectMapper objectMapper = new ObjectMapper(); + // OAuth credential private final AtomicReference oAuthCredential; @@ -75,14 +77,14 @@ public void refreshToken() throws IOException { CloseableHttpResponse httpResponse = httpClient.execute(makeRefreshTokenRequest()); String respBodyString = EntityUtils.toString(httpResponse.getEntity()); - if (httpResponse.getStatusLine().getStatusCode() == HttpStatusCodes.STATUS_CODE_OK) { - JsonObject respBody = JsonParser.parseString(respBodyString).getAsJsonObject(); + if (httpResponse.getStatusLine().getStatusCode() == HttpURLConnection.HTTP_OK) { + JsonNode respBody = objectMapper.readTree(respBodyString); if (respBody.has(ACCESS_TOKEN) && respBody.has(EXPIRES_IN)) { // Trim surrounding quotation marks String newAccessToken = respBody.get(ACCESS_TOKEN).toString().replaceAll("^\"|\"$", ""); oAuthCredential.get().setAccessToken(newAccessToken); - oAuthCredential.get().setExpiresIn(respBody.get(EXPIRES_IN).getAsInt()); + oAuthCredential.get().setExpiresIn(respBody.get(EXPIRES_IN).asInt()); return; } } diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java index f9e614a27..a33d18da3 100644 --- a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java +++ b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java @@ -6,7 +6,7 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; +import org.apache.http.client.utils.URIBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java index ec275e118..acfa42b0d 100644 --- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java +++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java @@ -15,14 +15,14 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.apache.http.entity.ContentType; -import net.snowflake.client.jdbc.internal.apache.http.entity.StringEntity; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; diff --git a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java index c9968bcf3..50d7015f3 100644 --- a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java +++ b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java @@ -8,15 +8,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.UUID; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.util.EntityUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; import net.snowflake.ingest.utils.BackOffException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java index 16688f290..1f3ccf4c5 100644 --- a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java +++ b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java @@ -14,9 +14,9 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.impl.client.CloseableHttpClient; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.client.jdbc.telemetry.TelemetryClient; import net.snowflake.client.jdbc.telemetry.TelemetryUtil; import net.snowflake.ingest.utils.Logging; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java index 3ccecfdb5..a1efd3e16 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java @@ -37,7 +37,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import net.snowflake.client.jdbc.internal.google.common.collect.Sets; +import com.google.common.collect.Sets; import net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat; import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10; import net.snowflake.ingest.streaming.internal.serialization.ByteArraySerializer; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java index f0f87e889..8d685dc20 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java @@ -38,7 +38,7 @@ import javax.crypto.BadPaddingException; import javax.crypto.IllegalBlockSizeException; import javax.crypto.NoSuchPaddingException; -import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java index c7df36d65..53da0771b 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/InternalStage.java @@ -28,12 +28,12 @@ import net.snowflake.client.jdbc.SnowflakeFileTransferMetadataV1; import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; -import net.snowflake.client.jdbc.internal.apache.commons.io.FileUtils; import net.snowflake.ingest.streaming.internal.fileTransferAgent.IcebergFileTransferAgent; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.Utils; +import org.apache.commons.io.FileUtils; /** Handles uploading files to the Snowflake Streaming Ingest Storage */ class InternalStage implements IStorage { @@ -42,13 +42,13 @@ class InternalStage implements IStorage { /** * Object mapper for parsing the client/configure response to Jackson version the same as * jdbc.internal.fasterxml.jackson. We need two different versions of ObjectMapper because {@link - * SnowflakeFileTransferAgent#getFileTransferMetadatas(net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode)} + * SnowflakeFileTransferAgent#getFileTransferMetadatas(com.fasterxml.jackson.databind.JsonNode)} * expects a different version of json object than {@link StreamingIngestResponse}. TODO: * SNOW-1493470 Align Jackson version */ - private static final net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper + private static final com.fasterxml.jackson.databind.ObjectMapper parseConfigureResponseMapper = - new net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper(); + new com.fasterxml.jackson.databind.ObjectMapper(); private static final long REFRESH_THRESHOLD_IN_MS = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES); @@ -260,7 +260,6 @@ private synchronized void setFileLocationInfo(FileLocationInfo fileLocationInfo) static SnowflakeFileTransferMetadataWithAge createFileTransferMetadataWithAge( FileLocationInfo fileLocationInfo) throws JsonProcessingException, - net.snowflake.client.jdbc.internal.fasterxml.jackson.core.JsonProcessingException, SnowflakeSQLException { final SnowflakeFileTransferMetadataWithAge fileTransferMetadataWithAge; @@ -324,10 +323,10 @@ SnowflakeFileTransferMetadataV1 fetchSignedURL(String fileName) return metadata; } - static net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode + static com.fasterxml.jackson.databind.JsonNode parseFileLocationInfo(FileLocationInfo fileLocationInfo) throws JsonProcessingException, - net.snowflake.client.jdbc.internal.fasterxml.jackson.core.JsonProcessingException { + com.fasterxml.jackson.core.JsonProcessingException { JsonNode fileLocationInfoNode = mapper.valueToTree(fileLocationInfo); // Currently there are a few mismatches between the client/configure response and what diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java index 95ec2a2a3..014f1fd6a 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java @@ -19,7 +19,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import net.snowflake.client.jdbc.internal.google.common.collect.Sets; +import com.google.common.collect.Sets; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.TelemetryService; import net.snowflake.ingest.streaming.InsertValidationResponse; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/PresignedUrlExternalVolume.java b/src/main/java/net/snowflake/ingest/streaming/internal/PresignedUrlExternalVolume.java new file mode 100644 index 000000000..e69de29bb diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java index 918c0a7f4..ae7578b8d 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeServiceClient.java @@ -22,7 +22,7 @@ import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS; import java.io.IOException; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.ServiceResponseHandler; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java index 8eb5cc066..5f07b1f37 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java @@ -53,8 +53,8 @@ import javax.management.MalformedObjectNameException; import javax.management.ObjectName; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.OAuthCredential; import net.snowflake.ingest.connection.RequestBuilder; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java index 771d97894..666ddb18a 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java @@ -11,9 +11,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.function.Function; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.ServiceResponseHandler; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java index f4dd51daf..061a1a23d 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergAzureClient.java @@ -24,16 +24,16 @@ import net.snowflake.client.jdbc.SnowflakeUtil; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.OperationContext; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.StorageCredentials; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.StorageCredentialsAnonymous; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.StorageException; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.StorageExtendedErrorInformation; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.blob.BlobRequestOptions; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.blob.CloudBlobClient; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.blob.CloudBlobContainer; -import net.snowflake.client.jdbc.internal.microsoft.azure.storage.blob.CloudBlockBlob; +import com.microsoft.azure.storage.OperationContext; +import com.microsoft.azure.storage.StorageCredentials; +import com.microsoft.azure.storage.StorageCredentialsAnonymous; +import com.microsoft.azure.storage.StorageCredentialsSharedAccessSignature; +import com.microsoft.azure.storage.StorageException; +import com.microsoft.azure.storage.StorageExtendedErrorInformation; +import com.microsoft.azure.storage.blob.BlobRequestOptions; +import com.microsoft.azure.storage.blob.CloudBlobClient; +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import com.microsoft.azure.storage.blob.CloudBlockBlob; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java index 0bd3a6cc1..cee2d9437 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergGCSClient.java @@ -22,13 +22,13 @@ import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import net.snowflake.client.jdbc.internal.google.api.gax.rpc.FixedHeaderProvider; -import net.snowflake.client.jdbc.internal.google.cloud.storage.Blob; -import net.snowflake.client.jdbc.internal.google.cloud.storage.BlobId; -import net.snowflake.client.jdbc.internal.google.cloud.storage.BlobInfo; -import net.snowflake.client.jdbc.internal.google.cloud.storage.Storage; -import net.snowflake.client.jdbc.internal.google.cloud.storage.StorageException; -import net.snowflake.client.jdbc.internal.google.cloud.storage.StorageOptions; +import com.google.api.gax.rpc.FixedHeaderProvider; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java index 024ef7a1a..7a8f1849b 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3Client.java @@ -27,29 +27,29 @@ import net.snowflake.client.jdbc.SnowflakeSQLLoggedException; import net.snowflake.client.jdbc.cloud.storage.S3HttpUtil; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import net.snowflake.client.jdbc.internal.amazonaws.AmazonClientException; -import net.snowflake.client.jdbc.internal.amazonaws.AmazonServiceException; -import net.snowflake.client.jdbc.internal.amazonaws.ClientConfiguration; -import net.snowflake.client.jdbc.internal.amazonaws.auth.AWSCredentials; -import net.snowflake.client.jdbc.internal.amazonaws.auth.AWSStaticCredentialsProvider; -import net.snowflake.client.jdbc.internal.amazonaws.auth.BasicAWSCredentials; -import net.snowflake.client.jdbc.internal.amazonaws.auth.BasicSessionCredentials; -import net.snowflake.client.jdbc.internal.amazonaws.client.builder.AwsClientBuilder; -import net.snowflake.client.jdbc.internal.amazonaws.client.builder.ExecutorFactory; -import net.snowflake.client.jdbc.internal.amazonaws.regions.Region; -import net.snowflake.client.jdbc.internal.amazonaws.regions.RegionUtils; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.AmazonS3; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.AmazonS3Builder; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.AmazonS3Client; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.model.AmazonS3Exception; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.model.ObjectMetadata; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.model.PutObjectRequest; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.transfer.TransferManager; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.transfer.TransferManagerBuilder; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.transfer.Upload; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.SSLConnectionSocketFactory; -import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.SSLInitializationException; +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.client.builder.ExecutorFactory; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.RegionUtils; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Builder; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.amazonaws.services.s3.transfer.Upload; +import org.apache.http.HttpStatus; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLInitializationException; import net.snowflake.client.jdbc.internal.snowflake.common.core.SqlState; import net.snowflake.client.util.SFPair; import net.snowflake.client.util.Stopwatch; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java index d4fa9128e..69a65e9ab 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergS3ObjectMetadata.java @@ -7,7 +7,7 @@ import java.util.Map; import net.snowflake.client.jdbc.SnowflakeUtil; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import net.snowflake.client.jdbc.internal.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.ObjectMetadata; /** * s3 implementation of platform independent StorageObjectMetadata interface, wraps an S3 diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java index a06337e38..b50658857 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/fileTransferAgent/IcebergStorageClientFactory.java @@ -6,7 +6,7 @@ import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; import net.snowflake.client.jdbc.cloud.storage.StorageObjectMetadata; -import net.snowflake.client.jdbc.internal.amazonaws.ClientConfiguration; +import com.amazonaws.ClientConfiguration; import net.snowflake.ingest.utils.Logging; /** diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java index 0da6370f7..0a24230f3 100644 --- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java +++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java @@ -17,29 +17,29 @@ import java.util.regex.Pattern; import javax.net.ssl.SSLContext; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.http.HttpHost; -import net.snowflake.client.jdbc.internal.apache.http.HttpRequest; -import net.snowflake.client.jdbc.internal.apache.http.HttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.NoHttpResponseException; -import net.snowflake.client.jdbc.internal.apache.http.auth.AuthScope; -import net.snowflake.client.jdbc.internal.apache.http.auth.Credentials; -import net.snowflake.client.jdbc.internal.apache.http.auth.UsernamePasswordCredentials; -import net.snowflake.client.jdbc.internal.apache.http.client.CredentialsProvider; -import net.snowflake.client.jdbc.internal.apache.http.client.HttpRequestRetryHandler; -import net.snowflake.client.jdbc.internal.apache.http.client.ServiceUnavailableRetryStrategy; -import net.snowflake.client.jdbc.internal.apache.http.client.config.RequestConfig; -import net.snowflake.client.jdbc.internal.apache.http.client.protocol.HttpClientContext; -import net.snowflake.client.jdbc.internal.apache.http.conn.routing.HttpRoute; -import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.DefaultHostnameVerifier; -import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.SSLConnectionSocketFactory; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.BasicCredentialsProvider; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.HttpClientBuilder; -import net.snowflake.client.jdbc.internal.apache.http.impl.conn.DefaultProxyRoutePlanner; -import net.snowflake.client.jdbc.internal.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import net.snowflake.client.jdbc.internal.apache.http.pool.PoolStats; -import net.snowflake.client.jdbc.internal.apache.http.protocol.HttpContext; -import net.snowflake.client.jdbc.internal.apache.http.ssl.SSLContexts; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.NoHttpResponseException; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.ServiceUnavailableRetryStrategy; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.pool.PoolStats; +import org.apache.http.protocol.HttpContext; +import org.apache.http.ssl.SSLContexts; import net.snowflake.ingest.streaming.internal.StreamingIngestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/utils/Utils.java b/src/main/java/net/snowflake/ingest/utils/Utils.java index 1759a9d8f..2c3f913a7 100644 --- a/src/main/java/net/snowflake/ingest/utils/Utils.java +++ b/src/main/java/net/snowflake/ingest/utils/Utils.java @@ -7,7 +7,6 @@ import static net.snowflake.ingest.utils.Constants.USER; import com.codahale.metrics.Timer; -import io.netty.util.internal.PlatformDependent; import java.io.IOException; import java.io.StringReader; import java.lang.management.BufferPoolMXBean; @@ -354,9 +353,8 @@ public static void showMemory() { Runtime runtime = Runtime.getRuntime(); logger.logInfo( - "Max direct memory={}, max runtime memory={}, total runtime memory={}, free runtime" + "Max max runtime memory={}, total runtime memory={}, free runtime" + " memory={}", - PlatformDependent.maxDirectMemory(), runtime.maxMemory(), runtime.totalMemory(), runtime.freeMemory()); diff --git a/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java b/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java new file mode 100644 index 000000000..54df85840 --- /dev/null +++ b/src/main/java/org/apache/commons/collections/map/UnmodifiableMap.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2022-2024 Snowflake Computing Inc. All rights reserved. + */ +package org.apache.commons.collections.map; + +import java.util.Collections; +import java.util.Map; + +/** + * Minimum viable implementation for Hadoop Configuration to avoid commons-collections dependency + */ +public class UnmodifiableMap { + + public static Map decorate(final Map map) { + return Collections.unmodifiableMap(map); + } +} diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java index 20a27eafb..1e1a9036a 100644 --- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java +++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java @@ -19,9 +19,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.Header; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; import net.snowflake.ingest.connection.HistoryRangeResponse; import net.snowflake.ingest.connection.HistoryResponse; import net.snowflake.ingest.connection.IngestResponse; diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java index 008eb8712..99a5f42c0 100644 --- a/src/test/java/net/snowflake/ingest/TestUtils.java +++ b/src/test/java/net/snowflake/ingest/TestUtils.java @@ -46,9 +46,9 @@ import java.util.Properties; import java.util.Random; import java.util.function.Supplier; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.client.utils.URIBuilder; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestClient; diff --git a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java index ee98bd04d..2b1ba91c8 100644 --- a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java +++ b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java @@ -2,7 +2,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; +import org.apache.http.client.utils.URIBuilder; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; diff --git a/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java b/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java index 94f4e0614..a2b659101 100644 --- a/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java +++ b/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java @@ -4,7 +4,7 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java index 1ce9407f5..2862870b6 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/ChannelCacheTest.java @@ -11,7 +11,7 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.OpenChannelRequest; import net.snowflake.ingest.utils.ParameterProvider; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java index 3ea364d4c..c116b6ad7 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/InsertRowsBenchmarkTest.java @@ -11,7 +11,7 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.OpenChannelRequest; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java index ee07a4cb1..da92c8b98 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/InternalStageTest.java @@ -29,6 +29,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import net.snowflake.client.core.HttpUtil; import net.snowflake.client.core.OCSPMode; import net.snowflake.client.core.SFSessionProperty; @@ -37,15 +38,13 @@ import net.snowflake.client.jdbc.SnowflakeFileTransferMetadataV1; import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; -import net.snowflake.client.jdbc.internal.amazonaws.util.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.entity.BasicHttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder; +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.ErrorCode; @@ -172,7 +171,9 @@ public void testPutRemote() throws Exception { capturedMetadata.getStageInfo().getStageType()); InputStream capturedInput = capturedConfig.getUploadStream(); - Assert.assertEquals("Hello Upload", IOUtils.toString(capturedInput)); + final byte[] captured = new byte[12]; + capturedInput.read(captured); + Assert.assertArrayEquals("Hello Upload".getBytes(), captured); } @Test @@ -259,7 +260,9 @@ public void doTestPutRemoteRefreshes() throws Exception { capturedMetadata.getStageInfo().getStageType()); InputStream capturedInput = capturedConfig.getUploadStream(); - Assert.assertEquals("Hello Upload", IOUtils.toString(capturedInput)); + final byte[] captured = new byte[12]; + capturedInput.read(captured); + Assert.assertArrayEquals("Hello Upload".getBytes(), captured); } @Test @@ -655,7 +658,9 @@ public Object answer(org.mockito.invocation.InvocationOnMock invocation) capturedMetadata.getStageInfo().getStageType()); InputStream capturedInput = capturedConfig.getUploadStream(); - Assert.assertEquals("Hello Upload", IOUtils.toString(capturedInput)); + final byte[] captured = new byte[12]; + capturedInput.read(captured); + Assert.assertArrayEquals("Hello Upload".getBytes(), captured); } private HttpEntity createHttpEntity(String content) { diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java b/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java index 8f7f98e39..8505e0e6e 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/MockSnowflakeServiceClient.java @@ -17,15 +17,15 @@ import java.util.List; import java.util.Map; import java.util.function.Function; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.HttpVersion; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.message.BasicStatusLine; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpStatus; +import org.apache.http.HttpVersion; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicStatusLine; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.Logging; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java index 5b1a8aa16..4ce4f3efa 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/RegisterServiceTest.java @@ -16,7 +16,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.Pair; import net.snowflake.ingest.utils.ParameterProvider; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java index 44949c458..b7ec55d91 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java @@ -29,14 +29,14 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.InsertValidationResponse; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java index 17afada70..e295166a1 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java @@ -38,16 +38,16 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import com.google.common.collect.Sets; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.google.common.collect.Sets; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.DropChannelRequest; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java index be0bf8d20..22a8a1ea4 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java @@ -9,7 +9,7 @@ import static net.snowflake.ingest.utils.Constants.CLIENT_CONFIGURE_ENDPOINT; import static net.snowflake.ingest.utils.Constants.RESPONSE_SUCCESS; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.JWTManager; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java index 6aadfbcae..711b13091 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java @@ -10,11 +10,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.InputStream; import java.util.ArrayList; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import org.junit.Assert; diff --git a/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java b/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java index a905848e6..99707cda5 100644 --- a/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java +++ b/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java @@ -8,11 +8,11 @@ import java.net.SocketException; import java.net.UnknownHostException; import javax.net.ssl.SSLException; -import net.snowflake.client.jdbc.internal.apache.http.HttpRequest; -import net.snowflake.client.jdbc.internal.apache.http.NoHttpResponseException; -import net.snowflake.client.jdbc.internal.apache.http.RequestLine; -import net.snowflake.client.jdbc.internal.apache.http.client.HttpRequestRetryHandler; -import net.snowflake.client.jdbc.internal.apache.http.client.protocol.HttpClientContext; +import org.apache.http.HttpRequest; +import org.apache.http.NoHttpResponseException; +import org.apache.http.RequestLine; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.protocol.HttpClientContext; import org.junit.Test; import org.mockito.Mockito;