From b42bdfc3b250593637e6ebc218e532a090fa8e76 Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Tue, 9 Apr 2024 14:21:35 -0700 Subject: [PATCH 01/14] Resolve dependency and license. --- pom.xml | 52 +++++++++++++++++++++++++++------ scripts/process_licenses.py | 57 ++++++++++++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index de5f2244f..1f6a4a0db 100644 --- a/pom.xml +++ b/pom.xml @@ -44,8 +44,8 @@ 3.14.0 1.2 1.10.0 - 2.14.0 - 32.0.1-jre + 2.15.3 + 32.1.1-jre 3.3.6 true 0.8.5 @@ -61,7 +61,7 @@ 3.1 1.13.1 UTF-8 - 3.19.6 + 3.23.2 net.snowflake.ingest.internal 1.7.36 1.1.10.4 @@ -130,8 +130,34 @@ net.snowflake - snowflake-jdbc + snowflake-jdbc-thin ${snowjdbc.version} + + + com.google.code.gson + gson + + + com.google.errorprone + error_prone_annotations + + + com.google.http-client + google-http-client + + + com.google.http-client + google-http-client-gson + + + io.grpc + grpc-context + + + org.apache.httpcomponents + httpclient + + org.apache.commons @@ -161,6 +187,10 @@ com.github.pjfanning jersey-json + + com.google.re2j + re2j + com.jcraft jsch @@ -439,7 +469,7 @@ net.snowflake - snowflake-jdbc + snowflake-jdbc-thin org.apache.commons @@ -807,6 +837,7 @@ EDL 1.0 The Go license Bouncy Castle Licence + CDDL + GPLv2 with classpath exception test,provided,system true @@ -817,10 +848,15 @@ |Apache-2.0 |Apache License, Version 2.0 |Apache 2.0 - |Apache License V2.0 + |Apache License V2.0 + |Apache 2 BSD 2-Clause License |The BSD License - The MIT License|MIT License + 3-Clause BSD License + |BSD-3-Clause + |BSD New license + The MIT License|MIT License|MIT license + The Go license|Go License @@ -971,7 +1007,7 @@ - net.snowflake:snowflake-jdbc + net.snowflake:snowflake-jdbc-thin org.slf4j:slf4j-api com.github.luben:zstd-jni diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py index bb43fbbf0..d969f6da1 100644 --- a/scripts/process_licenses.py +++ b/scripts/process_licenses.py @@ -33,7 +33,7 @@ BOUNCY_CASTLE_LICENSE = "Bouncy Castle Licence " # The SDK does not need to include licenses of dependencies, which aren't shaded -IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc", "org.slf4j:slf4j-api"} +IGNORED_DEPENDENCIES = {"net.snowflake:snowflake-jdbc-thin", "org.slf4j:slf4j-api"} # List of dependencies, which don't ship with a license file. # Only add a new record here after verifying that the dependency JAR does not contain a license! @@ -61,6 +61,61 @@ "org.bouncycastle:bcpkix-jdk18on": BOUNCY_CASTLE_LICENSE, "org.bouncycastle:bcutil-jdk18on": BOUNCY_CASTLE_LICENSE, "org.bouncycastle:bcprov-jdk18on": BOUNCY_CASTLE_LICENSE, + "com.amazonaws:aws-java-sdk-core": APACHE_LICENSE, + "software.amazon.ion:ion-java": APACHE_LICENSE, + "com.amazonaws:aws-java-sdk-kms": APACHE_LICENSE, + "com.amazonaws:jmespath-java": APACHE_LICENSE, + "com.amazonaws:aws-java-sdk-s3": APACHE_LICENSE, + "com.google.api:gax": BSD_3_CLAUSE_LICENSE, + "com.google.api:api-common": BSD_3_CLAUSE_LICENSE, + "com.google.auth:google-auth-library-credentials": BSD_3_CLAUSE_LICENSE, + "com.google.api.grpc:proto-google-common-protos": APACHE_LICENSE, + "io.opencensus:opencensus-api": APACHE_LICENSE, + "com.google.auth:google-auth-library-oauth2-http": BSD_3_CLAUSE_LICENSE, + "com.google.cloud:google-cloud-core": APACHE_LICENSE, + "com.google.auto.value:auto-value-annotations": APACHE_LICENSE, + "com.google.protobuf:protobuf-java-util": BSD_3_CLAUSE_LICENSE, + "com.google.api.grpc:proto-google-iam-v1": APACHE_LICENSE, + "com.google.cloud:google-cloud-storage": APACHE_LICENSE, + "io.opencensus:opencensus-contrib-http-util": APACHE_LICENSE, + "com.google.http-client:google-http-client-jackson2": APACHE_LICENSE, + "com.google.api-client:google-api-client": APACHE_LICENSE, + "com.google.oauth-client:google-oauth-client": APACHE_LICENSE, + "com.google.http-client:google-http-client-apache-v2": APACHE_LICENSE, + "com.google.apis:google-api-services-storage": APACHE_LICENSE, + "com.google.cloud:google-cloud-core-http": APACHE_LICENSE, + "com.google.http-client:google-http-client-appengine": APACHE_LICENSE, + "com.google.api:gax-httpjson": BSD_3_CLAUSE_LICENSE, + "com.google.cloud:google-cloud-core-grpc": APACHE_LICENSE, + "com.google.api:gax-grpc": BSD_3_CLAUSE_LICENSE, + "io.grpc:grpc-alts": APACHE_LICENSE, + "io.grpc:grpc-grpclb": APACHE_LICENSE, + "org.conscrypt:conscrypt-openjdk-uber": APACHE_LICENSE, + "io.grpc:grpc-auth": APACHE_LICENSE, + "io.grpc:grpc-protobuf": APACHE_LICENSE, + "io.grpc:grpc-protobuf-lite": APACHE_LICENSE, + "com.google.api.grpc:proto-google-cloud-storage-v2": APACHE_LICENSE, + "com.google.api.grpc:grpc-google-cloud-storage-v2": APACHE_LICENSE, + "com.google.api.grpc:gapic-google-cloud-storage-v2": APACHE_LICENSE, + "io.grpc:grpc-api": APACHE_LICENSE, + "io.perfmark:perfmark-api": APACHE_LICENSE, + "io.grpc:grpc-core": APACHE_LICENSE, + "com.google.android:annotations": APACHE_LICENSE, + "org.codehaus.mojo:animal-sniffer-annotations": MIT_LICENSE, + "io.grpc:grpc-stub": APACHE_LICENSE, + "io.grpc:grpc-googleapis": APACHE_LICENSE, + "io.grpc:grpc-xds": APACHE_LICENSE, + "io.opencensus:opencensus-proto": APACHE_LICENSE, + "io.grpc:grpc-services": APACHE_LICENSE, + "io.grpc:grpc-rls": APACHE_LICENSE, + "com.microsoft.azure:azure-storage": APACHE_LICENSE, + "com.microsoft.azure:azure-keyvault-core": APACHE_LICENSE, + "com.yammer.metrics:metrics-core": APACHE_LICENSE, + "com.yammer.metrics:metrics-servlet": APACHE_LICENSE, + "net.minidev:json-smart": APACHE_LICENSE, + "net.minidev:accessors-smart": APACHE_LICENSE, + "org.ow2.asm:asm": BSD_3_CLAUSE_LICENSE, + "com.google.flatbuffers:flatbuffers-java": APACHE_LICENSE, } From 30a5daf240dc44745718be54710fbda1ace12eb6 Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Tue, 9 Apr 2024 16:03:56 -0700 Subject: [PATCH 02/14] Resolve undeclared dependencies and avoid import shading dependencies. --- pom.xml | 68 +++++++++++++++---- scripts/process_licenses.py | 2 + .../snowflake/ingest/SimpleIngestManager.java | 8 +-- .../ingest/connection/OAuthClient.java | 24 +++---- .../ingest/connection/OAuthManager.java | 2 +- .../ingest/connection/RequestBuilder.java | 16 ++--- .../connection/ServiceResponseHandler.java | 18 ++--- .../ingest/connection/TelemetryService.java | 6 +- .../internal/DataValidationUtil.java | 2 +- .../streaming/internal/FlushService.java | 2 +- .../streaming/internal/ParquetRowBuffer.java | 2 +- ...nowflakeStreamingIngestClientInternal.java | 4 +- .../internal/StreamingIngestStage.java | 10 +-- .../internal/StreamingIngestUtils.java | 6 +- .../net/snowflake/ingest/utils/HttpUtil.java | 46 ++++++------- .../net/snowflake/ingest/SimpleIngestIT.java | 6 +- .../java/net/snowflake/ingest/TestUtils.java | 6 +- .../ingest/connection/MockOAuthClient.java | 2 +- .../connection/TelemetryServiceTest.java | 2 +- .../SnowflakeStreamingIngestChannelTest.java | 14 ++-- .../SnowflakeStreamingIngestClientTest.java | 16 ++--- .../internal/StreamingIngestStageTest.java | 16 ++--- .../internal/StreamingIngestUtilsIT.java | 2 +- .../internal/StreamingIngestUtilsTest.java | 10 +-- .../snowflake/ingest/utils/HttpUtilTest.java | 10 +-- 25 files changed, 173 insertions(+), 127 deletions(-) diff --git a/pom.xml b/pom.xml index 1f6a4a0db..df6dfc9d0 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,9 @@ + 4.5.14 + 4.4.16 + 1.12.501 1.77 1.9.13 1.15 @@ -45,6 +48,8 @@ 1.2 1.10.0 2.15.3 + 1.43.3 + 2.10.1 32.1.1-jre 3.3.6 true @@ -78,11 +83,26 @@ pom import + + com.amazonaws + aws-java-sdk-core + ${aws.version} + + + com.google.code.gson + gson + ${gson.version} + com.google.guava guava ${guava.version} + + com.google.http-client + google-http-client + ${google.httpclient.version} + com.google.protobuf protobuf-java @@ -133,18 +153,10 @@ snowflake-jdbc-thin ${snowjdbc.version} - - com.google.code.gson - gson - com.google.errorprone error_prone_annotations - - com.google.http-client - google-http-client - com.google.http-client google-http-client-gson @@ -153,10 +165,6 @@ io.grpc grpc-context - - org.apache.httpcomponents - httpclient - @@ -285,6 +293,17 @@ + + org.apache.httpcomponents + httpclient + ${apache.httpclient.version} + + + org.apache.httpcomponents + httpcore + ${apache.httpcomponments.version} + + org.apache.parquet parquet-column @@ -423,10 +442,18 @@ jsr305 3.0.2 + + com.google.code.gson + gson + com.google.guava guava + + com.google.http-client + google-http-client + @@ -437,6 +464,10 @@ commons-codec commons-codec + + commons-io + commons-io + @@ -479,6 +510,14 @@ org.apache.hadoop hadoop-common + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpcore + org.apache.parquet @@ -521,6 +560,11 @@ snappy-java runtime + + com.amazonaws + aws-java-sdk-core + test + diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py index d969f6da1..fd4a9ce79 100644 --- a/scripts/process_licenses.py +++ b/scripts/process_licenses.py @@ -116,6 +116,8 @@ "net.minidev:accessors-smart": APACHE_LICENSE, "org.ow2.asm:asm": BSD_3_CLAUSE_LICENSE, "com.google.flatbuffers:flatbuffers-java": APACHE_LICENSE, + "com.google.http-client:google-http-client": APACHE_LICENSE, + "io.grpc:grpc-context": APACHE_LICENSE, } diff --git a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java index 17caa2b9b..441cd4b81 100644 --- a/src/main/java/net/snowflake/ingest/SimpleIngestManager.java +++ b/src/main/java/net/snowflake/ingest/SimpleIngestManager.java @@ -18,10 +18,6 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.HistoryRangeResponse; import net.snowflake.ingest.connection.HistoryResponse; import net.snowflake.ingest.connection.IngestResponse; @@ -32,6 +28,10 @@ import net.snowflake.ingest.utils.HttpUtil; import net.snowflake.ingest.utils.StagedFileWrapper; import net.snowflake.ingest.utils.Utils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java index 61c736a42..04a6572b5 100644 --- a/src/main/java/net/snowflake/ingest/connection/OAuthClient.java +++ b/src/main/java/net/snowflake/ingest/connection/OAuthClient.java @@ -4,6 +4,9 @@ package net.snowflake.ingest.connection; +import com.google.api.client.http.HttpStatusCodes; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; @@ -11,21 +14,18 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.apache.http.entity.ContentType; -import net.snowflake.client.jdbc.internal.apache.http.entity.StringEntity; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.util.EntityUtils; -import net.snowflake.client.jdbc.internal.google.api.client.http.HttpStatusCodes; -import net.snowflake.client.jdbc.internal.google.gson.JsonObject; -import net.snowflake.client.jdbc.internal.google.gson.JsonParser; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.HttpUtil; import net.snowflake.ingest.utils.SFException; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java index f9e614a27..01cbcd846 100644 --- a/src/main/java/net/snowflake/ingest/connection/OAuthManager.java +++ b/src/main/java/net/snowflake/ingest/connection/OAuthManager.java @@ -6,10 +6,10 @@ import java.io.IOException; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; +import org.apache.http.client.utils.URIBuilder; /** This class manages creating and automatically refresh the OAuth token */ public final class OAuthManager extends SecurityManager { diff --git a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java index a1c5514bd..ad197ebf3 100644 --- a/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java +++ b/src/main/java/net/snowflake/ingest/connection/RequestBuilder.java @@ -15,18 +15,18 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.apache.http.entity.ContentType; -import net.snowflake.client.jdbc.internal.apache.http.entity.StringEntity; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.StagedFileWrapper; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java index cef6e631c..8b910ed9d 100644 --- a/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java +++ b/src/main/java/net/snowflake/ingest/connection/ServiceResponseHandler.java @@ -8,16 +8,16 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.UUID; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.HttpStatus; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpGet; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.util.EntityUtils; import net.snowflake.ingest.utils.BackOffException; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java index 15e2a0c5a..d2ceb846c 100644 --- a/src/main/java/net/snowflake/ingest/connection/TelemetryService.java +++ b/src/main/java/net/snowflake/ingest/connection/TelemetryService.java @@ -10,16 +10,16 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Snapshot; import com.codahale.metrics.Timer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.RateLimiter; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.client.jdbc.telemetry.TelemetryClient; import net.snowflake.client.jdbc.telemetry.TelemetryUtil; import net.snowflake.ingest.utils.Logging; +import org.apache.http.impl.client.CloseableHttpClient; /** * Telemetry service to collect logs in the SDK and send them to Snowflake through the JDBC client diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java index 814423c28..1f1780d7e 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/DataValidationUtil.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; +import com.google.common.collect.Sets; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; @@ -30,7 +31,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import net.snowflake.client.jdbc.internal.google.common.collect.Sets; import net.snowflake.client.jdbc.internal.snowflake.common.core.SnowflakeDateTimeFormat; import net.snowflake.client.jdbc.internal.snowflake.common.util.Power10; import net.snowflake.ingest.streaming.internal.serialization.ByteArraySerializer; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java index 5848cc10d..3b7ebb12a 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/FlushService.java @@ -13,6 +13,7 @@ import com.codahale.metrics.Timer; import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; import java.lang.management.ManagementFactory; import java.security.InvalidAlgorithmParameterException; @@ -39,7 +40,6 @@ import javax.crypto.IllegalBlockSizeException; import javax.crypto.NoSuchPaddingException; import net.snowflake.client.jdbc.SnowflakeSQLException; -import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java index 17aaa9136..63661722a 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/ParquetRowBuffer.java @@ -4,6 +4,7 @@ package net.snowflake.ingest.streaming.internal; +import com.google.common.collect.Sets; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigDecimal; @@ -18,7 +19,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import net.snowflake.client.jdbc.internal.google.common.collect.Sets; import net.snowflake.ingest.connection.TelemetryService; import net.snowflake.ingest.streaming.OffsetTokenVerificationFunction; import net.snowflake.ingest.streaming.OpenChannelRequest; diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java index 2990b49d8..4c99d9149 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientInternal.java @@ -62,8 +62,6 @@ import javax.management.MalformedObjectNameException; import javax.management.ObjectName; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.OAuthCredential; import net.snowflake.ingest.connection.RequestBuilder; @@ -81,6 +79,8 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; /** * The first version of implementation for SnowflakeStreamingIngestClient. The client internally diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java index e8e56f383..92f4beef0 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestStage.java @@ -11,6 +11,9 @@ import static net.snowflake.ingest.utils.HttpUtil.generateProxyPropertiesForJDBC; import static net.snowflake.ingest.utils.Utils.getStackTrace; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import java.io.ByteArrayInputStream; import java.io.File; @@ -29,17 +32,14 @@ import net.snowflake.client.jdbc.SnowflakeFileTransferMetadataV1; import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; -import net.snowflake.client.jdbc.internal.apache.commons.io.FileUtils; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.Utils; +import org.apache.commons.io.FileUtils; +import org.apache.http.impl.client.CloseableHttpClient; /** Handles uploading files to the Snowflake Streaming Ingest Stage */ class StreamingIngestStage { diff --git a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java index 56e960064..82192d3b3 100644 --- a/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java +++ b/src/main/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtils.java @@ -8,15 +8,15 @@ import java.io.IOException; import java.util.Map; import java.util.function.Function; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpUriRequest; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.connection.ServiceResponseHandler; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.Logging; import net.snowflake.ingest.utils.SFException; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.impl.client.CloseableHttpClient; public class StreamingIngestUtils { diff --git a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java index 1ff65a095..c89a4cf02 100644 --- a/src/main/java/net/snowflake/ingest/utils/HttpUtil.java +++ b/src/main/java/net/snowflake/ingest/utils/HttpUtil.java @@ -17,30 +17,30 @@ import java.util.regex.Pattern; import javax.net.ssl.SSLContext; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.http.HttpHost; -import net.snowflake.client.jdbc.internal.apache.http.HttpRequest; -import net.snowflake.client.jdbc.internal.apache.http.HttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.NoHttpResponseException; -import net.snowflake.client.jdbc.internal.apache.http.auth.AuthScope; -import net.snowflake.client.jdbc.internal.apache.http.auth.Credentials; -import net.snowflake.client.jdbc.internal.apache.http.auth.UsernamePasswordCredentials; -import net.snowflake.client.jdbc.internal.apache.http.client.CredentialsProvider; -import net.snowflake.client.jdbc.internal.apache.http.client.HttpRequestRetryHandler; -import net.snowflake.client.jdbc.internal.apache.http.client.ServiceUnavailableRetryStrategy; -import net.snowflake.client.jdbc.internal.apache.http.client.config.RequestConfig; -import net.snowflake.client.jdbc.internal.apache.http.client.protocol.HttpClientContext; -import net.snowflake.client.jdbc.internal.apache.http.conn.routing.HttpRoute; -import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.DefaultHostnameVerifier; -import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.SSLConnectionSocketFactory; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.BasicCredentialsProvider; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.HttpClientBuilder; -import net.snowflake.client.jdbc.internal.apache.http.impl.conn.DefaultProxyRoutePlanner; -import net.snowflake.client.jdbc.internal.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import net.snowflake.client.jdbc.internal.apache.http.pool.PoolStats; -import net.snowflake.client.jdbc.internal.apache.http.protocol.HttpContext; -import net.snowflake.client.jdbc.internal.apache.http.ssl.SSLContexts; import net.snowflake.ingest.streaming.internal.StreamingIngestUtils; +import org.apache.http.HttpHost; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.NoHttpResponseException; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.ServiceUnavailableRetryStrategy; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.pool.PoolStats; +import org.apache.http.protocol.HttpContext; +import org.apache.http.ssl.SSLContexts; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java index 20a27eafb..189f0be76 100644 --- a/src/test/java/net/snowflake/ingest/SimpleIngestIT.java +++ b/src/test/java/net/snowflake/ingest/SimpleIngestIT.java @@ -19,13 +19,13 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import net.snowflake.client.jdbc.internal.apache.http.Header; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; import net.snowflake.ingest.connection.HistoryRangeResponse; import net.snowflake.ingest.connection.HistoryResponse; import net.snowflake.ingest.connection.IngestResponse; import net.snowflake.ingest.utils.StagedFileWrapper; +import org.apache.http.Header; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.HttpPost; import org.junit.After; import org.junit.Before; import org.junit.Ignore; diff --git a/src/test/java/net/snowflake/ingest/TestUtils.java b/src/test/java/net/snowflake/ingest/TestUtils.java index ba14ff610..95be3369e 100644 --- a/src/test/java/net/snowflake/ingest/TestUtils.java +++ b/src/test/java/net/snowflake/ingest/TestUtils.java @@ -15,6 +15,8 @@ import static net.snowflake.ingest.utils.Constants.WAREHOUSE; import static net.snowflake.ingest.utils.ParameterProvider.BLOB_FORMAT_VERSION; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.math.BigDecimal; import java.net.URI; @@ -38,14 +40,12 @@ import java.util.Properties; import java.util.Random; import java.util.function.Supplier; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.node.ObjectNode; import net.snowflake.ingest.streaming.InsertValidationResponse; import net.snowflake.ingest.streaming.SnowflakeStreamingIngestChannel; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.Utils; import org.apache.commons.codec.binary.Base64; +import org.apache.http.client.utils.URIBuilder; import org.junit.Assert; public class TestUtils { diff --git a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java index ee98bd04d..4188735a9 100644 --- a/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java +++ b/src/test/java/net/snowflake/ingest/connection/MockOAuthClient.java @@ -2,9 +2,9 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; -import net.snowflake.client.jdbc.internal.apache.http.client.utils.URIBuilder; import net.snowflake.ingest.utils.ErrorCode; import net.snowflake.ingest.utils.SFException; +import org.apache.http.client.utils.URIBuilder; /** Mock implementation of {@link OAuthClient}, only use for test */ public class MockOAuthClient extends OAuthClient { diff --git a/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java b/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java index 91a8449a7..1d76536f9 100644 --- a/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java +++ b/src/test/java/net/snowflake/ingest/connection/TelemetryServiceTest.java @@ -4,7 +4,7 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Test; import org.mockito.Mockito; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java index 87e3f8f11..fae43c0fd 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestChannelTest.java @@ -24,13 +24,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.InsertValidationResponse; @@ -42,6 +35,13 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java index 1693e1520..bc505f7b8 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/SnowflakeStreamingIngestClientTest.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; import java.io.IOException; import java.io.StringWriter; import java.nio.charset.Charset; @@ -35,14 +36,6 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import net.snowflake.client.core.SFSessionProperty; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.HttpHeaders; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.HttpPost; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.google.common.collect.Sets; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.streaming.DropChannelRequest; @@ -56,6 +49,13 @@ import net.snowflake.ingest.utils.SFException; import net.snowflake.ingest.utils.SnowflakeURL; import net.snowflake.ingest.utils.Utils; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.impl.client.CloseableHttpClient; import org.bouncycastle.asn1.nist.NISTObjectIdentifiers; import org.bouncycastle.openssl.jcajce.JcaPEMWriter; import org.bouncycastle.operator.OperatorCreationException; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java index 1ba9f98df..eaabbc5e2 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestStageTest.java @@ -11,6 +11,10 @@ import static net.snowflake.ingest.utils.HttpUtil.shouldBypassProxy; import static org.mockito.Mockito.times; +import com.amazonaws.util.IOUtils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -32,19 +36,15 @@ import net.snowflake.client.jdbc.SnowflakeFileTransferMetadataV1; import net.snowflake.client.jdbc.SnowflakeSQLException; import net.snowflake.client.jdbc.cloud.storage.StageInfo; -import net.snowflake.client.jdbc.internal.amazonaws.util.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.entity.BasicHttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.JsonNode; -import net.snowflake.client.jdbc.internal.fasterxml.jackson.databind.ObjectMapper; -import net.snowflake.client.jdbc.internal.google.common.util.concurrent.ThreadFactoryBuilder; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.Constants; import net.snowflake.ingest.utils.ParameterProvider; import net.snowflake.ingest.utils.SFException; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.entity.BasicHttpEntity; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java index 4e054c209..f3287985c 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsIT.java @@ -8,13 +8,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.util.HashMap; import java.util.Map; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.IngestResponseException; import net.snowflake.ingest.connection.JWTManager; import net.snowflake.ingest.connection.RequestBuilder; import net.snowflake.ingest.utils.HttpUtil; import net.snowflake.ingest.utils.SnowflakeURL; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java index 6aadfbcae..854ac66ef 100644 --- a/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java +++ b/src/test/java/net/snowflake/ingest/streaming/internal/StreamingIngestUtilsTest.java @@ -10,13 +10,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.InputStream; import java.util.ArrayList; -import net.snowflake.client.jdbc.internal.apache.commons.io.IOUtils; -import net.snowflake.client.jdbc.internal.apache.http.HttpEntity; -import net.snowflake.client.jdbc.internal.apache.http.StatusLine; -import net.snowflake.client.jdbc.internal.apache.http.client.methods.CloseableHttpResponse; -import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient; import net.snowflake.ingest.TestUtils; import net.snowflake.ingest.connection.RequestBuilder; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.impl.client.CloseableHttpClient; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; diff --git a/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java b/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java index a905848e6..99707cda5 100644 --- a/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java +++ b/src/test/java/net/snowflake/ingest/utils/HttpUtilTest.java @@ -8,11 +8,11 @@ import java.net.SocketException; import java.net.UnknownHostException; import javax.net.ssl.SSLException; -import net.snowflake.client.jdbc.internal.apache.http.HttpRequest; -import net.snowflake.client.jdbc.internal.apache.http.NoHttpResponseException; -import net.snowflake.client.jdbc.internal.apache.http.RequestLine; -import net.snowflake.client.jdbc.internal.apache.http.client.HttpRequestRetryHandler; -import net.snowflake.client.jdbc.internal.apache.http.client.protocol.HttpClientContext; +import org.apache.http.HttpRequest; +import org.apache.http.NoHttpResponseException; +import org.apache.http.RequestLine; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.protocol.HttpClientContext; import org.junit.Test; import org.mockito.Mockito; From 17e6284d7a07706ed10a5f2ee141b1738e2f2f20 Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Tue, 9 Apr 2024 16:50:59 -0700 Subject: [PATCH 03/14] Add shading dependencies. --- pom.xml | 60 ++++++++++++++++++++++++++++++++++++++++ scripts/check_content.sh | 1 + 2 files changed, 61 insertions(+) diff --git a/pom.xml b/pom.xml index df6dfc9d0..e41eb7dfd 100644 --- a/pom.xml +++ b/pom.xml @@ -1057,6 +1057,66 @@ + + io.perfmark + ${shadeBase}.io.perfmark + + + io.grpc + ${shadeBase}.io.grpc + + + io.opencensus + ${shadeBase}.io.opencensus + + + com.amazonaws + ${shadeBase}.com.amazonaws + + + org.conscrypt + ${shadeBase}.org.conscrypt + + + com.microsoft + ${shadeBase}.com.microsoft + + + com.yammer + ${shadeBase}.com.yammer + + + org.jsoup + ${shadeBase}.org.jsoup + + + org.joda + ${shadeBase}.org.joda + + + org.threeten + ${shadeBase}.org.threeten + + + google + ${shadeBase}.google + + + grpc + ${shadeBase}.grpc + + + android + ${shadeBase}.android + + + opencensus + ${shadeBase}.opencensus + + + javax.servlet + ${shadeBase}.javax.servlet + com.nimbusds ${shadeBase}.com.nimbusds diff --git a/scripts/check_content.sh b/scripts/check_content.sh index e4d3e2076..4a2a1fa7f 100755 --- a/scripts/check_content.sh +++ b/scripts/check_content.sh @@ -28,6 +28,7 @@ if jar tvf $DIR/../target/snowflake-ingest-sdk.jar | awk '{print $8}' | \ grep -v PropertyList-1.0.dtd | \ grep -v properties.dtd | \ grep -v parquet.thrift | \ + grep -v dependencies.properties | \ # Native zstd libraries are allowed grep -v -E '^darwin' | \ From dd9569542c79fd0394d09eb257c46950aceb1262 Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Wed, 10 Apr 2024 14:10:50 -0700 Subject: [PATCH 04/14] Modify poms for build & test. --- e2e-jar-test/fips/pom.xml | 2 +- public_pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-jar-test/fips/pom.xml b/e2e-jar-test/fips/pom.xml index 039af9b7a..87934f480 100644 --- a/e2e-jar-test/fips/pom.xml +++ b/e2e-jar-test/fips/pom.xml @@ -29,7 +29,7 @@ net.snowflake - snowflake-jdbc + snowflake-jdbc-thin org.bouncycastle diff --git a/public_pom.xml b/public_pom.xml index fa8c7dd4a..05585f3a6 100644 --- a/public_pom.xml +++ b/public_pom.xml @@ -38,7 +38,7 @@ net.snowflake - snowflake-jdbc + snowflake-jdbc-thin 3.14.5 compile From 1abb1822cb313bb38d7c0601d5094247d366b9dd Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Fri, 12 Apr 2024 13:32:16 -0700 Subject: [PATCH 05/14] Modify poms for build & test. --- pom.xml | 15 +++++---------- public_pom.xml | 6 ------ 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/pom.xml b/pom.xml index e41eb7dfd..5c2fa276a 100644 --- a/pom.xml +++ b/pom.xml @@ -544,6 +544,11 @@ org.slf4j slf4j-api + + com.amazonaws + aws-java-sdk-core + runtime + com.github.luben zstd-jni @@ -560,11 +565,6 @@ snappy-java runtime - - com.amazonaws - aws-java-sdk-core - test - @@ -1051,7 +1051,6 @@ - net.snowflake:snowflake-jdbc-thin org.slf4j:slf4j-api com.github.luben:zstd-jni @@ -1069,10 +1068,6 @@ io.opencensus ${shadeBase}.io.opencensus - - com.amazonaws - ${shadeBase}.com.amazonaws - org.conscrypt ${shadeBase}.org.conscrypt diff --git a/public_pom.xml b/public_pom.xml index 05585f3a6..d80d6be89 100644 --- a/public_pom.xml +++ b/public_pom.xml @@ -36,12 +36,6 @@ - - net.snowflake - snowflake-jdbc-thin - 3.14.5 - compile - org.slf4j slf4j-api From 5c710fd7e62481916904350644fa4298ef32f134 Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Mon, 15 Apr 2024 10:57:34 -0700 Subject: [PATCH 06/14] Modify poms for fips. --- e2e-jar-test/fips/pom.xml | 16 ++++++++++------ pom.xml | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/e2e-jar-test/fips/pom.xml b/e2e-jar-test/fips/pom.xml index 87934f480..931d12852 100644 --- a/e2e-jar-test/fips/pom.xml +++ b/e2e-jar-test/fips/pom.xml @@ -15,6 +15,8 @@ UTF-8 1.8 1.8 + 1.0.2.4 + 1.0.5 @@ -27,10 +29,6 @@ net.snowflake snowflake-ingest-sdk - - net.snowflake - snowflake-jdbc-thin - org.bouncycastle bcpkix-jdk18on @@ -42,8 +40,14 @@ - net.snowflake - snowflake-jdbc-fips + org.bouncycastle + bc-fips + ${bouncycastle.bcfips.version} + + + org.bouncycastle + bcpkix-fips + ${bouncycastle.bcpkixfips.version} junit diff --git a/pom.xml b/pom.xml index 5c2fa276a..9e7698740 100644 --- a/pom.xml +++ b/pom.xml @@ -1210,6 +1210,22 @@ io.airlift.compress ${shadeBase}.io.airlift.compress + + net.snowflake + ${shadeBase}.net.snowflake + + + com.snowflake + ${shadeBase}.com.snowflake + + + com.amazonaws + ${shadeBase}.com.amazonaws + + + software.amazon + ${shadeBase}.software.amazon + From 090e3620e71b7ca0b52b7807bbedcee2d9ed620a Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Mon, 15 Apr 2024 13:23:54 -0700 Subject: [PATCH 07/14] Fix jar test. --- .../src/main/java/net/snowflake/IngestTestUtils.java | 6 +++++- pom.xml | 12 ++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java b/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java index e7db8d1c2..184a43b9f 100644 --- a/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java +++ b/e2e-jar-test/core/src/main/java/net/snowflake/IngestTestUtils.java @@ -98,7 +98,11 @@ private Properties loadProperties() throws IOException { private Connection getConnection() throws IOException, ClassNotFoundException, SQLException, NoSuchAlgorithmException, InvalidKeySpecException { - Class.forName("net.snowflake.client.jdbc.SnowflakeDriver"); + try { + Class.forName("net.snowflake.client.jdbc.SnowflakeDriver"); + } catch(ClassNotFoundException e) { + Class.forName("net.snowflake.ingest.internal.net.snowflake.client.jdbc.SnowflakeDriver"); + } Properties loadedProps = loadProperties(); diff --git a/pom.xml b/pom.xml index b45463477..69e93a249 100644 --- a/pom.xml +++ b/pom.xml @@ -1211,20 +1211,20 @@ ${shadeBase}.io.airlift.compress - net.snowflake - ${shadeBase}.net.snowflake + net.snowflake.client + ${shadeBase}.net.snowflake.client - com.snowflake - ${shadeBase}.com.snowflake + com.snowflake.client + ${shadeBase}.com.snowflake.client com.amazonaws ${shadeBase}.com.amazonaws - software.amazon - ${shadeBase}.software.amazon + software.amazon.ion + ${shadeBase}.software.amazon.ion From 737825d394d97cdc63a8a8d6a62535f875c426e7 Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Thu, 18 Apr 2024 16:41:09 -0700 Subject: [PATCH 08/14] Fix unshaded test. --- pom.xml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 69e93a249..7a3c90de9 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ 4.5.14 - 4.4.16 + 4.4.16 1.12.501 1.78 1.9.13 @@ -301,7 +301,7 @@ org.apache.httpcomponents httpcore - ${apache.httpcomponments.version} + ${apache.httpcomponents.version} @@ -665,7 +665,7 @@ org.apache.maven.plugins maven-failsafe-plugin - 3.0.0-M5 + 3.2.5 org.apache.maven.plugins @@ -1385,6 +1385,15 @@ org.apache.maven.plugins maven-failsafe-plugin + + + + org.apache.httpcomponents + httpclient + ${apache.httpclient.version} + + + From 8b00fb7e6d4b39ed27f2f4ffa5fff041f7368c0a Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Fri, 19 Apr 2024 15:02:05 -0700 Subject: [PATCH 09/14] Fix unshaded test. --- pom.xml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 7a3c90de9..cd1619e0b 100644 --- a/pom.xml +++ b/pom.xml @@ -1386,13 +1386,7 @@ org.apache.maven.plugins maven-failsafe-plugin - - - org.apache.httpcomponents - httpclient - ${apache.httpclient.version} - - + ${project.build.OutputDirectory} From 92a6eab8edef850ff1c1054829e8f9fa7ede7fad Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Mon, 22 Apr 2024 12:58:00 -0700 Subject: [PATCH 10/14] Fix gcp test. --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index cd1619e0b..b006b2e38 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,11 @@ google-http-client ${google.httpclient.version} + + com.google.http-client + google-http-client-gson + ${google.httpclient.version} + com.google.protobuf protobuf-java From a96463cfe6e8e317eee91538c5c19fb3f32def31 Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Mon, 22 Apr 2024 13:24:57 -0700 Subject: [PATCH 11/14] Fix gcp test. --- pom.xml | 4 ---- scripts/process_licenses.py | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index b006b2e38..3d6e8a761 100644 --- a/pom.xml +++ b/pom.xml @@ -162,10 +162,6 @@ com.google.errorprone error_prone_annotations - - com.google.http-client - google-http-client-gson - io.grpc grpc-context diff --git a/scripts/process_licenses.py b/scripts/process_licenses.py index fd4a9ce79..d3db413ca 100644 --- a/scripts/process_licenses.py +++ b/scripts/process_licenses.py @@ -117,6 +117,7 @@ "org.ow2.asm:asm": BSD_3_CLAUSE_LICENSE, "com.google.flatbuffers:flatbuffers-java": APACHE_LICENSE, "com.google.http-client:google-http-client": APACHE_LICENSE, + "com.google.http-client:google-http-client-gson": APACHE_LICENSE, "io.grpc:grpc-context": APACHE_LICENSE, } From f92d6665e63ddec60d6b76b02086ad43e67a0ddc Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Tue, 23 Apr 2024 11:32:05 -0700 Subject: [PATCH 12/14] Fix gcp test. --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 3d6e8a761..1e5959f7d 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ 1.2 1.10.0 2.15.3 + 2.3.0 1.43.3 2.10.1 32.1.1-jre @@ -88,6 +89,11 @@ aws-java-sdk-core ${aws.version} + + com.google.api-client + google-api-client + ${google.api.client.version} + com.google.code.gson gson @@ -1246,6 +1252,7 @@ META-INF/*.DSA META-INF/*.RSA google/protobuf/**/*.proto + google-api-client.properties From 861be1eb2c362d96ec99c0d59aed738bf062c094 Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Tue, 23 Apr 2024 14:35:12 -0700 Subject: [PATCH 13/14] Fix gcp test.................... --- pom.xml | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/pom.xml b/pom.xml index 1e5959f7d..7927643bd 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,6 @@ 1.2 1.10.0 2.15.3 - 2.3.0 1.43.3 2.10.1 32.1.1-jre @@ -89,11 +88,6 @@ aws-java-sdk-core ${aws.version} - - com.google.api-client - google-api-client - ${google.api.client.version} - com.google.code.gson gson @@ -1099,26 +1093,14 @@ org.threeten ${shadeBase}.org.threeten - - google - ${shadeBase}.google - - - grpc - ${shadeBase}.grpc - - - android - ${shadeBase}.android - - - opencensus - ${shadeBase}.opencensus - javax.servlet ${shadeBase}.javax.servlet + + android.annotation + ${shadeBase}.android.annotation + com.nimbusds ${shadeBase}.com.nimbusds @@ -1251,8 +1233,9 @@ META-INF/*.SF META-INF/*.DSA META-INF/*.RSA - google/protobuf/**/*.proto - google-api-client.properties + google/**/**/*.proto + grpc/**/**/*.proto + opencensus/proto/**/v1/*.proto From 768db23d5b29e5f42f35ecfd4ac19557bd3985ee Mon Sep 17 00:00:00 2001 From: Alec Huang Date: Wed, 24 Apr 2024 11:08:22 -0700 Subject: [PATCH 14/14] Fix snyk check --- pom.xml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 7927643bd..821c0f05d 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ 4.5.14 4.4.16 - 1.12.501 + 1.12.706 1.78 1.9.13 1.15 @@ -71,6 +71,7 @@ 1.7.36 1.1.10.4 3.14.5 + 1.6.9 0.13.0 @@ -166,6 +167,18 @@ io.grpc grpc-context + + javax.annotation + javax.annotation-api + + + javax.servlet + javax.servlet-api + + + joda-time + joda-time + @@ -370,6 +383,11 @@ slf4j-api ${slf4j.version} + + org.threeten + threetenbp + ${threetenbp.version} + org.xerial.snappy snappy-java @@ -882,7 +900,6 @@ EDL 1.0 The Go license Bouncy Castle Licence - CDDL + GPLv2 with classpath exception test,provided,system true