Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Snow 984877 test, no review pls #661

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
Draft
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -35,6 +35,8 @@ public static class Builder {
// flag to specify if we need to add account name in the request header
private boolean addAccountNameInRequest;

private int method;

private Builder(String name) {
this.name = name;
}
@@ -59,6 +61,11 @@ public Builder setParameterOverrides(Map<String, Object> parameterOverrides) {
return this;
}

public Builder setMethod(int method) {
this.method = method;
return this;
}

public SnowflakeStreamingIngestClient build() {
Utils.assertStringNotNullOrEmpty("client name", this.name);
Utils.assertNotNull("connection properties", this.prop);
@@ -71,7 +78,7 @@ public SnowflakeStreamingIngestClient build() {

if (addAccountNameInRequest) {
return new SnowflakeStreamingIngestClientInternal<>(
this.name, accountURL, prop, this.parameterOverrides, addAccountNameInRequest);
this.name, accountURL, prop, this.parameterOverrides, addAccountNameInRequest, method);
}
return new SnowflakeStreamingIngestClientInternal<>(
this.name, accountURL, prop, this.parameterOverrides);
Original file line number Diff line number Diff line change
@@ -158,7 +158,7 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
boolean isTestMode,
RequestBuilder requestBuilder,
Map<String, Object> parameterOverrides) {
this(name, accountURL, prop, httpClient, isTestMode, requestBuilder, parameterOverrides, false);
this(name, accountURL, prop, httpClient, isTestMode, requestBuilder, parameterOverrides, false, 0);
}

/**
@@ -181,13 +181,14 @@ public class SnowflakeStreamingIngestClientInternal<T> implements SnowflakeStrea
boolean isTestMode,
RequestBuilder requestBuilder,
Map<String, Object> parameterOverrides,
boolean addAccountNameInRequest) {
boolean addAccountNameInRequest,
int method) {
this.parameterProvider = new ParameterProvider(parameterOverrides, prop);

this.name = name;
String accountName = accountURL == null ? null : accountURL.getAccount();
this.isTestMode = isTestMode;
this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName) : httpClient;
this.httpClient = httpClient == null ? HttpUtil.getHttpClient(accountName, method) : httpClient;
this.channelCache = new ChannelCache<>();
this.isClosed = false;
this.requestBuilder = requestBuilder;
@@ -274,15 +275,17 @@ public SnowflakeStreamingIngestClientInternal(
SnowflakeURL accountURL,
Properties prop,
Map<String, Object> parameterOverrides,
boolean addAccountNameInRequest) {
boolean addAccountNameInRequest,
int method) {
this(name,
accountURL,
prop,
null,
false,
null,
parameterOverrides,
addAccountNameInRequest);
addAccountNameInRequest,
method);
}

/**
58 changes: 54 additions & 4 deletions src/main/java/net/snowflake/ingest/utils/HttpUtil.java
Original file line number Diff line number Diff line change
@@ -6,7 +6,15 @@

import static net.snowflake.ingest.utils.Utils.isNullOrEmpty;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.security.KeyStore;
import java.security.SecureRandom;
import java.security.Security;
import java.security.cert.CertificateFactory;
import java.security.cert.X509Certificate;
import java.security.cert.CertificateException;
import java.time.Duration;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
@@ -15,7 +23,8 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
import javax.net.ssl.SSLContext;
import javax.net.ssl.*;

import net.snowflake.client.core.SFSessionProperty;
import net.snowflake.client.jdbc.internal.apache.http.HttpHost;
import net.snowflake.client.jdbc.internal.apache.http.HttpRequest;
@@ -32,9 +41,11 @@
import net.snowflake.client.jdbc.internal.apache.http.conn.routing.HttpRoute;
import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.DefaultHostnameVerifier;
import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.SSLConnectionSocketFactory;
import net.snowflake.client.jdbc.internal.apache.http.conn.ssl.TrustAllStrategy;
import net.snowflake.client.jdbc.internal.apache.http.impl.client.BasicCredentialsProvider;
import net.snowflake.client.jdbc.internal.apache.http.impl.client.CloseableHttpClient;
import net.snowflake.client.jdbc.internal.apache.http.impl.client.HttpClientBuilder;
import net.snowflake.client.jdbc.internal.apache.http.impl.client.HttpClients;
import net.snowflake.client.jdbc.internal.apache.http.impl.conn.DefaultProxyRoutePlanner;
import net.snowflake.client.jdbc.internal.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import net.snowflake.client.jdbc.internal.apache.http.pool.PoolStats;
@@ -105,10 +116,14 @@ public class HttpUtil {
* @return Instance of CloseableHttpClient
*/
public static CloseableHttpClient getHttpClient(String accountName) {
return getHttpClient(accountName, 0);
}

public static CloseableHttpClient getHttpClient(String accountName, int method) {
if (httpClient == null) {
synchronized (HttpUtil.class) {
if (httpClient == null) {
initHttpClient(accountName);
initHttpClient(accountName, method);
}
}
}
@@ -120,11 +135,46 @@ public static CloseableHttpClient getHttpClient(String accountName) {

private static final Logger LOGGER = LoggerFactory.getLogger(HttpUtil.class);

private static void initHttpClient(String accountName) {
private static void initHttpClient(String accountName, int method) {

Security.setProperty("ocsp.enable", "true");

SSLContext sslContext = SSLContexts.createDefault();
// SSLContext sslContext = SSLContexts.createDefault();
SSLContext sslContext = null;
try {
final String CA_FILE = "/etc/pki/ca-trust/source/anchors/may_2020_ca_cert_dev.pem";
if (method == 0) {
LOGGER.info("using original way");
sslContext = SSLContexts.createDefault();
} else if (method == 1) {
LOGGER.info("using trustallstrategy");
sslContext = SSLContexts.custom().loadTrustMaterial(null, new TrustAllStrategy()).build();
} else if (method== 2) {
LOGGER.info("using trust store");

FileInputStream fis = new FileInputStream(CA_FILE);
X509Certificate ca = (X509Certificate) CertificateFactory.getInstance("X.509")
.generateCertificate(new BufferedInputStream(fis));

KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
ks.load(null, null);
ks.setCertificateEntry(Integer.toString(1), ca);

TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
tmf.init(ks);

sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, tmf.getTrustManagers(), null);
} else {
LOGGER.info("using system parameter way");
System.setProperty("javax.net.ssl.trustStore", CA_FILE);
sslContext = SSLContexts.createDefault();
}
} catch (Exception e) {
LOGGER.info(e.getMessage());
LOGGER.info(e.toString());
e.printStackTrace();
}

SSLConnectionSocketFactory f =
new SSLConnectionSocketFactory(
9 changes: 5 additions & 4 deletions src/main/java/net/snowflake/ingest/utils/Utils.java
Original file line number Diff line number Diff line change
@@ -41,14 +41,15 @@ public class Utils {

private static final Logging logger = new Logging(Utils.class);


private static final String DEFAULT_SECURITY_PROVIDER_NAME =
"org.bouncycastle.jce.provider.BouncyCastleProvider";

/** provider name */
private static final String BOUNCY_CASTLE_PROVIDER = "BC";
/** provider name for FIPS */
private static final String BOUNCY_CASTLE_FIPS_PROVIDER = "BCFIPS";

/**
static {
// Add Bouncy Castle to the security provider. This is required to
// verify the signature on OCSP response and attached certificates.
@@ -87,7 +88,7 @@ private static Provider instantiateSecurityProvider() {
ErrorCode.CRYPTO_PROVIDER_ERROR, DEFAULT_SECURITY_PROVIDER_NAME, ex.getMessage());
}
}

*/
/**
* Assert when the String is null or Empty
*
@@ -270,8 +271,8 @@ public static PrivateKey parseEncryptedPrivateKey(String key, String passphrase)
pemParser.close();
InputDecryptorProvider pkcs8Prov =
new JceOpenSSLPKCS8DecryptorProviderBuilder().build(passphrase.toCharArray());
JcaPEMKeyConverter converter =
new JcaPEMKeyConverter().setProvider(Utils.getProvider().getName());
JcaPEMKeyConverter converter = new JcaPEMKeyConverter();
//new JcaPEMKeyConverter().setProvider(Utils.getProvider().getName());
PrivateKeyInfo decryptedPrivateKeyInfo =
encryptedPrivateKeyInfo.decryptPrivateKeyInfo(pkcs8Prov);
return converter.getPrivateKey(decryptedPrivateKeyInfo);
Original file line number Diff line number Diff line change
@@ -141,6 +141,7 @@ public void setup() {
BDEC_VERSION);
}

/**
@Test
@Ignore // Until able to test in PROD
public void testConstructorParameters() throws Exception {
@@ -169,6 +170,7 @@ public void testConstructorParameters() throws Exception {
client.getParameterProvider().getInsertThrottleIntervalInMs());
Assert.assertFalse(client.isClosed());
}
*/

@Test
@Ignore
@@ -255,6 +257,7 @@ public void testClientFactoryInvalidPrivateKey() throws Exception {
}
}

/**
@Test
@Ignore // Wait for the client/configure endpoint to be available in PROD, can't mock the
// HttpUtil.executeGeneralRequest call because it's also used when setting up the
@@ -272,6 +275,7 @@ public void testClientFactorySuccess() throws Exception {
Assert.assertEquals("client", client.getName());
Assert.assertFalse(client.isClosed());
}
*/

@Test
@Ignore // SNOW-540567: NON-FIPS provider causing failures during release jekins job
@@ -300,7 +304,7 @@ private String generateAESKey(PrivateKey key, char[] passwd)
pemWriter.writeObject(
pkcs8EncryptedPrivateKeyInfoBuilder.build(
new JcePKCSPBEOutputEncryptorBuilder(NISTObjectIdentifiers.id_aes256_CBC)
.setProvider(Utils.getProvider().getName())
// .setProvider(Utils.getProvider().getName())
.build(passwd)));
pemWriter.close();
return writer.toString();