From 1e60c6a3232bc8ce59fd0cca5a8daf5fb82c3c27 Mon Sep 17 00:00:00 2001 From: Plamen Jeliazkov Date: Fri, 13 Jul 2018 15:14:45 -0700 Subject: [PATCH] [#61] Make NNA constructor that accepts configuration --- .../paypal/namenode/NNAnalyticsRestAPI.java | 169 +++++++----------- .../security/SecurityConfiguration.java | 4 + src/test/java/TestAuthorization.java | 8 +- src/test/java/TestFinderQuery.java | 9 +- src/test/java/TestINodeTransfer.java | 4 +- src/test/java/TestLdapAuth.java | 9 +- src/test/java/TestNNAnalytics.java | 9 +- src/test/java/TestNoHistorical.java | 9 +- src/test/java/TestQueryChecker.java | 9 +- src/test/java/TestTransforms.java | 19 +- src/test/java/TestWithMiniCluster.java | 9 +- 11 files changed, 122 insertions(+), 136 deletions(-) diff --git a/src/main/java/com/paypal/namenode/NNAnalyticsRestAPI.java b/src/main/java/com/paypal/namenode/NNAnalyticsRestAPI.java index 1b452d2c..d8662a2b 100644 --- a/src/main/java/com/paypal/namenode/NNAnalyticsRestAPI.java +++ b/src/main/java/com/paypal/namenode/NNAnalyticsRestAPI.java @@ -49,6 +49,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; @@ -63,6 +64,16 @@ import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields; import org.apache.hadoop.hdfs.server.namenode.NNAConstants; +import org.apache.hadoop.hdfs.server.namenode.NNAConstants.ENDPOINT; +import org.apache.hadoop.hdfs.server.namenode.NNAConstants.FILTER; +import org.apache.hadoop.hdfs.server.namenode.NNAConstants.FILTER_OP; +import org.apache.hadoop.hdfs.server.namenode.NNAConstants.FIND; +import org.apache.hadoop.hdfs.server.namenode.NNAConstants.HISTOGRAM; +import org.apache.hadoop.hdfs.server.namenode.NNAConstants.HISTOGRAM_OUTPUT; +import org.apache.hadoop.hdfs.server.namenode.NNAConstants.OPERATION; +import org.apache.hadoop.hdfs.server.namenode.NNAConstants.SET; +import org.apache.hadoop.hdfs.server.namenode.NNAConstants.SUM; +import org.apache.hadoop.hdfs.server.namenode.NNAConstants.TRANSFORM; import org.apache.hadoop.hdfs.server.namenode.NNLoader; import org.apache.hadoop.hdfs.server.namenode.TransferFsImageWrapper; import org.apache.hadoop.hdfs.server.namenode.operations.BaseOperation; @@ -150,76 +161,36 @@ public static void main(String[] args) throws InterruptedException, IllegalAccessException, NoSuchFieldException { try { NNAnalyticsRestAPI main = new NNAnalyticsRestAPI(); - main.initAuth(null, null); - main.initRestServer(); - main.initLoader(null, null); + SecurityConfiguration conf = new SecurityConfiguration(); + main.init(conf); } catch (Throwable e) { LOG.info("FATAL: {}", e); } } @VisibleForTesting - public NNLoader initLoader(GSet inodes, Boolean historical) - throws Exception { - return initLoader(inodes, historical, null); + public NNLoader getLoader() { + return nnLoader; + } + + public void init(SecurityConfiguration conf) throws Exception { + init(conf, null); } - /** - * This is the method responsible for initializing the core of NNA -- the NNLoader. It is made - * public for those wishing to set-up a customized NNA instance in tests. - * - * @param inodes Optional parameter if you wish to initialize NNLoader with your own set of - * INodes. - * @param historical Optional parameter whether to load up embedded SQL DB for historical - * tracking; null means default to configuration. tracking; null means default to - * configuration. - * @param preloadedHadoopConf Optional parameter for whether you want to load with predetermined - * configuration; null means to load configuration from classpath. - * @return An initialized NNLoader with predetermined INodes else INodes loaded from local - * FSImage. - * @throws Exception SQLException, InterruptedException, NoSuchFieldException, - * IllegalAccessException - */ @VisibleForTesting - public NNLoader initLoader( - GSet inodes, - Boolean historical, - Configuration preloadedHadoopConf) + public void init(SecurityConfiguration conf, GSet gSet) throws Exception { - SecurityConfiguration secConf = new SecurityConfiguration(); - hsqlDriver.dropConnection(); - if (historical != null) { - nnLoader.initHistoryRecorder(hsqlDriver, secConf, historical); - } else { - nnLoader.initHistoryRecorder(hsqlDriver, secConf, secConf.getHistoricalEnabled()); - } - nnLoader.load(inodes, preloadedHadoopConf); - nnLoader.initReloadThreads(internalService, secConf); - return nnLoader; + init(conf, gSet, null); } - /** - * This is the method responsible for initializing the security layer of NNA -- the - * SecurityContext. It is made public for those wishing to set-up a customized NNA instance in - * tests. - * - * @param overrideAuthentication Optional parameter dictating how you wish to override LDAP - * authentication; null means default to configuration. - * @param overrideAuthorization Optional parameter dictating how you wish to override - * authorization; null means default to configuration. - */ @VisibleForTesting - public void initAuth(Boolean overrideAuthentication, Boolean overrideAuthorization) { - SecurityConfiguration secConf = new SecurityConfiguration(); - if (overrideAuthentication != null) { - secConf.overrideLdapEnabled(overrideAuthentication); - } - if (overrideAuthorization != null) { - secConf.overrideAuthorization(overrideAuthorization); - } - - String sslKeystorePath = secConf.getSslKeystorePath(); - String sslKeystorePassword = secConf.getSslKeystorePassword(); + public void init( + SecurityConfiguration conf, + GSet gSet, + Configuration preloadedHadoopConf) + throws Exception { + String sslKeystorePath = conf.getSslKeystorePath(); + String sslKeystorePassword = conf.getSslKeystorePassword(); if (sslKeystorePath == null && sslKeystorePassword == null) { LOG.info("Running web server in HTTP mode."); } else if (sslKeystorePath != null && sslKeystorePassword != null) { @@ -230,35 +201,35 @@ public void initAuth(Boolean overrideAuthentication, Boolean overrideAuthorizati "Illegal SSL configuration. Check config/security.properties file."); } - boolean ldapEnabled = secConf.getLdapEnabled(); - Spark.port(secConf.getPort()); + boolean ldapEnabled = conf.getLdapEnabled(); + Spark.port(conf.getPort()); if (ldapEnabled) { LOG.info("Enabled web security."); // jwt: SignatureConfiguration sigConf = - new SecretSignatureConfiguration(secConf.getJwtSignatureSecret()); + new SecretSignatureConfiguration(conf.getJwtSignatureSecret()); EncryptionConfiguration encConf = new SecretEncryptionConfiguration( - secConf.getJwtEncryptionSecret(), JWEAlgorithm.DIR, EncryptionMethod.A128GCM); + conf.getJwtEncryptionSecret(), JWEAlgorithm.DIR, EncryptionMethod.A128GCM); JwtGenerator jwtGenerator = new JwtGenerator<>(sigConf, encConf); JwtAuthenticator jwtAuthenticator = new JwtAuthenticator(sigConf, encConf); // ldaptive: ConnectionConfig connectionConfig = new ConnectionConfig(); KeyStoreCredentialConfig keyStoreCredentialConfig = new KeyStoreCredentialConfig(); - keyStoreCredentialConfig.setTrustStore(secConf.getLdapTrustStorePath()); - keyStoreCredentialConfig.setTrustStorePassword(secConf.getLdapTruststorePassword()); - connectionConfig.setUseStartTLS(secConf.getLdapUseStartTLS()); + keyStoreCredentialConfig.setTrustStore(conf.getLdapTrustStorePath()); + keyStoreCredentialConfig.setTrustStorePassword(conf.getLdapTruststorePassword()); + connectionConfig.setUseStartTLS(conf.getLdapUseStartTLS()); connectionConfig.setUseSSL(true); connectionConfig.setSslConfig(new SslConfig(keyStoreCredentialConfig)); - connectionConfig.setConnectTimeout(secConf.getLdapConnectTimeout()); - connectionConfig.setResponseTimeout(secConf.getLdapResponseTimeout()); - connectionConfig.setLdapUrl(secConf.getLdapUrl()); + connectionConfig.setConnectTimeout(conf.getLdapConnectTimeout()); + connectionConfig.setResponseTimeout(conf.getLdapResponseTimeout()); + connectionConfig.setLdapUrl(conf.getLdapUrl()); DefaultConnectionFactory connectionFactory = new DefaultConnectionFactory(); connectionFactory.setConnectionConfig(connectionConfig); PoolConfig poolConfig = new PoolConfig(); - poolConfig.setMinPoolSize(secConf.getLdapConnectionPoolMinSize()); - poolConfig.setMaxPoolSize(secConf.getLdapConnectionPoolMaxSize()); + poolConfig.setMinPoolSize(conf.getLdapConnectionPoolMinSize()); + poolConfig.setMaxPoolSize(conf.getLdapConnectionPoolMaxSize()); poolConfig.setValidateOnCheckOut(true); poolConfig.setValidateOnCheckIn(true); poolConfig.setValidatePeriodically(false); @@ -281,22 +252,12 @@ public void initAuth(Boolean overrideAuthentication, Boolean overrideAuthorizati LdapAuthenticator ldapAuth = new LdapAuthenticator(); ldapAuth.setLdapAuthenticator(ldaptiveAuthenticator); - secContext.init(secConf, jwtAuthenticator, jwtGenerator, ldapAuth); + secContext.init(conf, jwtAuthenticator, jwtGenerator, ldapAuth); } else { - secContext.init(secConf, null, null, null); + secContext.init(conf, null, null, null); LOG.info("Disabled web security."); } - } - /** - * This is the main method for launching the SparkJava web server. Once this call is made the web - * server will run and REST API endpoints will be available. - * - *

It is made public for those wishing to have control of the Rest Server start-up of NNA - * instances in tests. - */ - @VisibleForTesting - public void initRestServer() { /* This is the call to load everything under ./resources/public as HTML resources. */ Spark.staticFileLocation("/public"); @@ -304,7 +265,7 @@ public void initRestServer() { get( "/endpoints", (req, res) -> { - NNAHelper.toJsonList(res.raw(), NNAConstants.ENDPOINT.values()); + NNAHelper.toJsonList(res.raw(), ENDPOINT.values()); return res; }); @@ -387,9 +348,9 @@ public void initRestServer() { sb.append("Ready to service history: ").append(isHistorical).append("\n"); sb.append("Ready to service suggestions: ").append(isProvidingSuggestions).append("\n\n"); if (isInit) { - long allSetSize = nnLoader.getINodeSet(NNAConstants.SET.all.name()).size(); - long fileSetSize = nnLoader.getINodeSet(NNAConstants.SET.files.name()).size(); - long dirSetSize = nnLoader.getINodeSet(NNAConstants.SET.dirs.name()).size(); + long allSetSize = nnLoader.getINodeSet(SET.all.name()).size(); + long fileSetSize = nnLoader.getINodeSet(SET.files.name()).size(); + long dirSetSize = nnLoader.getINodeSet(SET.dirs.name()).size(); sb.append("Current TxID: ").append(nnLoader.getCurrentTxID()).append("\n"); sb.append("INode GSet size: ").append(allSetSize).append("\n\n"); sb.append("INodeFile set size: ").append(fileSetSize).append("\n"); @@ -522,7 +483,7 @@ public void initRestServer() { (req, res) -> { res.header("Access-Control-Allow-Origin", "*"); res.header("Content-Type", "application/json; charset=UTF-8"); - NNAHelper.toJsonList(res.raw(), NNAConstants.HISTOGRAM.values()); + NNAHelper.toJsonList(res.raw(), HISTOGRAM.values()); return res; }); @@ -534,7 +495,7 @@ public void initRestServer() { (req, res) -> { res.header("Access-Control-Allow-Origin", "*"); res.header("Content-Type", "application/json; charset=UTF-8"); - NNAHelper.toJsonList(res.raw(), NNAConstants.HISTOGRAM_OUTPUT.values()); + NNAHelper.toJsonList(res.raw(), HISTOGRAM_OUTPUT.values()); return res; }); @@ -546,7 +507,7 @@ public void initRestServer() { (req, res) -> { res.header("Access-Control-Allow-Origin", "*"); res.header("Content-Type", "application/json; charset=UTF-8"); - NNAHelper.toJsonList(res.raw(), NNAConstants.FILTER.values()); + NNAHelper.toJsonList(res.raw(), FILTER.values()); return res; }); @@ -558,7 +519,7 @@ public void initRestServer() { (req, res) -> { res.header("Access-Control-Allow-Origin", "*"); res.header("Content-Type", "application/json; charset=UTF-8"); - NNAHelper.toJsonList(res.raw(), NNAConstants.FIND.values()); + NNAHelper.toJsonList(res.raw(), FIND.values()); return res; }); @@ -571,7 +532,7 @@ public void initRestServer() { (req, res) -> { res.header("Access-Control-Allow-Origin", "*"); res.header("Content-Type", "application/json; charset=UTF-8"); - NNAHelper.toJsonList(res.raw(), NNAConstants.TRANSFORM.values()); + NNAHelper.toJsonList(res.raw(), TRANSFORM.values()); return res; }); @@ -583,7 +544,7 @@ public void initRestServer() { (req, res) -> { res.header("Access-Control-Allow-Origin", "*"); res.header("Content-Type", "application/json; charset=UTF-8"); - NNAHelper.toJsonList(res.raw(), NNAConstants.SET.values()); + NNAHelper.toJsonList(res.raw(), SET.values()); return res; }); @@ -597,7 +558,7 @@ public void initRestServer() { (req, res) -> { res.header("Access-Control-Allow-Origin", "*"); res.header("Content-Type", "application/json; charset=UTF-8"); - NNAHelper.toJsonList(res.raw(), NNAConstants.FILTER_OP.values()); + NNAHelper.toJsonList(res.raw(), FILTER_OP.values()); return res; }); @@ -610,7 +571,7 @@ public void initRestServer() { (req, res) -> { res.header("Access-Control-Allow-Origin", "*"); res.header("Content-Type", "application/json; charset=UTF-8"); - NNAHelper.toJsonList(res.raw(), NNAConstants.SUM.values()); + NNAHelper.toJsonList(res.raw(), SUM.values()); return res; }); @@ -622,7 +583,7 @@ public void initRestServer() { (req, res) -> { res.header("Access-Control-Allow-Origin", "*"); res.header("Content-Type", "application/json; charset=UTF-8"); - NNAHelper.toJsonList(res.raw(), NNAConstants.OPERATION.values()); + NNAHelper.toJsonList(res.raw(), OPERATION.values()); return res; }); @@ -853,7 +814,7 @@ public void initRestServer() { QueryChecker.isValidQuery(set, filters, type, sum, filterOps, find); Collection inodes = NNAHelper.performFilters(nnLoader, set, filters, filterOps); - NNAConstants.HISTOGRAM htEnum = NNAConstants.HISTOGRAM.valueOf(histType); + HISTOGRAM htEnum = HISTOGRAM.valueOf(histType); Map> transformMap = Transforms.getAttributeTransforms( transformConditionsStr, transformFieldsStr, transformOutputsStr, nnLoader); @@ -976,8 +937,7 @@ public void initRestServer() { } // Return final histogram to Web UI as output type. - NNAConstants.HISTOGRAM_OUTPUT output = - NNAConstants.HISTOGRAM_OUTPUT.valueOf(outputType); + HISTOGRAM_OUTPUT output = HISTOGRAM_OUTPUT.valueOf(outputType); switch (output) { case chart: res.header("Content-Type", "application/json"); @@ -1044,7 +1004,7 @@ public void initRestServer() { } Collection inodes = NNAHelper.performFilters(nnLoader, set, filters, filterOps); - NNAConstants.HISTOGRAM htEnum = NNAConstants.HISTOGRAM.valueOf(histType); + HISTOGRAM htEnum = HISTOGRAM.valueOf(histType); List> histograms = new ArrayList<>(sums.length + finds.length); long startTime = System.currentTimeMillis(); @@ -1114,7 +1074,7 @@ public void initRestServer() { .flatMap(m -> m.entrySet().stream()) .collect( Collectors.groupingBy( - Map.Entry::getKey, + Entry::getKey, Collector.of( ArrayList::new, (list, item) -> list.add(item.getValue()), @@ -1142,8 +1102,7 @@ public void initRestServer() { LOG.info("Performing histogram2: {} took: {} ms.", histType, (endTime - startTime)); // Return final histogram to Web UI as output type. - NNAConstants.HISTOGRAM_OUTPUT output = - NNAConstants.HISTOGRAM_OUTPUT.valueOf(outputType); + HISTOGRAM_OUTPUT output = HISTOGRAM_OUTPUT.valueOf(outputType); switch (output) { case json: res.header("Content-Type", "application/json"); @@ -1286,10 +1245,10 @@ public void initRestServer() { limit = 1; } if (identity == null || identity.length() == 0) { - Set> entries = runningOperations.entrySet(); + Set> entries = runningOperations.entrySet(); StringBuilder sb = new StringBuilder(); sb.append("Total Operations: ").append(entries.size()).append('\n'); - for (Map.Entry entry : entries) { + for (Entry entry : entries) { BaseOperation operation = entry.getValue(); sb.append("ID: "); sb.append(entry.getKey()); @@ -1727,6 +1686,10 @@ public void initRestServer() { }); Spark.awaitInitialization(); + + nnLoader.initHistoryRecorder(hsqlDriver, conf, conf.getHistoricalEnabled()); + nnLoader.load(gSet, preloadedHadoopConf); + nnLoader.initReloadThreads(internalService, conf); } @VisibleForTesting diff --git a/src/main/java/com/paypal/security/SecurityConfiguration.java b/src/main/java/com/paypal/security/SecurityConfiguration.java index 5bfd111d..a1048a49 100644 --- a/src/main/java/com/paypal/security/SecurityConfiguration.java +++ b/src/main/java/com/paypal/security/SecurityConfiguration.java @@ -55,6 +55,10 @@ public SecurityConfiguration() { } } + public void set(String key, String value) { + properties.setProperty(key, value); + } + public boolean getHistoricalEnabled() { return Boolean.parseBoolean(properties.getProperty("nna.historical", NNA_HISTORICAL_DEFAULT)); } diff --git a/src/test/java/TestAuthorization.java b/src/test/java/TestAuthorization.java index ef16f171..e9f720c2 100644 --- a/src/test/java/TestAuthorization.java +++ b/src/test/java/TestAuthorization.java @@ -21,6 +21,7 @@ import static org.hamcrest.core.Is.is; import com.paypal.namenode.NNAnalyticsRestAPI; +import com.paypal.security.SecurityConfiguration; import java.io.IOException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hdfs.server.namenode.GSetGenerator; @@ -49,9 +50,10 @@ public static void beforeClass() throws Exception { gSetGenerator.clear(); GSet gset = gSetGenerator.getGSet((short) 3, 10, 500); nna = new NNAnalyticsRestAPI(); - nna.initAuth(false, true); - nna.initRestServer(); - nna.initLoader(gset, false); + SecurityConfiguration conf = new SecurityConfiguration(); + conf.set("ldap.enable", "false"); + conf.set("authorization.enable", "true"); + nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/java/TestFinderQuery.java b/src/test/java/TestFinderQuery.java index f614c282..f7d5df93 100644 --- a/src/test/java/TestFinderQuery.java +++ b/src/test/java/TestFinderQuery.java @@ -23,6 +23,7 @@ import static org.hamcrest.core.Is.is; import com.paypal.namenode.NNAnalyticsRestAPI; +import com.paypal.security.SecurityConfiguration; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -60,9 +61,11 @@ public static void beforeClass() throws Exception { gSetGenerator.clear(); GSet gset = gSetGenerator.getGSet((short) 3, 10, 500); nna = new NNAnalyticsRestAPI(); - nna.initAuth(false, false); - nna.initRestServer(); - nna.initLoader(gset, false); + SecurityConfiguration conf = new SecurityConfiguration(); + conf.set("ldap.enable", "false"); + conf.set("authorization.enable", "false"); + conf.set("nna.historical", "false"); + nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/java/TestINodeTransfer.java b/src/test/java/TestINodeTransfer.java index ac0ac3a2..0bfe9bff 100644 --- a/src/test/java/TestINodeTransfer.java +++ b/src/test/java/TestINodeTransfer.java @@ -58,9 +58,7 @@ public void putAll() { public void filterAll() { long start = System.currentTimeMillis(); List allNodes = - StreamSupport.stream(gset.spliterator(), true) - .filter(node -> true) - .collect(Collectors.toList()); + StreamSupport.stream(gset.spliterator(), true).collect(Collectors.toList()); long end = System.currentTimeMillis(); System.out.println("Took " + (end - start) + " ms."); assertThat(allNodes.size(), is(gset.size())); diff --git a/src/test/java/TestLdapAuth.java b/src/test/java/TestLdapAuth.java index 3cb2fd75..c0017bae 100644 --- a/src/test/java/TestLdapAuth.java +++ b/src/test/java/TestLdapAuth.java @@ -21,6 +21,7 @@ import static org.hamcrest.core.Is.is; import com.paypal.namenode.NNAnalyticsRestAPI; +import com.paypal.security.SecurityConfiguration; import java.io.IOException; import java.util.Base64; import org.apache.commons.io.IOUtils; @@ -51,9 +52,11 @@ public static void beforeClass() throws Exception { gSetGenerator.clear(); GSet gset = gSetGenerator.getGSet((short) 3, 10, 500); nna = new NNAnalyticsRestAPI(); - nna.initAuth(false, false); - nna.initRestServer(); - nna.initLoader(gset, false); + SecurityConfiguration conf = new SecurityConfiguration(); + conf.set("ldap.enable", "false"); + conf.set("authorization.enable", "false"); + conf.set("nna.historical", "false"); + nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/java/TestNNAnalytics.java b/src/test/java/TestNNAnalytics.java index ffea9e0d..031eb372 100644 --- a/src/test/java/TestNNAnalytics.java +++ b/src/test/java/TestNNAnalytics.java @@ -28,6 +28,7 @@ import com.google.gson.JsonObject; import com.google.gson.stream.JsonReader; import com.paypal.namenode.NNAnalyticsRestAPI; +import com.paypal.security.SecurityConfiguration; import java.io.IOException; import java.io.InputStreamReader; import java.text.ParseException; @@ -75,9 +76,11 @@ public static void beforeClass() throws Exception { gSetGenerator.clear(); GSet gset = gSetGenerator.getGSet((short) 3, 10, 500); nna = new NNAnalyticsRestAPI(); - nna.initAuth(false, false); - nna.initRestServer(); - nna.initLoader(gset, false); + SecurityConfiguration conf = new SecurityConfiguration(); + conf.set("ldap.enable", "false"); + conf.set("authorization.enable", "false"); + conf.set("nna.historical", "false"); + nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/java/TestNoHistorical.java b/src/test/java/TestNoHistorical.java index 547a5da0..1323b080 100644 --- a/src/test/java/TestNoHistorical.java +++ b/src/test/java/TestNoHistorical.java @@ -22,6 +22,7 @@ import static org.hamcrest.core.StringContains.containsString; import com.paypal.namenode.NNAnalyticsRestAPI; +import com.paypal.security.SecurityConfiguration; import java.io.IOException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hdfs.server.namenode.GSetGenerator; @@ -50,9 +51,11 @@ public static void beforeClass() throws Exception { gSetGenerator.clear(); GSet gset = gSetGenerator.getGSet((short) 3, 10, 500); nna = new NNAnalyticsRestAPI(); - nna.initAuth(false, false); - nna.initRestServer(); - nna.initLoader(gset, false); + SecurityConfiguration conf = new SecurityConfiguration(); + conf.set("ldap.enable", "false"); + conf.set("authorization.enable", "false"); + conf.set("nna.historical", "false"); + nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/java/TestQueryChecker.java b/src/test/java/TestQueryChecker.java index e4fa54da..b78d20cd 100644 --- a/src/test/java/TestQueryChecker.java +++ b/src/test/java/TestQueryChecker.java @@ -23,6 +23,7 @@ import static org.hamcrest.core.Is.is; import com.paypal.namenode.NNAnalyticsRestAPI; +import com.paypal.security.SecurityConfiguration; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -60,9 +61,11 @@ public static void beforeClass() throws Exception { gSetGenerator.clear(); GSet gset = gSetGenerator.getGSet((short) 3, 10, 500); nna = new NNAnalyticsRestAPI(); - nna.initAuth(false, false); - nna.initRestServer(); - nna.initLoader(gset, false); + SecurityConfiguration conf = new SecurityConfiguration(); + conf.set("ldap.enable", "false"); + conf.set("authorization.enable", "false"); + conf.set("nna.historical", "false"); + nna.init(conf, gset); hostPort = new HttpHost("localhost", 4567); } diff --git a/src/test/java/TestTransforms.java b/src/test/java/TestTransforms.java index d6145240..61b740c3 100644 --- a/src/test/java/TestTransforms.java +++ b/src/test/java/TestTransforms.java @@ -23,13 +23,13 @@ import static org.hamcrest.core.Is.is; import com.paypal.namenode.NNAnalyticsRestAPI; +import com.paypal.security.SecurityConfiguration; import java.util.Collection; import java.util.Map; import java.util.function.Function; import org.apache.hadoop.hdfs.server.namenode.GSetGenerator; import org.apache.hadoop.hdfs.server.namenode.INode; import org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields; -import org.apache.hadoop.hdfs.server.namenode.NNLoader; import org.apache.hadoop.hdfs.server.namenode.queries.Transforms; import org.apache.hadoop.util.GSet; import org.junit.AfterClass; @@ -42,7 +42,6 @@ public class TestTransforms { private static NNAnalyticsRestAPI nna; - private static NNLoader loader; @BeforeClass public static void beforeClass() throws Exception { @@ -50,9 +49,11 @@ public static void beforeClass() throws Exception { gSetGenerator.clear(); GSet gset = gSetGenerator.getGSet((short) 3, 10, 500); nna = new NNAnalyticsRestAPI(); - nna.initAuth(false, false); - nna.initRestServer(); - loader = nna.initLoader(gset, false); + SecurityConfiguration conf = new SecurityConfiguration(); + conf.set("ldap.enable", "false"); + conf.set("authorization.enable", "false"); + conf.set("nna.historical", "false"); + nna.init(conf, gset); } @AfterClass @@ -63,11 +64,11 @@ public static void tearDown() { @Test public void testTransformReplicationFactor() { Map> transformMap = - Transforms.getAttributeTransforms("fileReplica:gte:2", "fileReplica", "1", loader); + Transforms.getAttributeTransforms("fileReplica:gte:2", "fileReplica", "1", nna.getLoader()); assertThat(transformMap.size(), is(not(0))); Function fileReplicaTransform = transformMap.get("fileReplica"); assertThat(fileReplicaTransform, is(notNullValue())); - for (INode node : loader.getINodeSet("files")) { + for (INode node : nna.getLoader().getINodeSet("files")) { Long transformedFileReplica = fileReplicaTransform.apply(node); assertThat(transformedFileReplica, is(1L)); } @@ -76,11 +77,11 @@ public void testTransformReplicationFactor() { @Test public void testTransformDiskspaceConsumedByReplFactor() { Map> transformMap = - Transforms.getAttributeTransforms("fileReplica:gte:2", "fileReplica", "1", loader); + Transforms.getAttributeTransforms("fileReplica:gte:2", "fileReplica", "1", nna.getLoader()); assertThat(transformMap.size(), is(not(0))); Function fileReplicaTransform = transformMap.get("diskspaceConsumed"); assertThat(fileReplicaTransform, is(notNullValue())); - Collection files = loader.getINodeSet("files"); + Collection files = nna.getLoader().getINodeSet("files"); long diskspaceConsumed = files .stream() diff --git a/src/test/java/TestWithMiniCluster.java b/src/test/java/TestWithMiniCluster.java index 6064b8fb..cd7fcd81 100644 --- a/src/test/java/TestWithMiniCluster.java +++ b/src/test/java/TestWithMiniCluster.java @@ -23,6 +23,7 @@ import static org.hamcrest.core.StringContains.containsString; import com.paypal.namenode.NNAnalyticsRestAPI; +import com.paypal.security.SecurityConfiguration; import java.io.IOException; import java.util.concurrent.TimeUnit; import org.apache.commons.io.IOUtils; @@ -91,9 +92,11 @@ public static void beforeClass() throws Exception { CONF.set("dfs.nameservice.id", NAMESERVICE); nna = new NNAnalyticsRestAPI(); - nna.initAuth(false, false); - nna.initRestServer(); - nna.initLoader(null, null, CONF); + SecurityConfiguration nnaConf = new SecurityConfiguration(); + nnaConf.set("ldap.enable", "false"); + nnaConf.set("authorization.enable", "false"); + nnaConf.set("nna.historical", "false"); + nna.init(nnaConf, null, CONF); hostPort = new HttpHost("localhost", 4567); client = new DefaultHttpClient();