Skip to content

Commit

Permalink
[#61] Make NNA constructor that accepts configuration
Browse files Browse the repository at this point in the history
  • Loading branch information
@pjeli
pjeli authored Jul 13, 2018
1 parent d25e598 commit 1e60c6a
Show file tree
Hide file tree
Showing 11 changed files with 122 additions and 136 deletions.
169 changes: 66 additions & 103 deletions src/main/java/com/paypal/namenode/NNAnalyticsRestAPI.java
View file

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions src/main/java/com/paypal/security/SecurityConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ public SecurityConfiguration() {
}
}

public void set(String key, String value) {
properties.setProperty(key, value);
}

public boolean getHistoricalEnabled() {
return Boolean.parseBoolean(properties.getProperty("nna.historical", NNA_HISTORICAL_DEFAULT));
}
Expand Down
8 changes: 5 additions & 3 deletions src/test/java/TestAuthorization.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static org.hamcrest.core.Is.is;

import com.paypal.namenode.NNAnalyticsRestAPI;
import com.paypal.security.SecurityConfiguration;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hdfs.server.namenode.GSetGenerator;
Expand Down Expand Up @@ -49,9 +50,10 @@ public static void beforeClass() throws Exception {
gSetGenerator.clear();
GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
nna = new NNAnalyticsRestAPI();
nna.initAuth(false, true);
nna.initRestServer();
nna.initLoader(gset, false);
SecurityConfiguration conf = new SecurityConfiguration();
conf.set("ldap.enable", "false");
conf.set("authorization.enable", "true");
nna.init(conf, gset);
hostPort = new HttpHost("localhost", 4567);
}

Expand Down
9 changes: 6 additions & 3 deletions src/test/java/TestFinderQuery.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import static org.hamcrest.core.Is.is;

import com.paypal.namenode.NNAnalyticsRestAPI;
import com.paypal.security.SecurityConfiguration;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
Expand Down Expand Up @@ -60,9 +61,11 @@ public static void beforeClass() throws Exception {
gSetGenerator.clear();
GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
nna = new NNAnalyticsRestAPI();
nna.initAuth(false, false);
nna.initRestServer();
nna.initLoader(gset, false);
SecurityConfiguration conf = new SecurityConfiguration();
conf.set("ldap.enable", "false");
conf.set("authorization.enable", "false");
conf.set("nna.historical", "false");
nna.init(conf, gset);
hostPort = new HttpHost("localhost", 4567);
}

Expand Down
4 changes: 1 addition & 3 deletions src/test/java/TestINodeTransfer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,7 @@ public void putAll() {
public void filterAll() {
long start = System.currentTimeMillis();
List<INodeWithAdditionalFields> allNodes =
StreamSupport.stream(gset.spliterator(), true)
.filter(node -> true)
.collect(Collectors.toList());
StreamSupport.stream(gset.spliterator(), true).collect(Collectors.toList());
long end = System.currentTimeMillis();
System.out.println("Took " + (end - start) + " ms.");
assertThat(allNodes.size(), is(gset.size()));
Expand Down
9 changes: 6 additions & 3 deletions src/test/java/TestLdapAuth.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static org.hamcrest.core.Is.is;

import com.paypal.namenode.NNAnalyticsRestAPI;
import com.paypal.security.SecurityConfiguration;
import java.io.IOException;
import java.util.Base64;
import org.apache.commons.io.IOUtils;
Expand Down Expand Up @@ -51,9 +52,11 @@ public static void beforeClass() throws Exception {
gSetGenerator.clear();
GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
nna = new NNAnalyticsRestAPI();
nna.initAuth(false, false);
nna.initRestServer();
nna.initLoader(gset, false);
SecurityConfiguration conf = new SecurityConfiguration();
conf.set("ldap.enable", "false");
conf.set("authorization.enable", "false");
conf.set("nna.historical", "false");
nna.init(conf, gset);
hostPort = new HttpHost("localhost", 4567);
}

Expand Down
9 changes: 6 additions & 3 deletions src/test/java/TestNNAnalytics.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.google.gson.JsonObject;
import com.google.gson.stream.JsonReader;
import com.paypal.namenode.NNAnalyticsRestAPI;
import com.paypal.security.SecurityConfiguration;
import java.io.IOException;
import java.io.InputStreamReader;
import java.text.ParseException;
Expand Down Expand Up @@ -75,9 +76,11 @@ public static void beforeClass() throws Exception {
gSetGenerator.clear();
GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
nna = new NNAnalyticsRestAPI();
nna.initAuth(false, false);
nna.initRestServer();
nna.initLoader(gset, false);
SecurityConfiguration conf = new SecurityConfiguration();
conf.set("ldap.enable", "false");
conf.set("authorization.enable", "false");
conf.set("nna.historical", "false");
nna.init(conf, gset);
hostPort = new HttpHost("localhost", 4567);
}

Expand Down
9 changes: 6 additions & 3 deletions src/test/java/TestNoHistorical.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static org.hamcrest.core.StringContains.containsString;

import com.paypal.namenode.NNAnalyticsRestAPI;
import com.paypal.security.SecurityConfiguration;
import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hdfs.server.namenode.GSetGenerator;
Expand Down Expand Up @@ -50,9 +51,11 @@ public static void beforeClass() throws Exception {
gSetGenerator.clear();
GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
nna = new NNAnalyticsRestAPI();
nna.initAuth(false, false);
nna.initRestServer();
nna.initLoader(gset, false);
SecurityConfiguration conf = new SecurityConfiguration();
conf.set("ldap.enable", "false");
conf.set("authorization.enable", "false");
conf.set("nna.historical", "false");
nna.init(conf, gset);
hostPort = new HttpHost("localhost", 4567);
}

Expand Down
9 changes: 6 additions & 3 deletions src/test/java/TestQueryChecker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import static org.hamcrest.core.Is.is;

import com.paypal.namenode.NNAnalyticsRestAPI;
import com.paypal.security.SecurityConfiguration;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
Expand Down Expand Up @@ -60,9 +61,11 @@ public static void beforeClass() throws Exception {
gSetGenerator.clear();
GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
nna = new NNAnalyticsRestAPI();
nna.initAuth(false, false);
nna.initRestServer();
nna.initLoader(gset, false);
SecurityConfiguration conf = new SecurityConfiguration();
conf.set("ldap.enable", "false");
conf.set("authorization.enable", "false");
conf.set("nna.historical", "false");
nna.init(conf, gset);
hostPort = new HttpHost("localhost", 4567);
}

Expand Down
19 changes: 10 additions & 9 deletions src/test/java/TestTransforms.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,13 @@
import static org.hamcrest.core.Is.is;

import com.paypal.namenode.NNAnalyticsRestAPI;
import com.paypal.security.SecurityConfiguration;
import java.util.Collection;
import java.util.Map;
import java.util.function.Function;
import org.apache.hadoop.hdfs.server.namenode.GSetGenerator;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields;
import org.apache.hadoop.hdfs.server.namenode.NNLoader;
import org.apache.hadoop.hdfs.server.namenode.queries.Transforms;
import org.apache.hadoop.util.GSet;
import org.junit.AfterClass;
Expand All @@ -42,17 +42,18 @@
public class TestTransforms {

private static NNAnalyticsRestAPI nna;
private static NNLoader loader;

@BeforeClass
public static void beforeClass() throws Exception {
GSetGenerator gSetGenerator = new GSetGenerator();
gSetGenerator.clear();
GSet<INode, INodeWithAdditionalFields> gset = gSetGenerator.getGSet((short) 3, 10, 500);
nna = new NNAnalyticsRestAPI();
nna.initAuth(false, false);
nna.initRestServer();
loader = nna.initLoader(gset, false);
SecurityConfiguration conf = new SecurityConfiguration();
conf.set("ldap.enable", "false");
conf.set("authorization.enable", "false");
conf.set("nna.historical", "false");
nna.init(conf, gset);
}

@AfterClass
Expand All @@ -63,11 +64,11 @@ public static void tearDown() {
@Test
public void testTransformReplicationFactor() {
Map<String, Function<INode, Long>> transformMap =
Transforms.getAttributeTransforms("fileReplica:gte:2", "fileReplica", "1", loader);
Transforms.getAttributeTransforms("fileReplica:gte:2", "fileReplica", "1", nna.getLoader());
assertThat(transformMap.size(), is(not(0)));
Function<INode, Long> fileReplicaTransform = transformMap.get("fileReplica");
assertThat(fileReplicaTransform, is(notNullValue()));
for (INode node : loader.getINodeSet("files")) {
for (INode node : nna.getLoader().getINodeSet("files")) {
Long transformedFileReplica = fileReplicaTransform.apply(node);
assertThat(transformedFileReplica, is(1L));
}
Expand All @@ -76,11 +77,11 @@ public void testTransformReplicationFactor() {
@Test
public void testTransformDiskspaceConsumedByReplFactor() {
Map<String, Function<INode, Long>> transformMap =
Transforms.getAttributeTransforms("fileReplica:gte:2", "fileReplica", "1", loader);
Transforms.getAttributeTransforms("fileReplica:gte:2", "fileReplica", "1", nna.getLoader());
assertThat(transformMap.size(), is(not(0)));
Function<INode, Long> fileReplicaTransform = transformMap.get("diskspaceConsumed");
assertThat(fileReplicaTransform, is(notNullValue()));
Collection<INode> files = loader.getINodeSet("files");
Collection<INode> files = nna.getLoader().getINodeSet("files");
long diskspaceConsumed =
files
.stream()
Expand Down
9 changes: 6 additions & 3 deletions src/test/java/TestWithMiniCluster.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import static org.hamcrest.core.StringContains.containsString;

import com.paypal.namenode.NNAnalyticsRestAPI;
import com.paypal.security.SecurityConfiguration;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
Expand Down Expand Up @@ -91,9 +92,11 @@ public static void beforeClass() throws Exception {
CONF.set("dfs.nameservice.id", NAMESERVICE);

nna = new NNAnalyticsRestAPI();
nna.initAuth(false, false);
nna.initRestServer();
nna.initLoader(null, null, CONF);
SecurityConfiguration nnaConf = new SecurityConfiguration();
nnaConf.set("ldap.enable", "false");
nnaConf.set("authorization.enable", "false");
nnaConf.set("nna.historical", "false");
nna.init(nnaConf, null, CONF);
hostPort = new HttpHost("localhost", 4567);
client = new DefaultHttpClient();

Expand Down

0 comments on commit 1e60c6a

Please sign in to comment.