Skip to content

Commit

Permalink
Fix "export" endtoend Tests (#1354)
Browse files Browse the repository at this point in the history 
* Expect export files to be named after "test" source and not "ripe" source.
Use "public" for external (filtered) dump vs. "internal" for private dump

* Fixed tests (renamed external directory from "dbase" to "public")
  • Loading branch information
@eshryane
eshryane authored Nov 17, 2023
1 parent 15fc3d3 commit ee8e18c
Show file tree
Hide file tree
Showing 5 changed files with 87 additions and 75 deletions.
38 changes: 29 additions & 9 deletions whois-commons/src/test/resources/whois.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ port.nrtm=0
dir.rpsl.export=var${jvmId:}/export
dir.rpsl.export.tmp=var${jvmId:}/export_tmp
dir.rpsl.export.internal=internal
dir.rpsl.export.external=dbase
dir.rpsl.export.external=public
dir.update.audit.log=var${jvmId:}/log/audit
dir.grs.import.download=var${jvmId:}/grs

Expand All @@ -30,7 +30,29 @@ ipranges.trusted=127.0.0.1,::1
ipranges.loadbalancer=127.0.0.1,::1

# Bogon space (ref. http://www.team-cymru.com/bogon-reference.html)
ipranges.bogons=0.0.0.0/8,10.0.0.0/8,100.64.0.0/10,127.0.0.0/8,169.254.0.0/16,172.16.0.0/12,192.0.2.0/24,192.168.0.0/16,198.18.0.0/15,198.51.100.0/24,203.0.113.0/24,224.0.0.0/4,240.0.0.0/4,::/8,0100::/64,2001:2::/48,2001:10::/28,2001:db8::/32,3ffe::/16,fc00::/7,fe80::/10,fec0::/10,ff00::/8,
ipranges.bogons=0.0.0.0/8,\
10.0.0.0/8,\
100.64.0.0/10,\
127.0.0.0/8,\
169.254.0.0/16,\
172.16.0.0/12,\
192.0.2.0/24,\
192.168.0.0/16,\
198.18.0.0/15,\
198.51.100.0/24,\
203.0.113.0/24,\
224.0.0.0/4,\
240.0.0.0/4,\
::/8,\
0100::/64,\
2001:2::/48,\
2001:10::/28,\
2001:db8::/32,\
3ffe::/16,\
fc00::/7,\
fe80::/10,\
fec0::/10,\
ff00::/8

# Mail properties
mail.smtp.enabled=true
Expand All @@ -40,13 +62,10 @@ mail.from=RIPE Database Administration local <[email protected]>
mail.update.threads=2
mail.dequeue.interval=1000

mail.smtp.retrySending=true

# NRTM server
nrtm.enabled=true
nrtm.update.interval=15


# NRTM client
nrtm.import.enabled=false
nrtm.import.sources=
Expand Down Expand Up @@ -109,21 +128,21 @@ nrtm.slave.database.url=jdbc:mysql://${db.host:localhost}/NRTM_LOCAL
nrtm.slave.database.username=dbint
nrtm.slave.database.password=

# NRTMv4
nrtm.file.path=/tmp
nrtm.snapshot.window=23:00 - 05:00
nrtm.baseUrl= https://nrtm.db.ripe.net

#Feature Toggles
feature.toggle.changed.attr.available=true

#Dummy attributes
whois.dummy={'auth':'MD5-PW $1$SaltSalt$DummifiedMD5HashValue.', 'tech-c':'DR1-TEST', 'admin-c':'DR1-TEST'}

# Zonemaster
whois.zonemaster.baseUrl=http://zonemaster-test.ripe.net:5000

client.cert.auth.enabled=true

nrtm.baseUrl= https://nrtm.db.ripe.net
# RDAP properties
# RDAP
rdap.tnc.title=Terms and Conditions
rdap.tnc.description=This is the RIPE Database query service. The objects are in RDAP format.
rdap.tnc.linkrel=terms-of-service
Expand All @@ -147,3 +166,4 @@ instance.name=localhost

#Dump size limit in MB
dump.total.size.limit= 15

33 changes: 13 additions & 20 deletions whois-endtoend/src/test/java/net/ripe/db/whois/changedphase3/util/ExportRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,12 @@
import static org.junit.jupiter.api.Assertions.fail;

public class ExportRunner extends AbstractScenarioRunner {
private static final String EXPORT_DIR = "var" + System.getProperty("jvmId") + "/export";

private static final String EXPORT_DIR;
static {
final String jvmId = System.getProperty("jvmId");
EXPORT_DIR = "var" + (jvmId != null ? jvmId : "") + "/export";
}

public ExportRunner(final Context context) {
super(context);
Expand Down Expand Up @@ -56,7 +61,6 @@ public void delete(final Scenario scenario) {
}

private void prepareFromDump(final Scenario scenario, final Updater updater) {

try {
verifyPreCondition(scenario);

Expand All @@ -67,10 +71,9 @@ private void prepareFromDump(final Scenario scenario, final Updater updater) {

context.getDatabaseTextExport().run();

String dumpFile = readFile(EXPORT_DIR + "/dbase/ripe.db.gz");
String splitFile = readFile(EXPORT_DIR + "/dbase/split/ripe.db.mntner.gz");

String internalFile = readFile(EXPORT_DIR + "/internal/split/ripe.db.mntner.gz");
final String dumpFile = readFile(EXPORT_DIR + "/public/test.db.gz");
final String splitFile = readFile(EXPORT_DIR + "/public/split/test.db.mntner.gz");
final String internalFile = readFile(EXPORT_DIR + "/internal/split/test.db.mntner.gz");

if (scenario.getPostCond() == Scenario.ObjectStatus.OBJ_EXISTS_WITH_CHANGED) {
assertThat(dumpFile, containsString("TESTING-MNT"));
Expand Down Expand Up @@ -99,7 +102,7 @@ private void prepareFromDump(final Scenario scenario, final Updater updater) {
}

} catch (IOException exc) {
System.err.println("Error reading splitfile:" +exc.toString());
System.err.println("Error reading splitfile: " +exc.getClass().getName() + ": " + exc.getMessage());
fail();
} finally {
context.getNrtmServer().stop(true);
Expand All @@ -122,24 +125,14 @@ private void deleteExportDir() {
}

private String readFile(final String filename) throws IOException {
BufferedReader in = null;
try {
in = new BufferedReader(new InputStreamReader(
new GZIPInputStream(new FileInputStream(filename))));

StringBuffer sb = new StringBuffer();

try (final BufferedReader in = new BufferedReader(new InputStreamReader(
new GZIPInputStream(new FileInputStream(filename))))) {
final StringBuilder sb = new StringBuilder();
String chunk;
while ((chunk = in.readLine()) != null) {
sb.append(chunk);
}
return sb.toString();
} finally {
if (in != null) {
try {
in.close();
} catch( IOException exc) {}
}
}
}

Expand Down
22 changes: 11 additions & 11 deletions whois-rpsl/src/test/java/net/ripe/db/whois/common/rpsl/AttributeParserTestIntegration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public class AttributeParserTestIntegration {

@Test
public void parseAutnumAttributes() throws Exception {
parseAttributes("/export/opt/ripe.db.aut-num.gz",
parseAttributes("/export/opt/test.db.aut-num.gz",
new AttributeType[]{
AttributeType.EXPORT,
AttributeType.IMPORT,
Expand All @@ -38,7 +38,7 @@ public void parseAutnumAttributes() throws Exception {

@Test
public void parseInetRtrAttributes() throws Exception {
parseAttributes("/export/opt/ripe.db.inet-rtr.gz",
parseAttributes("/export/opt/test.db.inet-rtr.gz",
new AttributeType[]{
AttributeType.ALIAS,
AttributeType.IFADDR,
Expand All @@ -51,7 +51,7 @@ public void parseInetRtrAttributes() throws Exception {

@Test
public void parseAsSetAttributes() throws Exception {
parseAttributes("/export/opt/ripe.db.as-set.gz",
parseAttributes("/export/opt/test.db.as-set.gz",
new AttributeType[]{
AttributeType.MEMBERS
}
Expand All @@ -60,7 +60,7 @@ public void parseAsSetAttributes() throws Exception {

@Test
public void parseRouteSetAttributes() throws Exception {
parseAttributes("/export/opt/ripe.db.route-set.gz",
parseAttributes("/export/opt/test.db.route-set.gz",
new AttributeType[]{
AttributeType.MEMBERS,
AttributeType.MP_MEMBERS
Expand All @@ -70,7 +70,7 @@ public void parseRouteSetAttributes() throws Exception {

@Test
public void parseRtrSetAttributes() throws Exception {
parseAttributes("/export/opt/ripe.db.rtr-set.gz",
parseAttributes("/export/opt/test.db.rtr-set.gz",
new AttributeType[]{
AttributeType.MEMBERS,
AttributeType.MP_MEMBERS
Expand All @@ -80,7 +80,7 @@ public void parseRtrSetAttributes() throws Exception {

@Test
public void parseFilterSetAttributes() throws Exception {
parseAttributes("/export/opt/ripe.db.filter-set.gz",
parseAttributes("/export/opt/test.db.filter-set.gz",
new AttributeType[]{
AttributeType.FILTER,
AttributeType.MP_FILTER
Expand All @@ -90,7 +90,7 @@ public void parseFilterSetAttributes() throws Exception {

@Test
public void parsePeeringSetAttributes() throws Exception {
parseAttributes("/export/opt/ripe.db.peering-set.gz",
parseAttributes("/export/opt/test.db.peering-set.gz",
new AttributeType[]{
AttributeType.PEERING,
AttributeType.MP_PEERING
Expand All @@ -100,7 +100,7 @@ public void parsePeeringSetAttributes() throws Exception {

@Test
public void parseRouteAttributes() throws Exception {
parseAttributes("/export/opt/ripe.db.route.gz",
parseAttributes("/export/opt/test.db.route.gz",
new AttributeType[]{
AttributeType.INJECT,
AttributeType.AGGR_MTD,
Expand All @@ -114,7 +114,7 @@ public void parseRouteAttributes() throws Exception {

@Test
public void parseRoute6Attributes() throws Exception {
parseAttributes("/export/opt/ripe.db.route6.gz",
parseAttributes("/export/opt/test.db.route6.gz",
new AttributeType[]{
AttributeType.INJECT,
AttributeType.AGGR_MTD,
Expand All @@ -128,7 +128,7 @@ public void parseRoute6Attributes() throws Exception {

@Test
public void parseInetnumAttributes() throws Exception {
parseAttributes("/export/opt/ripe.db.inetnum.gz",
parseAttributes("/export/opt/test.db.inetnum.gz",
new AttributeType[]{
AttributeType.MNT_ROUTES
}
Expand All @@ -137,7 +137,7 @@ public void parseInetnumAttributes() throws Exception {

@Test
public void parseInet6numAttributes() throws Exception {
parseAttributes("/export/opt/ripe.db.inet6num.gz",
parseAttributes("/export/opt/test.db.inet6num.gz",
new AttributeType[]{
AttributeType.MNT_ROUTES
}
Expand Down
60 changes: 30 additions & 30 deletions .../src/test/java/net/ripe/db/whois/scheduler/task/export/ExportDatabaseTestIntegration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,13 +100,13 @@ public void export() throws IOException {
assertThat(exportDir.exists(), is(true));

for (final ObjectType objectType : ObjectType.values()) {
checkFile("dbase/split/test.db." + objectType.getName() + ".gz");
checkFile("public/split/test.db." + objectType.getName() + ".gz");
checkFile("internal/split/test.db." + objectType.getName() + ".gz");
}

checkFile("dbase/TEST.CURRENTSERIAL", "120");
checkFile("public/TEST.CURRENTSERIAL", "120");

checkFile("dbase/test.db.gz",
checkFile("public/test.db.gz",
"person: Placeholder Person Object\n",
"mntner: DEV-MNT0\n",
"mntner: DEV-MNT1\n",
Expand All @@ -128,9 +128,9 @@ public void export() throws IOException {
"remarks: * http://www.ripe.net/whois\n" +
"remarks: ****************************\n");

checkFile("dbase/split/test.db.person.gz", "person: Placeholder Person Object");
checkFile("public/split/test.db.person.gz", "person: Placeholder Person Object");

checkFile("dbase/split/test.db.mntner.gz",
checkFile("public/split/test.db.mntner.gz",
"mntner: DEV-MNT0\n",
"mntner: DEV-MNT1\n",
"mntner: DEV-MNT2\n",
Expand All @@ -140,16 +140,16 @@ public void export() throws IOException {
"mntner: DEV-MNT6\n",
"mntner: DEV-MNT7\n",
"" +
"mntner: DEV-MNT99\n" +
"auth: MD5-PW $1$SaltSalt$DummifiedMD5HashValue. # Real value hidden for security\n" +
"source: TEST\n" +
"remarks: ****************************\n" +
"remarks: * THIS OBJECT IS MODIFIED\n" +
"remarks: * Please note that all data that is generally regarded as personal\n" +
"remarks: * data has been removed from this object.\n" +
"remarks: * To view the original object, please query the RIPE Database at:\n" +
"remarks: * http://www.ripe.net/whois\n" +
"remarks: ****************************\n");
"mntner: DEV-MNT99\n" +
"auth: MD5-PW $1$SaltSalt$DummifiedMD5HashValue. # Real value hidden for security\n" +
"source: TEST\n" +
"remarks: ****************************\n" +
"remarks: * THIS OBJECT IS MODIFIED\n" +
"remarks: * Please note that all data that is generally regarded as personal\n" +
"remarks: * data has been removed from this object.\n" +
"remarks: * To view the original object, please query the RIPE Database at:\n" +
"remarks: * http://www.ripe.net/whois\n" +
"remarks: ****************************\n");

checkFile("internal/split/test.db.person.gz",
"person: Test person 0",
Expand All @@ -158,9 +158,9 @@ public void export() throws IOException {
"person: Test person 3",
"person: Test person 4",
"" +
"person: Test person 9\n" +
"nic-hdl: PN9-TEST\n" +
"source: TEST");
"person: Test person 9\n" +
"nic-hdl: PN9-TEST\n" +
"source: TEST");

checkFile("internal/split/test.db.role.gz",
"role: Test role 0",
Expand All @@ -169,9 +169,9 @@ public void export() throws IOException {
"role: Test role 3",
"role: Test role 4",
"" +
"role: Test role 9\n" +
"nic-hdl: ROLE9-TEST\n" +
"source: TEST");
"role: Test role 9\n" +
"nic-hdl: ROLE9-TEST\n" +
"source: TEST");

checkFile("internal/split/test.db.mntner.gz",
"" +
Expand Down Expand Up @@ -241,19 +241,19 @@ public void export_role_with_abuse_mailbox() throws IOException {
assertThat(exportDir.exists(), is(true));

for (final ObjectType objectType : ObjectType.values()) {
checkFile("dbase/split/test.db." + objectType.getName() + ".gz");
checkFile("public/split/test.db." + objectType.getName() + ".gz");
checkFile("internal/split/test.db." + objectType.getName() + ".gz");
}

checkFile("dbase/split/test.db.person.gz", "person: Placeholder Person Object");
checkFile("public/split/test.db.person.gz", "person: Placeholder Person Object");

checkFile("dbase/split/test.db.role.gz", "" +
checkFile("public/split/test.db.role.gz", "" +
"role: Abuse role\n" +
"nic-hdl: AR1-TEST\n" +
"abuse-mailbox: [email protected]\n" +
"source: TEST");

checkFile("dbase/split/test.db.organisation.gz", "" +
checkFile("public/split/test.db.organisation.gz", "" +
"organisation: ORG1\n" +
"abuse-c: AR1-TEST\n" +
"source: TEST");
Expand Down Expand Up @@ -320,17 +320,17 @@ public void export_mix_of_sources() throws IOException {
assertThat(exportDir.exists(), is(true));

for (final ObjectType objectType : ObjectType.values()) {
checkFile("dbase/split/test.db." + objectType.getName() + ".gz");
checkFile("public/split/test.db." + objectType.getName() + ".gz");
checkFile("internal/split/test.db." + objectType.getName() + ".gz");
if (ExportFileWriterFactory.NONAUTH_OBJECT_TYPES.contains(objectType)) {
checkFile("dbase/split/test-nonauth.db." + objectType.getName() + ".gz");
checkFile("public/split/test-nonauth.db." + objectType.getName() + ".gz");
checkFile("internal/split/test-nonauth.db." + objectType.getName() + ".gz");
}
}

checkFile("dbase/split/test.db.aut-num.gz", "aut-num: AS252");
checkFile("dbase/split/test-nonauth.db.aut-num.gz", "aut-num: AS251");
checkFile("dbase/split/test-nonauth.db.as-set.gz", "as-set: AS251:AS-ALL");
checkFile("public/split/test.db.aut-num.gz", "aut-num: AS252");
checkFile("public/split/test-nonauth.db.aut-num.gz", "aut-num: AS251");
checkFile("public/split/test-nonauth.db.as-set.gz", "as-set: AS251:AS-ALL");
}

}
Loading

0 comments on commit ee8e18c

Please sign in to comment.