From ee8e18c53aa528a323caf44873f6e8428077bdf5 Mon Sep 17 00:00:00 2001 From: Ed Shryane Date: Fri, 17 Nov 2023 23:33:40 +0100 Subject: [PATCH] Fix "export" endtoend Tests (#1354) * Expect export files to be named after "test" source and not "ripe" source. Use "public" for external (filtered) dump vs. "internal" for private dump * Fixed tests (renamed external directory from "dbase" to "public") --- .../src/test/resources/whois.properties | 38 +++++++++--- .../changedphase3/util/ExportRunner.java | 33 ++++------ .../rpsl/AttributeParserTestIntegration.java | 22 +++---- .../export/ExportDatabaseTestIntegration.java | 60 +++++++++---------- .../export/ExportFileWriterFactoryTest.java | 9 ++- 5 files changed, 87 insertions(+), 75 deletions(-) diff --git a/whois-commons/src/test/resources/whois.properties b/whois-commons/src/test/resources/whois.properties index c98e005530..bcab30593d 100644 --- a/whois-commons/src/test/resources/whois.properties +++ b/whois-commons/src/test/resources/whois.properties @@ -12,7 +12,7 @@ port.nrtm=0 dir.rpsl.export=var${jvmId:}/export dir.rpsl.export.tmp=var${jvmId:}/export_tmp dir.rpsl.export.internal=internal -dir.rpsl.export.external=dbase +dir.rpsl.export.external=public dir.update.audit.log=var${jvmId:}/log/audit dir.grs.import.download=var${jvmId:}/grs @@ -30,7 +30,29 @@ ipranges.trusted=127.0.0.1,::1 ipranges.loadbalancer=127.0.0.1,::1 # Bogon space (ref. http://www.team-cymru.com/bogon-reference.html) -ipranges.bogons=0.0.0.0/8,10.0.0.0/8,100.64.0.0/10,127.0.0.0/8,169.254.0.0/16,172.16.0.0/12,192.0.2.0/24,192.168.0.0/16,198.18.0.0/15,198.51.100.0/24,203.0.113.0/24,224.0.0.0/4,240.0.0.0/4,::/8,0100::/64,2001:2::/48,2001:10::/28,2001:db8::/32,3ffe::/16,fc00::/7,fe80::/10,fec0::/10,ff00::/8, +ipranges.bogons=0.0.0.0/8,\ +10.0.0.0/8,\ +100.64.0.0/10,\ +127.0.0.0/8,\ +169.254.0.0/16,\ +172.16.0.0/12,\ +192.0.2.0/24,\ +192.168.0.0/16,\ +198.18.0.0/15,\ +198.51.100.0/24,\ +203.0.113.0/24,\ +224.0.0.0/4,\ +240.0.0.0/4,\ +::/8,\ +0100::/64,\ +2001:2::/48,\ +2001:10::/28,\ +2001:db8::/32,\ +3ffe::/16,\ +fc00::/7,\ +fe80::/10,\ +fec0::/10,\ +ff00::/8 # Mail properties mail.smtp.enabled=true @@ -40,13 +62,10 @@ mail.from=RIPE Database Administration local mail.update.threads=2 mail.dequeue.interval=1000 -mail.smtp.retrySending=true - # NRTM server nrtm.enabled=true nrtm.update.interval=15 - # NRTM client nrtm.import.enabled=false nrtm.import.sources= @@ -109,8 +128,10 @@ nrtm.slave.database.url=jdbc:mysql://${db.host:localhost}/NRTM_LOCAL nrtm.slave.database.username=dbint nrtm.slave.database.password= +# NRTMv4 nrtm.file.path=/tmp nrtm.snapshot.window=23:00 - 05:00 +nrtm.baseUrl= https://nrtm.db.ripe.net #Feature Toggles feature.toggle.changed.attr.available=true @@ -118,12 +139,10 @@ feature.toggle.changed.attr.available=true #Dummy attributes whois.dummy={'auth':'MD5-PW $1$SaltSalt$DummifiedMD5HashValue.', 'tech-c':'DR1-TEST', 'admin-c':'DR1-TEST'} +# Zonemaster whois.zonemaster.baseUrl=http://zonemaster-test.ripe.net:5000 -client.cert.auth.enabled=true - -nrtm.baseUrl= https://nrtm.db.ripe.net -# RDAP properties +# RDAP rdap.tnc.title=Terms and Conditions rdap.tnc.description=This is the RIPE Database query service. The objects are in RDAP format. rdap.tnc.linkrel=terms-of-service @@ -147,3 +166,4 @@ instance.name=localhost #Dump size limit in MB dump.total.size.limit= 15 + diff --git a/whois-endtoend/src/test/java/net/ripe/db/whois/changedphase3/util/ExportRunner.java b/whois-endtoend/src/test/java/net/ripe/db/whois/changedphase3/util/ExportRunner.java index 8bb02488d9..5acd0423ae 100644 --- a/whois-endtoend/src/test/java/net/ripe/db/whois/changedphase3/util/ExportRunner.java +++ b/whois-endtoend/src/test/java/net/ripe/db/whois/changedphase3/util/ExportRunner.java @@ -16,7 +16,12 @@ import static org.junit.jupiter.api.Assertions.fail; public class ExportRunner extends AbstractScenarioRunner { - private static final String EXPORT_DIR = "var" + System.getProperty("jvmId") + "/export"; + + private static final String EXPORT_DIR; + static { + final String jvmId = System.getProperty("jvmId"); + EXPORT_DIR = "var" + (jvmId != null ? jvmId : "") + "/export"; + } public ExportRunner(final Context context) { super(context); @@ -56,7 +61,6 @@ public void delete(final Scenario scenario) { } private void prepareFromDump(final Scenario scenario, final Updater updater) { - try { verifyPreCondition(scenario); @@ -67,10 +71,9 @@ private void prepareFromDump(final Scenario scenario, final Updater updater) { context.getDatabaseTextExport().run(); - String dumpFile = readFile(EXPORT_DIR + "/dbase/ripe.db.gz"); - String splitFile = readFile(EXPORT_DIR + "/dbase/split/ripe.db.mntner.gz"); - - String internalFile = readFile(EXPORT_DIR + "/internal/split/ripe.db.mntner.gz"); + final String dumpFile = readFile(EXPORT_DIR + "/public/test.db.gz"); + final String splitFile = readFile(EXPORT_DIR + "/public/split/test.db.mntner.gz"); + final String internalFile = readFile(EXPORT_DIR + "/internal/split/test.db.mntner.gz"); if (scenario.getPostCond() == Scenario.ObjectStatus.OBJ_EXISTS_WITH_CHANGED) { assertThat(dumpFile, containsString("TESTING-MNT")); @@ -99,7 +102,7 @@ private void prepareFromDump(final Scenario scenario, final Updater updater) { } } catch (IOException exc) { - System.err.println("Error reading splitfile:" +exc.toString()); + System.err.println("Error reading splitfile: " +exc.getClass().getName() + ": " + exc.getMessage()); fail(); } finally { context.getNrtmServer().stop(true); @@ -122,24 +125,14 @@ private void deleteExportDir() { } private String readFile(final String filename) throws IOException { - BufferedReader in = null; - try { - in = new BufferedReader(new InputStreamReader( - new GZIPInputStream(new FileInputStream(filename)))); - - StringBuffer sb = new StringBuffer(); - + try (final BufferedReader in = new BufferedReader(new InputStreamReader( + new GZIPInputStream(new FileInputStream(filename))))) { + final StringBuilder sb = new StringBuilder(); String chunk; while ((chunk = in.readLine()) != null) { sb.append(chunk); } return sb.toString(); - } finally { - if (in != null) { - try { - in.close(); - } catch( IOException exc) {} - } } } diff --git a/whois-rpsl/src/test/java/net/ripe/db/whois/common/rpsl/AttributeParserTestIntegration.java b/whois-rpsl/src/test/java/net/ripe/db/whois/common/rpsl/AttributeParserTestIntegration.java index 75fe709e94..6b775ba24f 100644 --- a/whois-rpsl/src/test/java/net/ripe/db/whois/common/rpsl/AttributeParserTestIntegration.java +++ b/whois-rpsl/src/test/java/net/ripe/db/whois/common/rpsl/AttributeParserTestIntegration.java @@ -23,7 +23,7 @@ public class AttributeParserTestIntegration { @Test public void parseAutnumAttributes() throws Exception { - parseAttributes("/export/opt/ripe.db.aut-num.gz", + parseAttributes("/export/opt/test.db.aut-num.gz", new AttributeType[]{ AttributeType.EXPORT, AttributeType.IMPORT, @@ -38,7 +38,7 @@ public void parseAutnumAttributes() throws Exception { @Test public void parseInetRtrAttributes() throws Exception { - parseAttributes("/export/opt/ripe.db.inet-rtr.gz", + parseAttributes("/export/opt/test.db.inet-rtr.gz", new AttributeType[]{ AttributeType.ALIAS, AttributeType.IFADDR, @@ -51,7 +51,7 @@ public void parseInetRtrAttributes() throws Exception { @Test public void parseAsSetAttributes() throws Exception { - parseAttributes("/export/opt/ripe.db.as-set.gz", + parseAttributes("/export/opt/test.db.as-set.gz", new AttributeType[]{ AttributeType.MEMBERS } @@ -60,7 +60,7 @@ public void parseAsSetAttributes() throws Exception { @Test public void parseRouteSetAttributes() throws Exception { - parseAttributes("/export/opt/ripe.db.route-set.gz", + parseAttributes("/export/opt/test.db.route-set.gz", new AttributeType[]{ AttributeType.MEMBERS, AttributeType.MP_MEMBERS @@ -70,7 +70,7 @@ public void parseRouteSetAttributes() throws Exception { @Test public void parseRtrSetAttributes() throws Exception { - parseAttributes("/export/opt/ripe.db.rtr-set.gz", + parseAttributes("/export/opt/test.db.rtr-set.gz", new AttributeType[]{ AttributeType.MEMBERS, AttributeType.MP_MEMBERS @@ -80,7 +80,7 @@ public void parseRtrSetAttributes() throws Exception { @Test public void parseFilterSetAttributes() throws Exception { - parseAttributes("/export/opt/ripe.db.filter-set.gz", + parseAttributes("/export/opt/test.db.filter-set.gz", new AttributeType[]{ AttributeType.FILTER, AttributeType.MP_FILTER @@ -90,7 +90,7 @@ public void parseFilterSetAttributes() throws Exception { @Test public void parsePeeringSetAttributes() throws Exception { - parseAttributes("/export/opt/ripe.db.peering-set.gz", + parseAttributes("/export/opt/test.db.peering-set.gz", new AttributeType[]{ AttributeType.PEERING, AttributeType.MP_PEERING @@ -100,7 +100,7 @@ public void parsePeeringSetAttributes() throws Exception { @Test public void parseRouteAttributes() throws Exception { - parseAttributes("/export/opt/ripe.db.route.gz", + parseAttributes("/export/opt/test.db.route.gz", new AttributeType[]{ AttributeType.INJECT, AttributeType.AGGR_MTD, @@ -114,7 +114,7 @@ public void parseRouteAttributes() throws Exception { @Test public void parseRoute6Attributes() throws Exception { - parseAttributes("/export/opt/ripe.db.route6.gz", + parseAttributes("/export/opt/test.db.route6.gz", new AttributeType[]{ AttributeType.INJECT, AttributeType.AGGR_MTD, @@ -128,7 +128,7 @@ public void parseRoute6Attributes() throws Exception { @Test public void parseInetnumAttributes() throws Exception { - parseAttributes("/export/opt/ripe.db.inetnum.gz", + parseAttributes("/export/opt/test.db.inetnum.gz", new AttributeType[]{ AttributeType.MNT_ROUTES } @@ -137,7 +137,7 @@ public void parseInetnumAttributes() throws Exception { @Test public void parseInet6numAttributes() throws Exception { - parseAttributes("/export/opt/ripe.db.inet6num.gz", + parseAttributes("/export/opt/test.db.inet6num.gz", new AttributeType[]{ AttributeType.MNT_ROUTES } diff --git a/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportDatabaseTestIntegration.java b/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportDatabaseTestIntegration.java index eea86c6fcd..46967e41a6 100644 --- a/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportDatabaseTestIntegration.java +++ b/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportDatabaseTestIntegration.java @@ -100,13 +100,13 @@ public void export() throws IOException { assertThat(exportDir.exists(), is(true)); for (final ObjectType objectType : ObjectType.values()) { - checkFile("dbase/split/test.db." + objectType.getName() + ".gz"); + checkFile("public/split/test.db." + objectType.getName() + ".gz"); checkFile("internal/split/test.db." + objectType.getName() + ".gz"); } - checkFile("dbase/TEST.CURRENTSERIAL", "120"); + checkFile("public/TEST.CURRENTSERIAL", "120"); - checkFile("dbase/test.db.gz", + checkFile("public/test.db.gz", "person: Placeholder Person Object\n", "mntner: DEV-MNT0\n", "mntner: DEV-MNT1\n", @@ -128,9 +128,9 @@ public void export() throws IOException { "remarks: * http://www.ripe.net/whois\n" + "remarks: ****************************\n"); - checkFile("dbase/split/test.db.person.gz", "person: Placeholder Person Object"); + checkFile("public/split/test.db.person.gz", "person: Placeholder Person Object"); - checkFile("dbase/split/test.db.mntner.gz", + checkFile("public/split/test.db.mntner.gz", "mntner: DEV-MNT0\n", "mntner: DEV-MNT1\n", "mntner: DEV-MNT2\n", @@ -140,16 +140,16 @@ public void export() throws IOException { "mntner: DEV-MNT6\n", "mntner: DEV-MNT7\n", "" + - "mntner: DEV-MNT99\n" + - "auth: MD5-PW $1$SaltSalt$DummifiedMD5HashValue. # Real value hidden for security\n" + - "source: TEST\n" + - "remarks: ****************************\n" + - "remarks: * THIS OBJECT IS MODIFIED\n" + - "remarks: * Please note that all data that is generally regarded as personal\n" + - "remarks: * data has been removed from this object.\n" + - "remarks: * To view the original object, please query the RIPE Database at:\n" + - "remarks: * http://www.ripe.net/whois\n" + - "remarks: ****************************\n"); + "mntner: DEV-MNT99\n" + + "auth: MD5-PW $1$SaltSalt$DummifiedMD5HashValue. # Real value hidden for security\n" + + "source: TEST\n" + + "remarks: ****************************\n" + + "remarks: * THIS OBJECT IS MODIFIED\n" + + "remarks: * Please note that all data that is generally regarded as personal\n" + + "remarks: * data has been removed from this object.\n" + + "remarks: * To view the original object, please query the RIPE Database at:\n" + + "remarks: * http://www.ripe.net/whois\n" + + "remarks: ****************************\n"); checkFile("internal/split/test.db.person.gz", "person: Test person 0", @@ -158,9 +158,9 @@ public void export() throws IOException { "person: Test person 3", "person: Test person 4", "" + - "person: Test person 9\n" + - "nic-hdl: PN9-TEST\n" + - "source: TEST"); + "person: Test person 9\n" + + "nic-hdl: PN9-TEST\n" + + "source: TEST"); checkFile("internal/split/test.db.role.gz", "role: Test role 0", @@ -169,9 +169,9 @@ public void export() throws IOException { "role: Test role 3", "role: Test role 4", "" + - "role: Test role 9\n" + - "nic-hdl: ROLE9-TEST\n" + - "source: TEST"); + "role: Test role 9\n" + + "nic-hdl: ROLE9-TEST\n" + + "source: TEST"); checkFile("internal/split/test.db.mntner.gz", "" + @@ -241,19 +241,19 @@ public void export_role_with_abuse_mailbox() throws IOException { assertThat(exportDir.exists(), is(true)); for (final ObjectType objectType : ObjectType.values()) { - checkFile("dbase/split/test.db." + objectType.getName() + ".gz"); + checkFile("public/split/test.db." + objectType.getName() + ".gz"); checkFile("internal/split/test.db." + objectType.getName() + ".gz"); } - checkFile("dbase/split/test.db.person.gz", "person: Placeholder Person Object"); + checkFile("public/split/test.db.person.gz", "person: Placeholder Person Object"); - checkFile("dbase/split/test.db.role.gz", "" + + checkFile("public/split/test.db.role.gz", "" + "role: Abuse role\n" + "nic-hdl: AR1-TEST\n" + "abuse-mailbox: abuse@mailbox.com\n" + "source: TEST"); - checkFile("dbase/split/test.db.organisation.gz", "" + + checkFile("public/split/test.db.organisation.gz", "" + "organisation: ORG1\n" + "abuse-c: AR1-TEST\n" + "source: TEST"); @@ -320,17 +320,17 @@ public void export_mix_of_sources() throws IOException { assertThat(exportDir.exists(), is(true)); for (final ObjectType objectType : ObjectType.values()) { - checkFile("dbase/split/test.db." + objectType.getName() + ".gz"); + checkFile("public/split/test.db." + objectType.getName() + ".gz"); checkFile("internal/split/test.db." + objectType.getName() + ".gz"); if (ExportFileWriterFactory.NONAUTH_OBJECT_TYPES.contains(objectType)) { - checkFile("dbase/split/test-nonauth.db." + objectType.getName() + ".gz"); + checkFile("public/split/test-nonauth.db." + objectType.getName() + ".gz"); checkFile("internal/split/test-nonauth.db." + objectType.getName() + ".gz"); } } - checkFile("dbase/split/test.db.aut-num.gz", "aut-num: AS252"); - checkFile("dbase/split/test-nonauth.db.aut-num.gz", "aut-num: AS251"); - checkFile("dbase/split/test-nonauth.db.as-set.gz", "as-set: AS251:AS-ALL"); + checkFile("public/split/test.db.aut-num.gz", "aut-num: AS252"); + checkFile("public/split/test-nonauth.db.aut-num.gz", "aut-num: AS251"); + checkFile("public/split/test-nonauth.db.as-set.gz", "as-set: AS251:AS-ALL"); } } diff --git a/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactoryTest.java b/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactoryTest.java index 1d1ecb5869..14ad158e01 100644 --- a/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactoryTest.java +++ b/whois-scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactoryTest.java @@ -38,12 +38,12 @@ public class ExportFileWriterFactoryTest { @BeforeEach public void setup() { - subject = new ExportFileWriterFactory(dummifierNrtm, "internal", "dbase", "TEST", "TEST-NONAUTH"); + subject = new ExportFileWriterFactory(dummifierNrtm, "internal", "public", "TEST", "TEST-NONAUTH"); } @Test public void createExportFileWriters_existing_dir() throws IOException { - Files.createDirectories(folder.resolve("dbase")); + Files.createDirectories(folder.resolve("public")); assertThrows(IllegalStateException.class, () -> { subject.createExportFileWriters(folder.toFile(), LAST_SERIAL); @@ -59,8 +59,7 @@ public void createExportFileWriters() { assertThat(files, not(nullValue())); assertThat(files.length, is(2)); - assertThat(Arrays.stream(files).map(File::getAbsolutePath).toList(), containsInAnyOrder(endsWith("internal"), - endsWith("dbase"))); + assertThat(Arrays.stream(files).map(File::getAbsolutePath).toList(), containsInAnyOrder(endsWith("internal"), endsWith("public"))); } @@ -89,7 +88,7 @@ public void isExportDir_created() { public void isLastSerialFile_created() throws IOException { subject.createExportFileWriters(folder.toFile(), LAST_SERIAL); - final File currentSerialFile = new File(folder.toFile(), "dbase/TEST.CURRENTSERIAL"); + final File currentSerialFile = new File(folder.toFile(), "public/TEST.CURRENTSERIAL"); assertThat(currentSerialFile.exists(), is(true)); final String savedSerial = new String(FileCopyUtils.copyToByteArray(currentSerialFile), StandardCharsets.ISO_8859_1);