Skip to content

Commit

Permalink
Include AS-SET in NONAUTH dump (#1352)
Browse files Browse the repository at this point in the history 
* Include AS-SET in NONAUTH dump.

* Added tests

* Define NONAUTH object types once in export code. Don't hard-code source.

* Fixed test
  • Loading branch information
@eshryane
eshryane authored Nov 17, 2023
1 parent 6895a43 commit 3c99142
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 68 deletions.
22 changes: 13 additions & 9 deletions ...eduler/src/main/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,20 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Set;

import static net.ripe.db.whois.common.rpsl.ObjectType.AS_SET;
import static net.ripe.db.whois.common.rpsl.ObjectType.AUT_NUM;
import static net.ripe.db.whois.common.rpsl.ObjectType.ROUTE;
import static net.ripe.db.whois.common.rpsl.ObjectType.ROUTE6;

@Component
class ExportFileWriterFactory {

protected static final Set<ObjectType> NONAUTH_OBJECT_TYPES = Sets.immutableEnumSet(AS_SET, AUT_NUM, ROUTE, ROUTE6);

private static final String SPLITFILE_FOLDERNAME = "split";
private static final String CURRENTSERIAL_FILENAME = "RIPE.CURRENTSERIAL";
private static final String CURRENTSERIAL_NONAUTH_FILENAME = "RIPE-NONAUTH.CURRENTSERIAL";
private static final String CURRENTSERIAL_SUFFIX = "CURRENTSERIAL";

private final DummifierNrtm dummifierNrtm;
private final String externalExportDir;
Expand Down Expand Up @@ -52,20 +56,20 @@ public List<ExportFileWriter> createExportFileWriters(final File baseDir, final
initDirs(fullDir, splitDir, internalDir);

try {
FileCopyUtils.copy(String.valueOf(lastSerial).getBytes(StandardCharsets.ISO_8859_1), new File(fullDir, CURRENTSERIAL_FILENAME));
FileCopyUtils.copy(String.valueOf(lastSerial).getBytes(StandardCharsets.ISO_8859_1), new File(fullDir, CURRENTSERIAL_NONAUTH_FILENAME));
FileCopyUtils.copy(String.valueOf(lastSerial).getBytes(StandardCharsets.ISO_8859_1), new File(fullDir, String.format("%s.%s", source, CURRENTSERIAL_SUFFIX)));
FileCopyUtils.copy(String.valueOf(lastSerial).getBytes(StandardCharsets.ISO_8859_1), new File(fullDir, String.format("%s.%s", nonAuthSource, CURRENTSERIAL_SUFFIX)));
} catch (IOException e) {
throw new RuntimeException("Writing current serial", e);
}

final FilenameStrategy singleFile = new FilenameStrategy.SingleFile();
final FilenameStrategy splitFile = new FilenameStrategy.SplitFile();
final FilenameStrategy singleFile = new FilenameStrategy.SingleFile(source);
final FilenameStrategy splitFile = new FilenameStrategy.SplitFile(source);

final FilenameStrategy nonAuthSingleFile = new FilenameStrategy.NonAuthSingleFile();
final FilenameStrategy nonAuthSplitFile = new FilenameStrategy.NonAuthSplitFile();
final FilenameStrategy nonAuthSingleFile = new FilenameStrategy.NonAuthSingleFile(nonAuthSource);
final FilenameStrategy nonAuthSplitFile = new FilenameStrategy.NonAuthSplitFile(nonAuthSource, NONAUTH_OBJECT_TYPES);

final ExportFilter sourceFilter = new ExportFilter.SourceExportFilter(source, ImmutableSet.copyOf(ObjectType.values()));
final ExportFilter nonAuthSourceFilter = new ExportFilter.SourceExportFilter(nonAuthSource, Sets.immutableEnumSet(AUT_NUM, ROUTE, ROUTE6), false);
final ExportFilter nonAuthSourceFilter = new ExportFilter.SourceExportFilter(nonAuthSource, NONAUTH_OBJECT_TYPES, false);

return Lists.newArrayList(
new ExportFileWriter(fullDir, singleFile, new DecorationStrategy.DummifySplitFiles(dummifierNrtm), sourceFilter),
Expand Down
59 changes: 45 additions & 14 deletions whois-scheduler/src/main/java/net/ripe/db/whois/scheduler/task/export/FilenameStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,41 +2,72 @@

import net.ripe.db.whois.common.rpsl.ObjectType;

import static net.ripe.db.whois.common.rpsl.ObjectType.AUT_NUM;
import static net.ripe.db.whois.common.rpsl.ObjectType.ROUTE;
import static net.ripe.db.whois.common.rpsl.ObjectType.ROUTE6;
import java.util.Set;

public interface FilenameStrategy {
public abstract class FilenameStrategy {

final String source;

protected FilenameStrategy(final String source) {
this.source = source.toLowerCase();
}

protected String getSource() {
return this.source;
}

static class SplitFile extends FilenameStrategy {

public SplitFile(final String source) {
super(source);
}

class SplitFile implements FilenameStrategy {
@Override
public String getFilename(final ObjectType objectType) {
return "ripe.db." + objectType.getName();
return String.format("%s.db.%s", getSource(), objectType.getName());
}
}

class SingleFile implements FilenameStrategy {
static class SingleFile extends FilenameStrategy {

public SingleFile(final String source) {
super(source);
}

@Override
public String getFilename(final ObjectType objectType) {
return "ripe.db";
return String.format("%s.db", getSource());
}
}

class NonAuthSingleFile implements FilenameStrategy {
static class NonAuthSingleFile extends FilenameStrategy {

public NonAuthSingleFile(final String source) {
super(source);
}

@Override
public String getFilename(final ObjectType objectType) {
return "ripe-nonauth.db";
return String.format("%s.db", getSource());
}
}

class NonAuthSplitFile implements FilenameStrategy {
static class NonAuthSplitFile extends FilenameStrategy {

final Set<ObjectType> nonAuthObjectTypes;

public NonAuthSplitFile(final String source, final Set<ObjectType> nonAuthObjectTypes) {
super(source);
this.nonAuthObjectTypes = nonAuthObjectTypes;
}

@Override
public String getFilename(final ObjectType objectType) {
return objectType == AUT_NUM || objectType == ROUTE || objectType == ROUTE6?
"ripe-nonauth.db." + objectType.getName() :
return (nonAuthObjectTypes.contains(objectType)) ?
String.format("%s.db.%s", getSource(), objectType.getName()) :
null;
}
}

String getFilename(ObjectType objectType);
abstract String getFilename(ObjectType objectType);
}
76 changes: 37 additions & 39 deletions .../src/test/java/net/ripe/db/whois/scheduler/task/export/ExportDatabaseTestIntegration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,6 @@ public void setTmpDir(final String tmpDirNAme) {

Set<RpslObject> objects;

private final static Set<ObjectType> NON_AUTH_TYPES = Sets.immutableEnumSet(ObjectType.AUT_NUM, ObjectType.ROUTE, ObjectType.ROUTE6);

@BeforeEach
public void setupServer() {
objects = Sets.newHashSet();
Expand All @@ -69,15 +67,15 @@ public void setupServer() {
for (int i = 0; i < 10; i++) {
final RpslObject personObject = RpslObject.parse("" +
"person: Test person " + i + "\n" +
"nic-hdl: PN" + i + "-RIPE\n" +
"nic-hdl: PN" + i + "-TEST\n" +
"source: TEST");

databaseHelper.addObject(personObject);
objects.add(personObject);

final RpslObject roleObject = RpslObject.parse("" +
"role: Test role " + i + "\n" +
"nic-hdl: ROLE" + i + "-RIPE\n" +
"nic-hdl: ROLE" + i + "-TEST\n" +
"source: TEST");

databaseHelper.addObject(roleObject);
Expand All @@ -102,13 +100,13 @@ public void export() throws IOException {
assertThat(exportDir.exists(), is(true));

for (final ObjectType objectType : ObjectType.values()) {
checkFile("dbase/split/ripe.db." + objectType.getName() + ".gz");
checkFile("internal/split/ripe.db." + objectType.getName() + ".gz");
checkFile("dbase/split/test.db." + objectType.getName() + ".gz");
checkFile("internal/split/test.db." + objectType.getName() + ".gz");
}

checkFile("dbase/RIPE.CURRENTSERIAL", "120");
checkFile("dbase/TEST.CURRENTSERIAL", "120");

checkFile("dbase/ripe.db.gz",
checkFile("dbase/test.db.gz",
"person: Placeholder Person Object\n",
"mntner: DEV-MNT0\n",
"mntner: DEV-MNT1\n",
Expand All @@ -130,9 +128,9 @@ public void export() throws IOException {
"remarks: * http://www.ripe.net/whois\n" +
"remarks: ****************************\n");

checkFile("dbase/split/ripe.db.person.gz", "person: Placeholder Person Object");
checkFile("dbase/split/test.db.person.gz", "person: Placeholder Person Object");

checkFile("dbase/split/ripe.db.mntner.gz",
checkFile("dbase/split/test.db.mntner.gz",
"mntner: DEV-MNT0\n",
"mntner: DEV-MNT1\n",
"mntner: DEV-MNT2\n",
Expand All @@ -153,29 +151,29 @@ public void export() throws IOException {
"remarks: * http://www.ripe.net/whois\n" +
"remarks: ****************************\n");

checkFile("internal/split/ripe.db.person.gz",
checkFile("internal/split/test.db.person.gz",
"person: Test person 0",
"person: Test person 1",
"person: Test person 2",
"person: Test person 3",
"person: Test person 4",
"" +
"person: Test person 9\n" +
"nic-hdl: PN9-RIPE\n" +
"nic-hdl: PN9-TEST\n" +
"source: TEST");

checkFile("internal/split/ripe.db.role.gz",
checkFile("internal/split/test.db.role.gz",
"role: Test role 0",
"role: Test role 1",
"role: Test role 2",
"role: Test role 3",
"role: Test role 4",
"" +
"role: Test role 9\n" +
"nic-hdl: ROLE9-RIPE\n" +
"nic-hdl: ROLE9-TEST\n" +
"source: TEST");

checkFile("internal/split/ripe.db.mntner.gz",
checkFile("internal/split/test.db.mntner.gz",
"" +
"mntner: DEV-MNT0\n" +
"auth: MD5-PW $1$xNv6umMG$cBd9DXqWEpsqeBq2AUjGy/\n" +
Expand Down Expand Up @@ -226,13 +224,13 @@ public void export() throws IOException {
public void export_role_with_abuse_mailbox() throws IOException {
databaseHelper.addObject(RpslObject.parse("" +
"role: Abuse role\n" +
"nic-hdl: AR1-RIPE\n" +
"nic-hdl: AR1-TEST\n" +
"abuse-mailbox: [email protected]\n" +
"source: TEST"));

databaseHelper.addObject(RpslObject.parse("" +
"organisation: ORG1\n" +
"abuse-c: AR1-RIPE\n" +
"abuse-c: AR1-TEST\n" +
"source: TEST"));

sourceContext.removeCurrentSource();
Expand All @@ -243,32 +241,32 @@ public void export_role_with_abuse_mailbox() throws IOException {
assertThat(exportDir.exists(), is(true));

for (final ObjectType objectType : ObjectType.values()) {
checkFile("dbase/split/ripe.db." + objectType.getName() + ".gz");
checkFile("internal/split/ripe.db." + objectType.getName() + ".gz");
checkFile("dbase/split/test.db." + objectType.getName() + ".gz");
checkFile("internal/split/test.db." + objectType.getName() + ".gz");
}

checkFile("dbase/split/ripe.db.person.gz", "person: Placeholder Person Object");
checkFile("dbase/split/test.db.person.gz", "person: Placeholder Person Object");

checkFile("dbase/split/ripe.db.role.gz", "" +
checkFile("dbase/split/test.db.role.gz", "" +
"role: Abuse role\n" +
"nic-hdl: AR1-RIPE\n" +
"nic-hdl: AR1-TEST\n" +
"abuse-mailbox: [email protected]\n" +
"source: TEST");

checkFile("dbase/split/ripe.db.organisation.gz", "" +
checkFile("dbase/split/test.db.organisation.gz", "" +
"organisation: ORG1\n" +
"abuse-c: AR1-RIPE\n" +
"abuse-c: AR1-TEST\n" +
"source: TEST");

checkFile("internal/split/ripe.db.role.gz", "" +
checkFile("internal/split/test.db.role.gz", "" +
"role: Abuse role\n" +
"nic-hdl: AR1-RIPE\n" +
"nic-hdl: AR1-TEST\n" +
"abuse-mailbox: [email protected]\n" +
"source: TEST");

checkFile("internal/split/ripe.db.organisation.gz", "" +
checkFile("internal/split/test.db.organisation.gz", "" +
"organisation: ORG1\n" +
"abuse-c: AR1-RIPE\n" +
"abuse-c: AR1-TEST\n" +
"source: TEST");
}

Expand Down Expand Up @@ -308,31 +306,31 @@ public void export_mix_of_sources() throws IOException {
databaseHelper.addObject(RpslObject.parse("" +
"aut-num: AS252\n" +
"source: TEST"));

databaseHelper.addObject(RpslObject.parse("" +
"aut-num: AS251\n" +
"source: TEST-NONAUTH"));
databaseHelper.addObject(RpslObject.parse("" +
"as-set: AS251:AS-ALL\n" +
"source: TEST-NONAUTH"));

sourceContext.removeCurrentSource();

rpslObjectsExporter.export();

assertThat(tmpDir.exists(), is(false));
assertThat(exportDir.exists(), is(true));

for (final ObjectType objectType : ObjectType.values()) {
checkFile("dbase/split/ripe.db." + objectType.getName() + ".gz");
checkFile("internal/split/ripe.db." + objectType.getName() + ".gz");
if (NON_AUTH_TYPES.contains(objectType)) {
checkFile("dbase/split/ripe-nonauth.db." + objectType.getName() + ".gz");
checkFile("internal/split/ripe-nonauth.db." + objectType.getName() + ".gz");
checkFile("dbase/split/test.db." + objectType.getName() + ".gz");
checkFile("internal/split/test.db." + objectType.getName() + ".gz");
if (ExportFileWriterFactory.NONAUTH_OBJECT_TYPES.contains(objectType)) {
checkFile("dbase/split/test-nonauth.db." + objectType.getName() + ".gz");
checkFile("internal/split/test-nonauth.db." + objectType.getName() + ".gz");
}
}



checkFile("dbase/split/ripe.db.aut-num.gz", "aut-num: AS252");
checkFile("dbase/split/ripe-nonauth.db.aut-num.gz", "aut-num: AS251");
checkFile("dbase/split/test.db.aut-num.gz", "aut-num: AS252");
checkFile("dbase/split/test-nonauth.db.aut-num.gz", "aut-num: AS251");
checkFile("dbase/split/test-nonauth.db.as-set.gz", "as-set: AS251:AS-ALL");
}

}
4 changes: 2 additions & 2 deletions ...er/src/test/java/net/ripe/db/whois/scheduler/task/export/ExportFileWriterFactoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public class ExportFileWriterFactoryTest {

@BeforeEach
public void setup() {
subject = new ExportFileWriterFactory(dummifierNrtm, "internal", "dbase", "test", "test-nonauth");
subject = new ExportFileWriterFactory(dummifierNrtm, "internal", "dbase", "TEST", "TEST-NONAUTH");
}

@Test
Expand Down Expand Up @@ -89,7 +89,7 @@ public void isExportDir_created() {
public void isLastSerialFile_created() throws IOException {
subject.createExportFileWriters(folder.toFile(), LAST_SERIAL);

final File currentSerialFile = new File(folder.toFile(), "dbase/RIPE.CURRENTSERIAL");
final File currentSerialFile = new File(folder.toFile(), "dbase/TEST.CURRENTSERIAL");
assertThat(currentSerialFile.exists(), is(true));

final String savedSerial = new String(FileCopyUtils.copyToByteArray(currentSerialFile), StandardCharsets.ISO_8859_1);
Expand Down
8 changes: 4 additions & 4 deletions ...scheduler/src/test/java/net/ripe/db/whois/scheduler/task/export/FilenameStrategyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@
public class FilenameStrategyTest {
@Test
public void getFilename_SingleFile() {
final FilenameStrategy subject = new FilenameStrategy.SingleFile();
final FilenameStrategy subject = new FilenameStrategy.SingleFile("TEST");

assertThat(subject.getFilename(ObjectType.MNTNER), is("ripe.db"));
assertThat(subject.getFilename(ObjectType.MNTNER), is("test.db"));
}

@Test
public void getFilename_SplitFile() {
final FilenameStrategy subject = new FilenameStrategy.SplitFile();
final FilenameStrategy subject = new FilenameStrategy.SplitFile("TEST");

assertThat(subject.getFilename(ObjectType.MNTNER), is("ripe.db.mntner"));
assertThat(subject.getFilename(ObjectType.MNTNER), is("test.db.mntner"));
}
}

0 comments on commit 3c99142

Please sign in to comment.