Skip to content

Commit

Permalink
feat: merge
Browse files Browse the repository at this point in the history
  • Loading branch information
@MiguelAHM
MiguelAHM committed Dec 17, 2024
2 parents 7e6d656 + 61bf6ed commit 0de852e
Show file tree
Hide file tree
Showing 7 changed files with 119 additions and 60 deletions.
82 changes: 82 additions & 0 deletions ...api/src/test/java/net/ripe/db/whois/api/nrtmv4/SnapshotFileGenerationTestIntegration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,88 @@ public void should_get_snapshot_file_test_source() throws IOException, JSONExcep
"ORG-TEST1-TEST"));
}

@Test
public void should_get_placeholder_when_role_abuse_object() throws IOException, JSONException {

databaseHelper.addObject("" +
"role: Miguel Herran\n" +
"nic-hdl: MH-RIPE\n" +
"abuse-mailbox: [email protected]\n" +
"e-mail: [email protected]\n" +
"created: 2024-12-16T17:00:00Z\n" +
"last-modified: 2024-12-16T17:00:00Z\n" +
"source: TEST");

databaseHelper.addObject("" +
"role: Ed Shryane\n" +
"nic-hdl: ES-RIPE\n" +
"abuse-mailbox: [email protected]\n" +
"e-mail: [email protected]\n" +
"created: 2024-12-16T17:00:00Z\n" +
"last-modified: 2024-12-16T17:00:00Z\n" +
"source: TEST");

snapshotFileGenerator.createSnapshot();
updateNotificationFileGenerator.generateFile();

final Response response = getSnapshotFromUpdateNotificationBySource("TEST");

assertThat(response.getStatus(), is(200));
assertThat(response.getHeaderString(HttpHeaders.CACHE_CONTROL), is("public, max-age=604800"));

final String[] records = getSnapshotRecords(response.readEntity(byte[].class));
assertNrtmFileInfo(records[0], "snapshot", 1, "TEST");

final List<SnapshotFileRecord> snapshotRecords = getSnapshotRecords(records);

assertThat(snapshotRecords.size(), is(10));
assertThat(snapshotRecords.stream().map( record -> record.getObject().getKey().toString()).collect(Collectors.toSet()), containsInAnyOrder("::/0",
"0.0.0.0 - 255.255.255.255",
"AS100 - AS200",
"AS102",
"31.12.202.in-addr.arpa",
"OWNER-MNT",
"ORG-TEST1-TEST",
"ES-RIPE",
"MH-RIPE",
"DUMY-RIPE"));
}


@Test
public void should_not_get_placeholder_when_role_not_abuse_object() throws IOException, JSONException {

databaseHelper.addObject("" +
"role: Miguel Herran\n" +
"nic-hdl: MH-RIPE\n" +
"e-mail: [email protected]\n" +
"created: 2024-12-16T17:00:00Z\n" +
"last-modified: 2024-12-16T17:00:00Z\n" +
"source: TEST");

snapshotFileGenerator.createSnapshot();
updateNotificationFileGenerator.generateFile();

final Response response = getSnapshotFromUpdateNotificationBySource("TEST");

assertThat(response.getStatus(), is(200));
assertThat(response.getHeaderString(HttpHeaders.CACHE_CONTROL), is("public, max-age=604800"));

final String[] records = getSnapshotRecords(response.readEntity(byte[].class));
assertNrtmFileInfo(records[0], "snapshot", 1, "TEST");

final List<SnapshotFileRecord> snapshotRecords = getSnapshotRecords(records);

assertThat(snapshotRecords.size(), is(7));
assertThat(snapshotRecords.stream().map( record -> record.getObject().getKey().toString()).collect(Collectors.toSet()), containsInAnyOrder("::/0",
"0.0.0.0 - 255.255.255.255",
"AS100 - AS200",
"AS102",
"31.12.202.in-addr.arpa",
"OWNER-MNT",
"ORG-TEST1-TEST"));
}

@Test
public void should_get_snapshot_file_test_non_auth_source() throws IOException, JSONException {
snapshotFileGenerator.createSnapshot();
Expand Down
41 changes: 22 additions & 19 deletions whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/DummifierNrtm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,25 +215,28 @@ private static List<RpslAttribute> getDummificationRemarks(final RpslObject rpsl
}

public static RpslObject getPlaceholderPersonObject() {
return RpslObject.parse("" +
"person: Placeholder Person Object\n" +
"address: RIPE Network Coordination Centre\n" +
"address: P.O. Box 10096\n" +
"address: 1001 EB Amsterdam\n" +
"address: The Netherlands\n" +
"phone: +31 20 535 4444\n" +
"nic-hdl: DUMY-RIPE\n" +
"mnt-by: RIPE-DBM-MNT\n" +
"remarks: **********************************************************\n" +
"remarks: * This is a placeholder object to protect personal data.\n" +
"remarks: * To view the original object, please query the RIPE\n" +
"remarks: * Database at:\n" +
"remarks: * http://www.ripe.net/whois\n" +
"remarks: **********************************************************\n" +
"created: 2009-07-24T17:00:00Z\n" +
"last-modified: 2009-07-24T17:00:00Z\n" +
"source: RIPE"
);
return getPlaceholderPersonObject(CIString.ciString("RIPE"));
}

public static RpslObject getPlaceholderPersonObject(final CIString source){
return RpslObject.parse(String.format("""
person: Placeholder Person Object
address: RIPE Network Coordination Centre
address: P.O. Box 10096
address: 1001 EB Amsterdam
address: The Netherlands
phone: +31 20 535 4444
nic-hdl: DUMY-RIPE
mnt-by: RIPE-DBM-MNT
remarks: **********************************************************
remarks: * This is a placeholder object to protect personal data.
remarks: * To view the original object, please query the RIPE
remarks: * Database at:
remarks: * http://www.ripe.net/whois
remarks: **********************************************************
created: 2009-07-24T17:00:00Z
last-modified: 2009-07-24T17:00:00Z
source: %s""", source));
}

public static RpslObject getPlaceholderRoleObject() {
Expand Down
7 changes: 7 additions & 0 deletions whois-commons/src/main/java/net/ripe/db/whois/common/rpsl/DummifierNrtmV4.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,20 @@ public class DummifierNrtmV4 extends DummifierNrtm {
DUMMIFICATION_REPLACEMENTS.put(AttributeType.REMARKS, "Dummified");
}

private final Set<ObjectType> writtenPlaceHolders = Sets.newHashSet();

public RpslObject dummify(final RpslObject rpslObject) {
final List<RpslAttribute> attributes = Lists.newArrayList(rpslObject.getAttributes());
dummifyAttributes(attributes, rpslObject.getKey());

return dummify(4, new RpslObject(rpslObject, attributes));
}

public boolean shouldCreatePlaceHolder(final RpslObject rpslObject) {
final ObjectType objectType = rpslObject.getType();
return writtenPlaceHolders.add(objectType) && (objectType.equals(ObjectType.ROLE) || objectType.equals(ObjectType.PERSON));
}

public boolean isAllowed(final RpslObject rpslObject) {
return isAllowed(4, rpslObject);
}
Expand Down
42 changes: 2 additions & 40 deletions ...-nrtm4-client/src/main/java/net/ripe/db/nrtm4/client/importer/SnapshotMirrorImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,14 +89,14 @@ public void doImport(final String source,
final AtomicInteger snapshotVersion = new AtomicInteger(0);
final AtomicInteger processedCount = new AtomicInteger(0);

persisSnapshot(source, payload, sessionId, snapshotVersion, processedCount);
persisSnapshot(payload, sessionId, snapshotVersion, processedCount);
persistSnapshotVersion(source, snapshotVersion.get(), sessionId);

stopwatch.stop();
LOGGER.info("Loading snapshot file took {} for source {} and added {} records", stopwatch, source, processedCount.get());
}

private void persisSnapshot(final String source, final byte[] payload, final String sessionId,
private void persisSnapshot(final byte[] payload, final String sessionId,
final AtomicInteger snapshotVersion, final AtomicInteger processedCount){
final Timer timer = new Timer();
printProgress(timer, processedCount);
Expand All @@ -115,30 +115,13 @@ private void persisSnapshot(final String source, final byte[] payload, final Str
throw new IllegalStateException(ex);
}

persistDummyObjectIfNotExist(source);
timer.cancel();
}


final void persistSnapshotVersion(final String source, final int version, final String sessionId) throws IllegalArgumentException {
nrtm4ClientInfoRepository.saveSnapshotFileVersion(source, version, sessionId);
}

@Transactional(transactionManager = NrtmClientTransactionConfiguration.NRTM_CLIENT_UPDATE_TRANSACTION)
private void persistDummyObjectIfNotExist(final String source){
final RpslObject dummyObject = getPlaceholderPersonObject();
if (!source.equals(dummyObject.getValueForAttribute(AttributeType.SOURCE).toString())){
return;
}

final RpslObjectUpdateInfo rpslObjectUpdateInfo = nrtm4ClientRepository.getMirroredObjectId(dummyObject.getType(), dummyObject.getKey().toString());
if (rpslObjectUpdateInfo != null){
return;
}
final RpslObjectUpdateInfo createdDummy = nrtm4ClientRepository.persistRpslObject(dummyObject);
nrtm4ClientRepository.createIndexes(dummyObject, createdDummy);
}

private void printProgress(final Timer timer, final AtomicInteger processedCount) {
timer.schedule(new TimerTask() {
@Override
Expand Down Expand Up @@ -216,27 +199,6 @@ private Future<?> processRecord(final ExecutorService executor, final AtomicInte
});
}

public static RpslObject getPlaceholderPersonObject() {
return RpslObject.parse("" +
"person: Placeholder Person Object\n" +
"address: RIPE Network Coordination Centre\n" +
"address: P.O. Box 10096\n" +
"address: 1001 EB Amsterdam\n" +
"address: The Netherlands\n" +
"phone: +31 20 535 4444\n" +
"nic-hdl: DUMY-RIPE\n" +
"mnt-by: RIPE-DBM-MNT\n" +
"remarks: **********************************************************\n" +
"remarks: * This is a placeholder object to protect personal data.\n" +
"remarks: * To view the original object, please query the RIPE\n" +
"remarks: * Database at:\n" +
"remarks: * http://www.ripe.net/whois\n" +
"remarks: **********************************************************\n" +
"created: 2009-07-24T17:00:00Z\n" +
"last-modified: 2009-07-24T17:00:00Z\n" +
"source: RIPE"
);
}

private void validateSession(final String updateNotificationSessionId, final String firstRecord) throws IllegalArgumentException{
final JSONObject jsonObject = new JSONObject(firstRecord);
Expand Down
2 changes: 1 addition & 1 deletion whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/NrtmServerDummy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public class NrtmServerDummy implements Stub {

private final static String RIPE_NONAUTH_SNAP_HASH = "148c3c411b8f044f5fc0ab201f6dd03e80c862e27ad1a63488aee337dc7eb4a2";

private final static String RIPE_SNAP_HASH = "8aeb6b6c93fc51aeb868de2ebca3eed6c98559b438901bb80ddc24c47c8069dd";
private final static String RIPE_SNAP_HASH = "7c9d1a1ebc73dc719e11c1046fae6598c35ae507a391d142beebe33865f077a0";

private final static Map<String, String> RIPE_NONAUTH_DELTA_HASH = ImmutableMap.of(
"1", "d671895977aa9dd8ca2b7ba74cd96cd28a786b0e19b3db709c9c5a02a52466ef",
Expand Down
1 change: 1 addition & 0 deletions whois-nrtm4-client/src/test/resources/mock/nrtm-snapshot.1.RIPE.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@
{"object":"inet6num: 2a00:2381:c3be::/48\nnetname: A-J\ndescr: Dummified\ncountry: GB\nadmin-c: DUMY-RIPE\ntech-c: DUMY-RIPE\nstatus: AGGREGATED-BY-LIR\nassignment-size:56\nmnt-by: BTNET-MNT\nmnt-lower: BTNET-MNT\nmnt-routes: BTNET-MNT\nremarks: Dummified\ncreated: 2018-11-23T14:05:34Z\nlast-modified: 2018-11-23T14:05:34Z\nsource: RIPE\nremarks: ****************************\nremarks: * THIS OBJECT IS MODIFIED\nremarks: * Please note that all data that is generally regarded as personal\nremarks: * data has been removed from this object.\nremarks: * To view the original object, please query the RIPE Database at:\nremarks: * http://www.ripe.net/whois\nremarks: ****************************\n"}
{"object":"route: 176.240.50.0/24\ndescr: Dummified\norigin: AS47524\nmnt-by: MNT-AS47524\ncreated: 2020-04-06T11:26:00Z\nlast-modified: 2020-04-06T11:26:00Z\nsource: RIPE\nremarks: ****************************\nremarks: * THIS OBJECT IS MODIFIED\nremarks: * Please note that all data that is generally regarded as personal\nremarks: * data has been removed from this object.\nremarks: * To view the original object, please query the RIPE Database at:\nremarks: * http://www.ripe.net/whois\nremarks: ****************************\n"}
{"object":"aut-num: AS211871\nas-name: OWEB\norg: ORG-OBTA3-RIPE\nimport: from AS15924 accept ANY\nexport: to AS15924 announce AS211871\nimport: from AS9121 accept ANY\nexport: to AS9121 announce AS211871\nimport: from AS51559 accept ANY\nexport: to AS51559 announce AS211871\nimport: from AS396998 accept ANY\nexport: to AS396998 announce AS211871\nimport: from AS198351 accept ANY\nexport: to AS198351 announce AS211871\nsponsoring-org: ORG-OBTS2-RIPE\nadmin-c: DUMY-RIPE\ntech-c: DUMY-RIPE\nstatus: ASSIGNED\nmnt-by: RIPE-NCC-END-MNT\nmnt-by: OWEB-MNT\ncreated: 2021-02-04T13:49:19Z\nlast-modified: 2024-07-30T07:53:29Z\nsource: RIPE\nremarks: ****************************\nremarks: * THIS OBJECT IS MODIFIED\nremarks: * Please note that all data that is generally regarded as personal\nremarks: * data has been removed from this object.\nremarks: * To view the original object, please query the RIPE Database at:\nremarks: * http://www.ripe.net/whois\nremarks: ****************************\n"}
{"object":"person: Placeholder Person Object\naddress: RIPE Network Coordination Centre\nnic-hdl: DUMY-RIPE\nmnt-by: RIPE-DBM-MNT\ncreated: 2021-02-04T13:49:19Z\nlast-modified: 2024-07-30T07:53:29Z\nsource: RIPE\nremarks: ****************************\nremarks: * THIS OBJECT IS MODIFIED\nremarks: * Please note that all data that is generally regarded as personal\nremarks: * data has been removed from this object.\nremarks: * To view the original object, please query the RIPE Database at:\nremarks: * http://www.ripe.net/whois\nremarks: ****************************\n"}
4 changes: 4 additions & 0 deletions whois-nrtm4/src/main/java/net/ripe/db/nrtm4/generator/SnapshotFileGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import net.ripe.db.whois.common.DateTimeProvider;
import net.ripe.db.whois.common.domain.CIString;
import net.ripe.db.whois.common.rpsl.AttributeType;
import net.ripe.db.whois.common.rpsl.DummifierNrtm;
import net.ripe.db.whois.common.rpsl.DummifierNrtmV4;
import net.ripe.db.whois.common.rpsl.RpslObject;
import org.slf4j.Logger;
Expand Down Expand Up @@ -110,6 +111,9 @@ private Map<CIString, byte[]> writeToGzipStream(final SnapshotState snapshotStat
try {
final RpslObject rpslObject = RpslObject.parse(object);
if (dummifierNrtmV4.isAllowed(rpslObject)) {
if (dummifierNrtmV4.shouldCreatePlaceHolder(rpslObject)){
rpslObjects.add(DummifierNrtm.getPlaceholderPersonObject(rpslObject.getValueForAttribute(AttributeType.SOURCE)));
}
rpslObjects.add(dummifierNrtmV4.dummify(rpslObject));
}
} catch (final Exception e) {
Expand Down

0 comments on commit 0de852e

Please sign in to comment.