From f9c50defc1a35a279f1eb5bb4a7412320c67bceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Herr=C3=A1n?= <106821523+MiguelAHM@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:03:15 +0100 Subject: [PATCH] NRTM Client Integration Test Set Up (#1574) * feat: nrtmv4 client setup * feat: add whois-nrtm4-client everywhere * feat: get common dependencies in whois-nrtm-client (aspectJ in them) * feat: read unf and store main information * feat: fix repository * feat: clarify reader * feat: remove unnecesary value injection * feat: add schema creation changes * feat: fix schema syntax error * feat: hardcode for now nrtm baseUrl * feat: hardcode RC * feat: fix compilation issues * feat: add logs and fix the name * feat: change properties and put logs in the condition * feat: add logs * feat: fix log * feat: create nrtm rest client * feat: fix typo * feat: run schedule task each minute * feat: add nrtm4 client to api db endtoend modules * feat: add source log * feat: try catch duplicated key exception when the version already exist * feat: refactor * feat: remove the changes of rest client * feat: remove unused import * feat: change nrtm data soure configurations name * feat: use a rowmapper in between sources call and jsonnode * feat: fix compilation issue * feat: add retrn statment * feat: fix get resources * feat: fix baseUrl * feat: remove the slash from url * feat: ignore unknown properties * feat: filter by version and fix the mapper * feat: handle when table is null * feat: refactor for clarifications * feat: fix sql * feat: check not null * feat: add logs * feat: remove logs and add the group by to fix the query * feat: refactor * feat: put the dependency again in whois-api * feat: do no restart if notification is the same * feat: remove initializeNRTMClient module body * feat: create nrtm dummy server * feat: remove unused import and add the conditional * feat: rename it by client * feat: rename the condition * feat: add dependsOn dependency and conditional just for the config * feat: do not use dependsOn use conditional * feat: add UNF client ITs --- ...4_mirror_data.sql => nrtm_client_data.sql} | 0 ...rror_schema.sql => nrtm_client_schema.sql} | 30 ++-- .../whois/common/dao/jdbc/DatabaseHelper.java | 4 + .../src/test/resources/whois.properties | 4 +- .../scheduler/Nrtm4ClientCondition.java | 2 - .../AbstractNrtmClientIntegrationTest.java | 37 +++++ .../ripe/db/nrtm4/client/NrtmServerDummy.java | 138 ++++++++++++++++++ ...NotificationFileReaderTestIntegration.java | 39 +++++ .../applicationContext-nrtm4-client-test.xml | 20 +++ .../resources/mock/nrtm-non-auth-unf.json | 14 ++ .../test/resources/mock/nrtm-ripe-unf.json | 20 +++ .../src/test/resources/mock/nrtm-sources.html | 6 + 12 files changed, 296 insertions(+), 18 deletions(-) rename whois-commons/src/main/resources/{nrtmv4_mirror_data.sql => nrtm_client_data.sql} (100%) rename whois-commons/src/main/resources/{nrtmv4_mirror_schema.sql => nrtm_client_schema.sql} (65%) create mode 100644 whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/AbstractNrtmClientIntegrationTest.java create mode 100644 whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/NrtmServerDummy.java create mode 100644 whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/reader/UpdateNotificationFileReaderTestIntegration.java create mode 100644 whois-nrtm4-client/src/test/resources/applicationContext-nrtm4-client-test.xml create mode 100644 whois-nrtm4-client/src/test/resources/mock/nrtm-non-auth-unf.json create mode 100644 whois-nrtm4-client/src/test/resources/mock/nrtm-ripe-unf.json create mode 100644 whois-nrtm4-client/src/test/resources/mock/nrtm-sources.html diff --git a/whois-commons/src/main/resources/nrtmv4_mirror_data.sql b/whois-commons/src/main/resources/nrtm_client_data.sql similarity index 100% rename from whois-commons/src/main/resources/nrtmv4_mirror_data.sql rename to whois-commons/src/main/resources/nrtm_client_data.sql diff --git a/whois-commons/src/main/resources/nrtmv4_mirror_schema.sql b/whois-commons/src/main/resources/nrtm_client_schema.sql similarity index 65% rename from whois-commons/src/main/resources/nrtmv4_mirror_schema.sql rename to whois-commons/src/main/resources/nrtm_client_schema.sql index 3c3bac2148..f7664f1592 100644 --- a/whois-commons/src/main/resources/nrtmv4_mirror_schema.sql +++ b/whois-commons/src/main/resources/nrtm_client_schema.sql @@ -1,17 +1,20 @@ -/*!40101 SET @OLD_CHARACTER_SET_CLIENT = @@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS = @@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION = @@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8mb4 */; -/*!40103 SET @OLD_TIME_ZONE = @@TIME_ZONE */; -/*!40103 SET TIME_ZONE = '+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS = @@UNIQUE_CHECKS, UNIQUE_CHECKS = 0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS = @@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS = 0 */; -/*!40101 SET @OLD_SQL_MODE = @@SQL_MODE, SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES = @@SQL_NOTES, SQL_NOTES = 0 */; +-- MySQL dump 10.13 Distrib 5.1.61, for redhat-linux-gnu (x86_64) +-- +-- Host: localhost Database: NRTM_CLIENT_RIPE +-- ------------------------------------------------------ +-- Server version 5.1.61-log + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; -DROP DATABASE IF EXISTS NRTMV4_CLIENT_RIPE; -CREATE DATABASE NRTMV4_CLIENT_RIPE; -USE NRTMV4_CLIENT_RIPE; DROP TABLE IF EXISTS `version`; DROP TABLE IF EXISTS `public_key`; @@ -60,7 +63,6 @@ CREATE TABLE `last_mirror` `object` longblob NOT NULL, `pkey` varchar(254) NOT NULL DEFAULT '', PRIMARY KEY (`object_id`,`sequence_id`), - CONSTRAINT `last_mirror__version_id__fk` FOREIGN KEY (`version_id`) REFERENCES `version_info` (`id`), KEY `last_pkey` (`pkey`), KEY `object_type_index` (`object_type`) ) ENGINE=InnoDB diff --git a/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java b/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java index 78300b7aca..93f3a5aec0 100644 --- a/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java +++ b/whois-commons/src/test/java/net/ripe/db/whois/common/dao/jdbc/DatabaseHelper.java @@ -174,6 +174,7 @@ public static synchronized void setupDatabase() { setupDatabase(jdbcTemplate, "whois.db", "WHOIS", "whois_schema.sql", "whois_data.sql"); setupDatabase(jdbcTemplate, "internals.database", "INTERNALS", "internals_schema.sql", "internals_data.sql"); setupDatabase(jdbcTemplate, "nrtm.database", "NRTM", "nrtm_schema.sql", "nrtm_data.sql"); + setupDatabase(jdbcTemplate, "nrtm.client.database", "NRTM_CLIENT", "nrtm_client_schema.sql", "nrtm_client_data.sql"); final String masterUrl = String.format("jdbc:log:mariadb://%s/%s_WHOIS;driver=%s", DB_HOST, dbBaseName, JDBC_DRIVER); System.setProperty("whois.db.master.url", masterUrl); @@ -189,6 +190,9 @@ public static synchronized void setupDatabase() { final String nrtmSlaveUrl = String.format("jdbc:mariadb://%s/%s_NRTM", DB_HOST, dbBaseName); System.setProperty("nrtm.slave.database.url", nrtmSlaveUrl); + final String nrtmClientSlaveUrl = String.format("jdbc:mariadb://%s/%s_NRTM_CLIENT", DB_HOST, dbBaseName); + System.setProperty("nrtm.client.slave.database.url", nrtmClientSlaveUrl); + final String grsSlaveUrl = String.format("jdbc:mariadb://%s/%s", DB_HOST, dbBaseName); System.setProperty("whois.db.grs.slave.baseurl", grsSlaveUrl); System.setProperty("whois.db.grs.master.baseurl", grsSlaveUrl); diff --git a/whois-commons/src/test/resources/whois.properties b/whois-commons/src/test/resources/whois.properties index a087e2c9bc..720e161fc9 100644 --- a/whois-commons/src/test/resources/whois.properties +++ b/whois-commons/src/test/resources/whois.properties @@ -132,11 +132,11 @@ nrtm.slave.database.url=jdbc:mysql://${db.host:localhost}/NRTM_LOCAL nrtm.slave.database.username=dbint nrtm.slave.database.password= -nrtm.client.database.url=jdbc:mysql://${db.host:localhost}/NRTM4_CLIENT_LOCAL +nrtm.client.database.url=jdbc:mysql://${db.host:localhost}/NRTM_CLIENT_LOCAL nrtm.client.database.username=dbint nrtm.client.database.password= -nrtm.client.slave.database.url=jdbc:mysql://${db.host:localhost}/NRTM4_CLIENT_LOCAL +nrtm.client.slave.database.url=jdbc:mysql://${db.host:localhost}/NRTM_CLIENT_LOCAL nrtm.client.slave.database.username=dbint nrtm.client.slave.database.password= diff --git a/whois-nrtm4-client/src/main/java/net/ripe/db/nrtm4/client/scheduler/Nrtm4ClientCondition.java b/whois-nrtm4-client/src/main/java/net/ripe/db/nrtm4/client/scheduler/Nrtm4ClientCondition.java index d9beb5b605..f790ff6616 100644 --- a/whois-nrtm4-client/src/main/java/net/ripe/db/nrtm4/client/scheduler/Nrtm4ClientCondition.java +++ b/whois-nrtm4-client/src/main/java/net/ripe/db/nrtm4/client/scheduler/Nrtm4ClientCondition.java @@ -13,10 +13,8 @@ public class Nrtm4ClientCondition implements Condition { @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { if( !context.getEnvironment().containsProperty("nrtm4.client.enabled")) { - LOGGER.info("Nrtm4 client is not enabled"); return false; } - LOGGER.info("Nrtm4 client is enabled {}", Boolean.parseBoolean(context.getEnvironment().getProperty("nrtm4.client.enabled"))); return Boolean.parseBoolean(context.getEnvironment().getProperty("nrtm4.client.enabled")); } } diff --git a/whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/AbstractNrtmClientIntegrationTest.java b/whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/AbstractNrtmClientIntegrationTest.java new file mode 100644 index 0000000000..2d0dd801f8 --- /dev/null +++ b/whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/AbstractNrtmClientIntegrationTest.java @@ -0,0 +1,37 @@ +package net.ripe.db.nrtm4.client; + +import net.ripe.db.nrtm4.client.dao.Nrtm4ClientMirrorRepository; +import net.ripe.db.nrtm4.client.reader.UpdateNotificationFileReader; +import net.ripe.db.whois.common.dao.jdbc.AbstractDatabaseHelperIntegrationTest; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; + +@ContextConfiguration(locations = {"classpath:applicationContext-nrtm4-client-test.xml"}) +public class AbstractNrtmClientIntegrationTest extends AbstractDatabaseHelperIntegrationTest { + + @Autowired + protected Nrtm4ClientMirrorRepository nrtm4ClientMirrorRepository; + + @Autowired + protected UpdateNotificationFileReader updateNotificationFileReader; + + @BeforeEach + public void restoreDatabase(){ + nrtm4ClientMirrorRepository.truncateTables(); + } + + @BeforeAll + public static void setUp(){ + System.setProperty("nrtm4.client.enabled", "true"); + } + + @AfterAll + public static void tearDown(){ + System.clearProperty("nrtm4.client.enabled"); + } + + +} diff --git a/whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/NrtmServerDummy.java b/whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/NrtmServerDummy.java new file mode 100644 index 0000000000..ab62c89ebd --- /dev/null +++ b/whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/NrtmServerDummy.java @@ -0,0 +1,138 @@ +package net.ripe.db.nrtm4.client; + +import com.google.common.io.Resources; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import net.ripe.db.nrtm4.client.client.NrtmRestClient; +import net.ripe.db.whois.common.Stub; +import net.ripe.db.whois.common.aspects.RetryFor; +import net.ripe.db.whois.common.profiles.WhoisProfile; +import org.apache.commons.compress.utils.Lists; +import org.eclipse.jetty.server.NetworkConnector; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; +import org.springframework.test.util.ReflectionTestUtils; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.List; + +@Profile({WhoisProfile.TEST}) +@Component +public class NrtmServerDummy implements Stub { + + private static final Logger LOGGER = LoggerFactory.getLogger(NrtmServerDummy.class); + + private Server server; + private int port = 0; + + private final NrtmRestClient nrtmRestClient; + + private final List mocks; + + @Autowired + public NrtmServerDummy(final NrtmRestClient nrtmRestClient) { + this.nrtmRestClient = nrtmRestClient; + this.mocks = Lists.newArrayList(); + initialiseMocks(); + } + + @PostConstruct + @RetryFor(attempts = 5, value = Exception.class) + public void start() { + server = new Server(0); + server.setHandler(new NrtmTestHandler()); + try { + server.start(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + this.port = ((NetworkConnector)server.getConnectors()[0]).getLocalPort(); + final String restUrl = String.format("http://localhost:%s/nrtmv4", getPort()); + LOGGER.info("NRTM Service dummy server restUrl: {}", restUrl); + ReflectionTestUtils.setField(nrtmRestClient, "baseUrl", restUrl); + } + + @PreDestroy + public void stop() throws Exception { + server.stop(); + } + + public int getPort() { + return port; + } + + @Override + public void reset() { + } + + + private class NrtmTestHandler extends AbstractHandler { + @Override + public void handle(final String target, final Request baseRequest, final HttpServletRequest request, final HttpServletResponse response) + throws IOException { + response.setContentType("text/xml;charset=utf-8"); + baseRequest.setHandled(true); + + for (Mock mock : mocks) { + if (mock.matches(request)) { + response.setStatus(HttpServletResponse.SC_OK); + response.setContentType(((NrtmResponseMock)mock).mediaType); + response.getWriter().println(mock.response()); + } + } + } + } + + private void initialiseMocks() { + mocks.add(new NrtmResponseMock("/nrtmv4", "nrtm-sources.html", "application/html")); + mocks.add(new NrtmResponseMock("/nrtmv4/RIPE-NONAUTH/update-notification-file.json", "nrtm-non-auth-unf.json", "application/json")); + mocks.add(new NrtmResponseMock("/nrtmv4/RIPE/update-notification-file.json", "nrtm-ripe-unf.json", "application/json")); + } + + + private interface Mock { + + String PATH = "mock/"; + boolean matches(final HttpServletRequest request); + + String response(); + + default String getResource(final String resource) { + try { + // resource is in file + return Resources.toString(Resources.getResource(PATH + resource), Charset.defaultCharset()); + } catch (IllegalArgumentException e) { + // resource doesn't exist (use resource as content) + return resource; + } catch (IOException e) { + // error reading content from resource + throw new IllegalStateException(e); + } + } + + } + + private record NrtmResponseMock(String fileType, String response, String mediaType) implements Mock { + + private NrtmResponseMock(final String fileType, final String response, final String mediaType) { + this.fileType = fileType; + this.response = getResource(response); + this.mediaType = mediaType; + } + + @Override + public boolean matches(final HttpServletRequest request) { + return request.getRequestURI().endsWith(fileType); + } + } +} diff --git a/whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/reader/UpdateNotificationFileReaderTestIntegration.java b/whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/reader/UpdateNotificationFileReaderTestIntegration.java new file mode 100644 index 0000000000..7baa48aea4 --- /dev/null +++ b/whois-nrtm4-client/src/test/java/net/ripe/db/nrtm4/client/reader/UpdateNotificationFileReaderTestIntegration.java @@ -0,0 +1,39 @@ +package net.ripe.db.nrtm4.client.reader; + +import net.ripe.db.nrtm4.client.AbstractNrtmClientIntegrationTest; +import net.ripe.db.nrtm4.client.dao.NrtmClientVersionInfo; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +@Tag("IntegrationTest") +public class UpdateNotificationFileReaderTestIntegration extends AbstractNrtmClientIntegrationTest { + + @Test + public void readUNFThenVersionAdded() { + updateNotificationFileReader.readFile(); + + final List versionInfosPerSource = nrtm4ClientMirrorRepository.getNrtmLastVersionInfo(); + assertThat(versionInfosPerSource.size(), is(2)); + } + + @Test + public void readUNFWhenAlreadyCreatedSameVersionThenVersionNotAdded(){ + nrtm4ClientMirrorRepository.saveUpdateNotificationFileVersion("RIPE-NONAUTH", 1, "6328095e-7d46-415b-9333-8f2ae274b7c8"); + nrtm4ClientMirrorRepository.saveUpdateNotificationFileVersion("RIPE", 1, "6328095e-7d46-415b-9333-8f2ae274b7c8"); + + updateNotificationFileReader.readFile(); + + final List versionInfosPerSource = nrtm4ClientMirrorRepository.getNrtmLastVersionInfo(); + assertThat(versionInfosPerSource.getFirst().version(), is(1L)); + } + + @Test + public void readUNFButDBInfoAheadThenReInitialize(){ + // TODO: [MH] Re-initialize + } +} diff --git a/whois-nrtm4-client/src/test/resources/applicationContext-nrtm4-client-test.xml b/whois-nrtm4-client/src/test/resources/applicationContext-nrtm4-client-test.xml new file mode 100644 index 0000000000..63524ebc11 --- /dev/null +++ b/whois-nrtm4-client/src/test/resources/applicationContext-nrtm4-client-test.xml @@ -0,0 +1,20 @@ + + + + + + + + + + diff --git a/whois-nrtm4-client/src/test/resources/mock/nrtm-non-auth-unf.json b/whois-nrtm4-client/src/test/resources/mock/nrtm-non-auth-unf.json new file mode 100644 index 0000000000..aded500ea2 --- /dev/null +++ b/whois-nrtm4-client/src/test/resources/mock/nrtm-non-auth-unf.json @@ -0,0 +1,14 @@ +{ + "nrtm_version": 1, + "timestamp": "2024-10-24T13:20:00Z", + "type": "notification", + "source": "RIPE-NONAUTH", + "session_id": "6328095e-7d46-415b-9333-8f2ae274b7c8", + "version": 1, + "snapshot": { + "version": 1, + "url": "https://localhost/nrtmv4/RIPE-NONAUTH/nrtm-snapshot.1.RIPE-NONAUTH.6328095e-7d46-415b-9333-8f2ae274b7c8.f1195bb8a666fe7b97fa74009a70cefa.json.gz", + "hash": "e4fa2cc60156a0daace8b7179f68661b0ad765ce09a9ca4bdab0c978b13da973" + }, + "deltas": [] +} \ No newline at end of file diff --git a/whois-nrtm4-client/src/test/resources/mock/nrtm-ripe-unf.json b/whois-nrtm4-client/src/test/resources/mock/nrtm-ripe-unf.json new file mode 100644 index 0000000000..aa46c7e3c0 --- /dev/null +++ b/whois-nrtm4-client/src/test/resources/mock/nrtm-ripe-unf.json @@ -0,0 +1,20 @@ +{ + "nrtm_version": 4, + "timestamp": "2024-10-25T00:07:00Z", + "type": "notification", + "source": "RIPE", + "session_id": "4521174b-548f-4e51-98fc-dfd720011a0c", + "version": 1, + "snapshot": { + "version": 1, + "url": "https://localhost/nrtmv4/RIPE/nrtm-snapshot.4.RIPE.4521174b-548f-4e51-98fc-dfd720011a0c.82542bd048e111fe57db404d08b6433e.json.gz", + "hash": "a15f097005d8082dae14fa28bdb25dc09e59243ef632bcadde44149ba34b373d" + }, + "deltas": [ + { + "version": 1, + "url": "https://localhost/nrtmv4/RIPE/nrtm-delta.4.RIPE.4521174b-548f-4e51-98fc-dfd720011a0c.e3be41ff312010046b67d099faa58f44.json", + "hash": "c50dd7554cb35ef5f2f45d7bfa09fc51033cbe1152d29b36cb1178319e22be3e" + } + ] +} \ No newline at end of file diff --git a/whois-nrtm4-client/src/test/resources/mock/nrtm-sources.html b/whois-nrtm4-client/src/test/resources/mock/nrtm-sources.html new file mode 100644 index 0000000000..d1e1a0c881 --- /dev/null +++ b/whois-nrtm4-client/src/test/resources/mock/nrtm-sources.html @@ -0,0 +1,6 @@ + +
NRTM Version 4
+RIPE
RIPE-NONAUTH
+ + \ No newline at end of file