NRTMv4 Client setup and persisting Update Notification File info (#1560)

* feat: nrtmv4 client setup * feat: add whois-nrtm4-client everywhere * feat: get common dependencies in whois-nrtm-client (aspectJ in them) * feat: read unf and store main information * feat: fix repository * feat: clarify reader * feat: remove unnecesary value injection * feat: add schema creation changes * feat: fix schema syntax error * feat: hardcode for now nrtm baseUrl * feat: hardcode RC * feat: fix compilation issues * feat: add logs and fix the name * feat: change properties and put logs in the condition * feat: add logs * feat: fix log * feat: create nrtm rest client * feat: fix typo * feat: run schedule task each minute * feat: add nrtm4 client to api db endtoend modules * feat: add source log * feat: try catch duplicated key exception when the version already exist * feat: refactor * feat: remove the changes of rest client * feat: remove unused import * feat: change nrtm data soure configurations name * feat: use a rowmapper in between sources call and jsonnode * feat: fix compilation issue * feat: add retrn statment * feat: fix get resources * feat: fix baseUrl * feat: remove the slash from url * feat: ignore unknown properties * feat: filter by version and fix the mapper * feat: handle when table is null * feat: refactor for clarifications * feat: fix sql * feat: check not null * feat: add logs * feat: remove logs and add the group by to fix the query * feat: refactor * feat: put the dependency again in whois-api * feat: do no restart if notification is the same * feat: remove initializeNRTMClient module body * feat: remove unused import and add the conditional * feat: rename it by client * feat: rename the condition * feat: add dependsOn dependency and conditional just for the config * feat: do not use dependsOn use conditional
RIPE-NCC · Oct 25, 2024 · 9618a7f · 9618a7f
1 parent 9d14ab8
commit 9618a7f
Show file tree

Hide file tree

Showing 15 changed files with 589 additions and 2 deletions.
diff --git a/pom.xml b/pom.xml
@@ -131,6 +131,7 @@
         <module>whois-db</module>
         <module>whois-endtoend</module>
         <module>whois-nrtm4</module>
+        <module>whois-nrtm4-client</module>
     </modules>
 
     <dependencyManagement>
@@ -208,6 +209,18 @@
                 <type>test-jar</type>
                 <scope>test</scope>
             </dependency>
+            <dependency>
+                <groupId>net.ripe.db</groupId>
+                <artifactId>whois-nrtm4-client</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>net.ripe.db</groupId>
+                <artifactId>whois-nrtm4-client</artifactId>
+                <version>${project.version}</version>
+                <type>test-jar</type>
+                <scope>test</scope>
+            </dependency>
             <dependency>
                 <groupId>net.ripe.db</groupId>
                 <artifactId>whois-query</artifactId>

diff --git a/whois-api/pom.xml b/whois-api/pom.xml
@@ -44,6 +44,16 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>net.ripe.db</groupId>
+            <artifactId>whois-nrtm4-client</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.bouncycastle</groupId>
+                    <artifactId>bcprov-jdk15on</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>commons-validator</groupId>
             <artifactId>commons-validator</artifactId>

diff --git a/whois-commons/src/main/java/net/ripe/db/whois/common/DataSourceConfigurations.java b/whois-commons/src/main/java/net/ripe/db/whois/common/DataSourceConfigurations.java
@@ -12,7 +12,7 @@
 @Configuration
 public class DataSourceConfigurations {
 
-    private static final String DRIVER_CLASS_NAME = "org.mariadb.jdbc.Driver";
+    public static final String DRIVER_CLASS_NAME = "org.mariadb.jdbc.Driver";
 
     @Bean
     public DataSource aclDataSource(@Value("${acl.database.url}") final String jdbcUrl, @Value("${acl.database.username}") final String jdbcUser, @Value("${acl.database.password}") final String jdbcPass) throws PropertyVetoException {
@@ -55,7 +55,7 @@ public ComboPooledDataSource nrtmSlaveDataSource(@Value("${nrtm.slave.database.u
         return createDataSource(DRIVER_CLASS_NAME, jdbcUrl, jdbcUser, jdbcPass);
     }
 
-    private ComboPooledDataSource createDataSource(final String jdbcDriver, final String jdbcUrl, final String jdbcUser, final String jdbcPass) throws PropertyVetoException {
+    public static ComboPooledDataSource createDataSource(final String jdbcDriver, final String jdbcUrl, final String jdbcUser, final String jdbcPass) throws PropertyVetoException {
         final ComboPooledDataSource source = new ComboPooledDataSource();
 
         source.setJdbcUrl(jdbcUrl);

diff --git a/whois-commons/src/main/resources/nrtmv4_mirror_data.sql b/whois-commons/src/main/resources/nrtmv4_mirror_data.sql
@@ -0,0 +1,23 @@
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT = @@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS = @@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION = @@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+/*!40103 SET @OLD_TIME_ZONE = @@TIME_ZONE */;
+/*!40103 SET TIME_ZONE = '+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS = @@UNIQUE_CHECKS, UNIQUE_CHECKS = 0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS = @@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS = 0 */;
+/*!40101 SET @OLD_SQL_MODE = @@SQL_MODE, SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES = @@SQL_NOTES, SQL_NOTES = 0 */;
+
+INSERT INTO `version`
+VALUES ('whois-1.115');
+
+/*!40103 SET TIME_ZONE = @OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE = @OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS = @OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS = @OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT = @OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS = @OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION = @OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES = @OLD_SQL_NOTES */;
diff --git a/whois-commons/src/main/resources/nrtmv4_mirror_schema.sql b/whois-commons/src/main/resources/nrtmv4_mirror_schema.sql
@@ -0,0 +1,67 @@
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT = @@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS = @@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION = @@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8mb4 */;
+/*!40103 SET @OLD_TIME_ZONE = @@TIME_ZONE */;
+/*!40103 SET TIME_ZONE = '+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS = @@UNIQUE_CHECKS, UNIQUE_CHECKS = 0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS = @@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS = 0 */;
+/*!40101 SET @OLD_SQL_MODE = @@SQL_MODE, SQL_MODE = 'NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES = @@SQL_NOTES, SQL_NOTES = 0 */;
+
+DROP DATABASE IF EXISTS NRTMV4_CLIENT_RIPE;
+CREATE DATABASE NRTMV4_CLIENT_RIPE;
+USE NRTMV4_CLIENT_RIPE;
+
+DROP TABLE IF EXISTS `version`;
+DROP TABLE IF EXISTS `public_key`;
+DROP TABLE IF EXISTS `last_mirror`;
+DROP TABLE IF EXISTS `version_info`;
+
+
+CREATE TABLE `version`
+(
+    `version` varchar(80)
+) ENGINE = InnoDB
+  DEFAULT CHARSET = utf8mb4;
+
+
+CREATE TABLE `public_key`
+(
+    `id`          int unsigned    NOT NULL AUTO_INCREMENT,
+    `public_key`  VARBINARY(3000) NOT NULL,
+    `next_public_key`     VARBINARY(3000),
+    UNIQUE KEY `public_key_name_uk` (`public_key`),
+    PRIMARY KEY (`id`)
+) ENGINE = InnoDB
+  DEFAULT CHARSET = utf8mb4;
+
+
+CREATE TABLE `version_info`
+(
+    `id`             int unsigned    NOT NULL AUTO_INCREMENT,
+    `source`         varchar(40)     NOT NULL,
+    `version`        int unsigned    NOT NULL,
+    `session_id`     varchar(128)    NOT NULL,
+    `type`           varchar(128)    NOT NULL,
+    `created`        bigint unsigned NOT NULL,
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `version_info__session__source__version__type__uk` (`session_id`, `source`, `version`, `type`)
+) ENGINE = InnoDB
+  DEFAULT CHARSET = utf8mb4;
+
+
+CREATE TABLE `last_mirror`
+(
+    `object_id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+    `sequence_id` int(10) unsigned NOT NULL DEFAULT '1',
+    `timestamp` int(10) unsigned NOT NULL DEFAULT '0',
+    `object_type` tinyint(3) unsigned NOT NULL DEFAULT '0',
+    `object` longblob NOT NULL,
+    `pkey` varchar(254) NOT NULL DEFAULT '',
+    PRIMARY KEY (`object_id`,`sequence_id`),
+    CONSTRAINT `last_mirror__version_id__fk` FOREIGN KEY (`version_id`) REFERENCES `version_info` (`id`),
+    KEY `last_pkey` (`pkey`),
+    KEY `object_type_index` (`object_type`)
+) ENGINE=InnoDB
+  DEFAULT CHARSET=utf8mb4;
diff --git a/whois-commons/src/test/resources/whois.properties b/whois-commons/src/test/resources/whois.properties
@@ -132,6 +132,15 @@ nrtm.slave.database.url=jdbc:mysql://${db.host:localhost}/NRTM_LOCAL
 nrtm.slave.database.username=dbint
 nrtm.slave.database.password=
 
+nrtm.client.database.url=jdbc:mysql://${db.host:localhost}/NRTM4_CLIENT_LOCAL
+nrtm.client.database.username=dbint
+nrtm.client.database.password=
+
+nrtm.client.slave.database.url=jdbc:mysql://${db.host:localhost}/NRTM4_CLIENT_LOCAL
+nrtm.client.slave.database.username=dbint
+nrtm.client.slave.database.password=
+
+
 # NRTMv4
 nrtm.file.path=/tmp
 nrtm.snapshot.window=23:00 - 05:00

diff --git a/whois-nrtm4-client/pom.xml b/whois-nrtm4-client/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>whois</artifactId>
+        <groupId>net.ripe.db</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>whois-nrtm4-client</artifactId>
+    <packaging>jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.mariadb.jdbc</groupId>
+            <artifactId>mariadb-java-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-jdbc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>net.ripe.db</groupId>
+            <artifactId>whois-commons</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.jetty</groupId>
+            <artifactId>jetty-rewrite</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcprov-jdk18on</artifactId>
+        </dependency>
+
+        <!-- test -->
+
+        <dependency>
+            <groupId>net.ripe.db</groupId>
+            <artifactId>whois-commons</artifactId>
+            <type>test-jar</type>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>aspectj-maven-plugin</artifactId>
+                <version>1.15.0</version>
+                <configuration>
+                    <aspectLibraries>
+                        <aspectLibrary>
+                            <groupId>org.springframework</groupId>
+                            <artifactId>spring-aspects</artifactId>
+                        </aspectLibrary>
+                        <aspectLibrary>
+                            <groupId>net.ripe.db</groupId>
+                            <artifactId>whois-commons</artifactId>
+                        </aspectLibrary>
+                    </aspectLibraries>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>
diff --git a/whois-nrtm4-client/src/main/java/net/ripe/db/nrtm4/client/client/NrtmRestClient.java b/whois-nrtm4-client/src/main/java/net/ripe/db/nrtm4/client/client/NrtmRestClient.java
@@ -0,0 +1,99 @@
+package net.ripe.db.nrtm4.client.client;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.introspect.AnnotationIntrospectorPair;
+import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector;
+import com.fasterxml.jackson.databind.json.JsonMapper;
+import com.fasterxml.jackson.databind.type.TypeFactory;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.fasterxml.jackson.jakarta.rs.json.JacksonJsonProvider;
+import com.fasterxml.jackson.module.jakarta.xmlbind.JakartaXmlBindAnnotationIntrospector;
+import jakarta.ws.rs.client.Client;
+import jakarta.ws.rs.client.ClientBuilder;
+import jakarta.ws.rs.core.MediaType;
+import net.ripe.db.nrtm4.client.scheduler.Nrtm4ClientCondition;
+import org.apache.commons.compress.utils.Lists;
+import org.glassfish.jersey.client.ClientProperties;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Conditional;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Component
+@Conditional(Nrtm4ClientCondition.class)
+public class NrtmRestClient {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(NrtmRestClient.class);
+
+    private final String baseUrl;
+
+    private static final int CLIENT_CONNECT_TIMEOUT = 10_000;
+
+    private static final int CLIENT_READ_TIMEOUT = 60_000;
+
+    private final Client client;
+
+    public NrtmRestClient(@Value("${nrtm.baseUrl}") final String baseUrl) {
+        final ObjectMapper objectMapper = JsonMapper.builder()
+                .enable(SerializationFeature.INDENT_OUTPUT)
+                .build();
+        objectMapper.setAnnotationIntrospector(
+                new AnnotationIntrospectorPair(
+                        new JacksonAnnotationIntrospector(),
+                        new JakartaXmlBindAnnotationIntrospector(TypeFactory.defaultInstance())));
+        objectMapper.registerModule(new JavaTimeModule());
+        final JacksonJsonProvider jsonProvider = (new JacksonJsonProvider())
+                .configure(DeserializationFeature.UNWRAP_ROOT_VALUE, false)
+                .configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true);
+        jsonProvider.setMapper(objectMapper);
+
+        this.client = (ClientBuilder.newBuilder()
+                .register(jsonProvider))
+                .property(ClientProperties.CONNECT_TIMEOUT, CLIENT_CONNECT_TIMEOUT)
+                .property(ClientProperties.READ_TIMEOUT, CLIENT_READ_TIMEOUT)
+                .build();
+        this.baseUrl = "https://nrtm-rc.db.ripe.net/nrtmv4"; //use the baseUrl in the future
+    }
+
+    public List<String> getNrtmAvailableSources(){
+        try {
+            final String response = client.target(baseUrl)
+                    .request(MediaType.TEXT_HTML_TYPE)
+                    .get(String.class);
+
+            return extractSources(response);
+        } catch (final Exception e) {
+            LOGGER.error("Unable to get the available sources", e);
+            return Lists.newArrayList();
+        }
+    }
+
+    public UpdateNotificationFileResponse getNotificationFile(final String source){
+        return client.target(String.format("%s/%s", baseUrl, source))
+                .path("update-notification-file.json")
+                .request(MediaType.APPLICATION_JSON_TYPE)
+                .get(UpdateNotificationFileResponse.class);
+    }
+
+    private static List<String> extractSources(final String html) {
+        final List<String> sources = com.google.common.collect.Lists.newArrayList();
+
+        final String regex = "<a[^>]*>(.*?)</a>";
+        final Pattern pattern = Pattern.compile(regex);
+        final Matcher matcher = pattern.matcher(html);
+
+        while (matcher.find()) {
+            final String source = matcher.group(1).trim();
+            sources.add(source);
+        }
+
+        return sources;
+    }
+}
diff --git a/...-client/src/main/java/net/ripe/db/nrtm4/client/client/UpdateNotificationFileResponse.java b/...-client/src/main/java/net/ripe/db/nrtm4/client/client/UpdateNotificationFileResponse.java
@@ -0,0 +1,26 @@
+package net.ripe.db.nrtm4.client.client;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class UpdateNotificationFileResponse {
+
+    @JsonProperty("session_id")
+    private final String sessionID;
+    private final long version;
+
+    public UpdateNotificationFileResponse() {
+        this.sessionID = null;
+        this.version = 0L;
+    }
+
+    public String getSessionID() {
+        return sessionID;
+    }
+
+    public long getVersion() {
+        return version;
+    }
+
+}