RIPE NCC has merged 0d5cbac

* Fix recently introduced test smells [a6c591d6]
* Update dependency io.freefair.lombok:io.freefair.lombok.gradle.plugin to v8.4 [abf6ba07]
* Update dependency gradle to v8.4 [9c081c9e]
* Fix the filename of the jsonl log file [de69c5b0]
* Use @UtilityClass [4cdea886]
* SonarQube feedback [2bcecd21]
* Improve style of testcase [2c2417bb]
* Base published_at for published objects on (signed) object time [09820351]
* This unused class does not spark joy either [a585d9e3]
* Remove unused class [3a7e6c0b]
* Update dependency commons-io:commons-io to v2.14.0 [bf9f955c]
* Update postgres to 15 [1ceaee79]
* Update dependency io.freefair.lombok:io.freefair.lombok.gradle.plugin to v8.3 [9ece42a7]
* Add counter on progress on All CA Update Service progress [93e80ba3]

.gitlab-ci.yml

@@ -1,5 +1,5 @@
 default:
-  image: gradle:8.3-jdk11
+  image: gradle:8.4-jdk11
 
 # Explicit version of the Mergerequests-Pipelines workflow, with the main branch
 # added.
@@ -45,7 +45,7 @@ build:
   # slow builds in augusts 2023
   timeout: 60m
   services:
-    - postgres:14
+    - postgres:15
   variables:
       POSTGRES_DB: certdb_test
       POSTGRES_USER: certdb

build.gradle

@@ -58,7 +58,7 @@ dependencies {
     implementation 'com.google.code.gson:gson:2.10.1'
     implementation 'com.jamesmurty.utils:java-xmlbuilder:1.3'
     implementation 'commons-codec:commons-codec:1.16.0'
-    implementation 'commons-io:commons-io:2.13.0'
+    implementation 'commons-io:commons-io:2.14.0'
     implementation 'ch.qos.logback.contrib:logback-json-classic:0.1.5'
     implementation 'ch.qos.logback.contrib:logback-jackson:0.1.5'
     implementation 'net.logstash.logback:logstash-logback-encoder:7.3'

buildSrc/build.gradle

@@ -9,7 +9,7 @@ repositories {
 }
 
 dependencies {
-    implementation 'io.freefair.lombok:io.freefair.lombok.gradle.plugin:8.1.0'
+    implementation 'io.freefair.lombok:io.freefair.lombok.gradle.plugin:8.4'
     implementation('com.gorylenko.gradle-git-properties:com.gorylenko.gradle-git-properties.gradle.plugin:2.4.1') {
         exclude group: 'org.eclipse.jgit', module: 'org.eclipse.jgit'
     }

docker-compose.yml

@@ -2,7 +2,7 @@ version: "3.8"
 
 services:
   postgres:
-    image: postgres:14
+    image: postgres:15
     # Uncomment to expose postgres instance
     # ports:
     #   - 5432:5432

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -145,15 +145,15 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -202,11 +202,11 @@ fi
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
-# Collect all arguments for the java command;
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
-#     shell script including quotes and variable substitutions, so put them in
-#     double quotes to make sure that they get re-expanded; and
-#   * put everything else in single quotes, so that it's not re-expanded.
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \

src/main/java/net/ripe/rpki/domain/GenericPublishedObject.java

@@ -3,6 +3,7 @@
 import lombok.Getter;
 import lombok.NonNull;
 import net.ripe.rpki.ncc.core.domain.support.EntitySupport;
+import org.joda.time.Instant;
 
 import javax.persistence.Column;
 import javax.persistence.EnumType;
@@ -39,11 +40,16 @@ public abstract class GenericPublishedObject extends EntitySupport {
     @NonNull
     protected byte[] content = new byte[0];
 
+    @Column(name = "created_at", nullable = false)
+    @Getter
+    private Instant createdAt;
+
     protected GenericPublishedObject() {
     }
 
-    protected GenericPublishedObject(@NonNull byte[] content) {
+    protected GenericPublishedObject(@NonNull byte[] content, Instant createdAt) {
         this.content = Arrays.copyOf(content, content.length);
+        this.createdAt = createdAt;
     }
 
     @NonNull

src/main/java/net/ripe/rpki/domain/PublishedObject.java

@@ -5,6 +5,7 @@
 import net.ripe.rpki.commons.crypto.ValidityPeriod;
 import net.ripe.rpki.domain.manifest.ManifestEntity;
 import org.apache.commons.lang3.Validate;
+import org.joda.time.Instant;
 
 import javax.persistence.*;
 import java.net.URI;
@@ -69,7 +70,7 @@ public PublishedObject(
         @NonNull URI publicationDirectory,
         @NonNull ValidityPeriod validityPeriod
     ) {
-        super(content);
+        super(content, validityPeriod.getNotValidBefore().toInstant());
         this.issuingKeyPair = issuingKeyPair;
         this.filename = filename;
         this.includedInManifest = includedInManifest;

src/main/java/net/ripe/rpki/domain/TrustAnchorPublishedObject.java

@@ -1,25 +1,24 @@
 package net.ripe.rpki.domain;
 
+import lombok.NoArgsConstructor;
 import lombok.NonNull;
+import org.joda.time.Instant;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Table;
 import java.net.URI;
 
+@NoArgsConstructor
 @Entity
 @Table(name = "ta_published_object")
 public class TrustAnchorPublishedObject extends GenericPublishedObject {
-
     @Column(name = "uri", nullable = false)
     @NonNull
     private String uri = "";
 
-    protected TrustAnchorPublishedObject() {
-    }
-
-    public TrustAnchorPublishedObject(@NonNull URI uri, byte[] content) {
-        super(content);
+    public TrustAnchorPublishedObject(@NonNull URI uri, byte[] content, Instant createdAt) {
+        super(content, createdAt);
         this.uri = uri.toString();
     }
 

src/main/java/net/ripe/rpki/offline/ra/service/TrustAnchorResponseProcessor.java

@@ -17,6 +17,7 @@
 import net.ripe.rpki.commons.ta.domain.response.SigningResponse;
 import net.ripe.rpki.commons.ta.domain.response.TaResponse;
 import net.ripe.rpki.commons.ta.domain.response.TrustAnchorResponse;
+import net.ripe.rpki.util.PublishedObjectUtil;
 import org.joda.time.DateTime;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
@@ -101,10 +102,10 @@ List<TrustAnchorPublishedObject> applyChangeToPublishedObjects(Map<URI, Certific
                 if (!objectsAreSame(publishedObject, objectToPublish, uri)) {
                     publishedObject.withdraw();
                     result.add(publishedObject);
-                    result.add(new TrustAnchorPublishedObject(uri, objectToPublish.getEncoded()));
+                    result.add(new TrustAnchorPublishedObject(uri, objectToPublish.getEncoded(), PublishedObjectUtil.getFileCreationTime(uri, objectToPublish.getEncoded())));
                 }
             } else {
-                result.add(new TrustAnchorPublishedObject(uri, objectToPublish.getEncoded()));
+                result.add(new TrustAnchorPublishedObject(uri, objectToPublish.getEncoded(), PublishedObjectUtil.getFileCreationTime(uri, objectToPublish.getEncoded())));
             }
         });
         withdrawObjects(activeObjects.values());

src/main/java/net/ripe/rpki/services/impl/background/AllCaCertificateUpdateServiceBean.java

@@ -1,5 +1,7 @@
 package net.ripe.rpki.services.impl.background;
 
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.MeterRegistry;
 import lombok.AllArgsConstructor;
 import net.ripe.rpki.core.services.background.BackgroundTaskRunner;
 import net.ripe.rpki.core.services.background.SequentialBackgroundServiceWithAdminPrivilegesOnActiveNode;
@@ -28,6 +30,8 @@
 public class AllCaCertificateUpdateServiceBean extends SequentialBackgroundServiceWithAdminPrivilegesOnActiveNode {
     private final int updateBatchSize;
 
+    private final Counter certificateUpdates;
+
     private final CertificateAuthorityViewService caViewService;
     private final CommandService commandService;
     private final ResourceCache resourceCache;
@@ -39,13 +43,18 @@ public AllCaCertificateUpdateServiceBean(BackgroundTaskRunner backgroundTaskRunn
                                              CommandService commandService,
                                              ResourceCache resourceCache,
                                              RepositoryConfiguration repositoryConfiguration,
-                                             @Value("${certificate.authority.update.batch.size:1000}") int updateBatchSize) {
+                                             @Value("${certificate.authority.update.batch.size:1000}") int updateBatchSize,
+                                             MeterRegistry meterRegistry) {
         super(backgroundTaskRunner);
         this.caViewService = caViewService;
         this.commandService = commandService;
         this.resourceCache = resourceCache;
         this.repositoryConfiguration = repositoryConfiguration;
         this.updateBatchSize = updateBatchSize;
+
+        certificateUpdates = Counter.builder("rpkicore.all.certificate.update.progress")
+                .description("Number of certificates updated by All CA certificate update service")
+                .register(meterRegistry);
     }
 
     @Override
@@ -126,6 +135,7 @@ private BackgroundTaskRunner.Task<Boolean> updateParentAndChildrenTask(Certifica
                     boolean updated = updateIncomingCertificates(parentCa);
                     if (updated) {
                         remainingCounter.decrementAndGet();
+                        certificateUpdates.increment();
                     }
 
                     long updateCount = updateChildren(parentCa);

src/main/java/net/ripe/rpki/util/PublishedObjectUtil.java

@@ -0,0 +1,51 @@
+package net.ripe.rpki.util;
+
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+import net.ripe.rpki.commons.crypto.cms.GenericRpkiSignedObjectParser;
+import net.ripe.rpki.commons.crypto.crl.X509Crl;
+import net.ripe.rpki.commons.crypto.x509cert.X509ResourceCertificateParser;
+import net.ripe.rpki.commons.util.RepositoryObjectType;
+import net.ripe.rpki.commons.validation.ValidationResult;
+import org.joda.time.Instant;
+
+import java.net.URI;
+import java.util.Base64;
+
+@Slf4j
+@UtilityClass
+public class PublishedObjectUtil {
+
+    // FIXME: Should be moved into rpki-commons after we use >=1.35 in core because this is a port of code present in commons test-cases and in rsyncit.
+    public static Instant getFileCreationTime(URI uri, byte[] decoded) {
+        var objectUri = uri.toString();
+        final RepositoryObjectType objectType = RepositoryObjectType.parse(objectUri);
+        try {
+            switch (objectType) {
+                case Manifest:
+                case Aspa:
+                case Roa:
+                case Gbr:
+                    var signedObjectParser = new GenericRpkiSignedObjectParser();
+
+                    signedObjectParser.parse(ValidationResult.withLocation(objectUri), decoded);
+                    return Instant.ofEpochMilli(signedObjectParser.getSigningTime().getMillis());
+                case Certificate:
+                    X509ResourceCertificateParser x509CertificateParser = new X509ResourceCertificateParser();
+                    x509CertificateParser.parse(ValidationResult.withLocation(objectUri), decoded);
+                    final var cert = x509CertificateParser.getCertificate().getCertificate();
+                    return Instant.ofEpochMilli(cert.getNotBefore().getTime());
+                case Crl:
+                    var x509Crl = X509Crl.parseDerEncoded(decoded, ValidationResult.withLocation(objectUri));
+                    var crl = x509Crl.getCrl();
+                    return Instant.ofEpochMilli(crl.getThisUpdate().getTime());
+                case Unknown:
+                    log.error("Unknown object type for object url = {}");
+                    return Instant.now();
+            }
+        } catch (Exception e) {
+            log.error("Could not parse the object url = {}, body = {} :", objectUri, Base64.getEncoder().encodeToString(decoded));
+        }
+        return Instant.now();
+    }
+}

src/main/resources/logback/logback-production.xml

@@ -17,7 +17,7 @@
 
     <appender name="certification-jsonl" class="ch.qos.logback.core.rolling.RollingFileAppender">
         <encoder class="net.logstash.logback.encoder.LogstashEncoder" />
-        <file>../logs/certification-jsonl.log</file>
+        <file>../logs/certification-jsonl</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
             <fileNamePattern>../logs/certification.jsonl.%d{yyyy-MM-dd}</fileNamePattern>
         </rollingPolicy>

src/test/java/net/ripe/rpki/domain/manifest/ManifestEntityTest.java

@@ -6,17 +6,15 @@
 import net.ripe.rpki.domain.*;
 import net.ripe.rpki.domain.interca.CertificateIssuanceRequest;
 import net.ripe.rpki.domain.interca.CertificateIssuanceResponse;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeConstants;
-import org.joda.time.DateTimeUtils;
-import org.joda.time.DateTimeZone;
+import org.joda.time.*;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.springframework.transaction.annotation.Transactional;
 
 import java.net.URI;
 import java.security.KeyPair;
+import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -49,8 +47,12 @@ public void setUp() {
         ca = createInitialisedProdCaWithRipeResources();
         currentKeyPair = ca.getCurrentKeyPair();
         subject = new ManifestEntity(currentKeyPair);
-        publishedObject1 = new PublishedObject(currentKeyPair, "foo.crl", new byte[]{1, 2, 3, 4}, true, PUBLICATION_DIRECTORY, new ValidityPeriod());
-        publishedObject2 = new PublishedObject(currentKeyPair, "foo.roa", new byte[]{5, 6, 7, 8}, true, PUBLICATION_DIRECTORY, new ValidityPeriod());
+
+        var start = now.toDate();
+        var end = now.plus(Duration.standardDays(7)).toDate();
+
+        publishedObject1 = new PublishedObject(currentKeyPair, "foo.crl", new byte[]{1, 2, 3, 4}, true, PUBLICATION_DIRECTORY, new ValidityPeriod(start, end));
+        publishedObject2 = new PublishedObject(currentKeyPair, "foo.roa", new byte[]{5, 6, 7, 8}, true, PUBLICATION_DIRECTORY, new ValidityPeriod(start, end));
         initialEntries = Collections.singleton(publishedObject1);
 
         eeKeyPair = PregeneratedKeyPairFactory.getInstance().generate();