feat: manage Notifier storage usage [DHIS2-17998]

dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/UID.java

@@ -55,14 +55,14 @@
 @Getter
 @EqualsAndHashCode
 public final class UID implements Serializable {
-  private static final String VALID_UID_FORMAT =
-      "UID must be an alphanumeric string of 11 characters starting with a letter.";
 
   private final String value;
 
   private UID(String value) {
     if (!CodeGenerator.isValidUid(value)) {
-      throw new IllegalArgumentException(VALID_UID_FORMAT);
+      throw new IllegalArgumentException(
+          "UID must be an alphanumeric string of 11 characters starting with a letter, but was: "
+              + value);
     }
     this.value = value;
   }

dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java

@@ -192,6 +192,8 @@ public boolean isDefaultExecutedByCreator() {
   }
 
   /**
+   * @implNote since 2.42 all jobs forward to the {@code Notifier} but those not included here use
+   *     {@link org.hisp.dhis.system.notification.NotificationLevel#ERROR}.
    * @return true, if {@link JobProgress} events should be forwarded to the {@link
    *     org.eclipse.emf.common.notify.Notifier} API, otherwise false
    */

dhis-2/dhis-api/src/main/java/org/hisp/dhis/setting/SystemSettings.java

@@ -49,6 +49,7 @@
 import org.hisp.dhis.period.RelativePeriodEnum;
 import org.hisp.dhis.scheduling.JobConfiguration;
 import org.hisp.dhis.security.LoginPageLayout;
+import org.hisp.dhis.system.notification.NotificationLevel;
 import org.hisp.dhis.translation.Translatable;
 
 /**
@@ -747,6 +748,59 @@ default boolean isHideUnapprovedDataInAnalytics() {
   }
 
   /**
+   * @since 2.42
+   * @return the minimum level required to include a notification in the list for notifications
+   *     forwarded from scheduling.
+   */
+  default NotificationLevel getNotifierLogLevel() {
+    return asEnum("notifierLogLevel", NotificationLevel.DEBUG);
+  }
+
+  /**
+   * @since 2.42
+   * @return the maximum number of messages kept for each job. When the limit is exceeded the oldest
+   *     message is dropped (FIFO).
+   */
+  default int getNotifierMaxMessagesPerJob() {
+    return asInt("notifierMaxMessagesPerJob", 500);
+  }
+
+  /**
+   * @since 2.42
+   * @return notifications and summaries older than this number of days are discarded automatically
+   */
+  default int getNotifierMaxAgeDays() {
+    return asInt("notifierMaxAgeDays", 7);
+  }
+
+  /**
+   * @since 2.42
+   * @return notifications and summaries from this number of jobs per job type are kept (youngest
+   *     remain). The oldest jobs exceeding this number per type are discarded.
+   */
+  default int getNotifierMaxJobsPerType() {
+    return asInt("notifierMaxJobsPerType", 500);
+  }
+
+  /**
+   * @since 2.42
+   * @return when true, only the first and last message are included in the message stack when
+   *     listing multiple jobs in the single or all job types overview.
+   */
+  default boolean isNotifierGistOverview() {
+    return asBoolean("notifierGistOverview", true);
+  }
+
+  /**
+   * @since 2.42
+   * @return the duration the notifier has to be idle to run an automatic cleanup cycle
+   */
+  default long getNotifierCleanAfterIdleTime() {
+    return asInt("notifierCleanAfterIdleTime", 60_0000); // 1 minute
+  }
+
+  /**
+   * @since 2.42
    * @return A regex pattern string that enforces the current password validation rules
    */
   default String getPasswordValidationPattern() {

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java

@@ -50,7 +50,9 @@
 import org.hisp.dhis.feedback.NotFoundException;
 import org.hisp.dhis.leader.election.LeaderManager;
 import org.hisp.dhis.message.MessageService;
+import org.hisp.dhis.setting.SystemSettings;
 import org.hisp.dhis.setting.SystemSettingsProvider;
+import org.hisp.dhis.system.notification.NotificationLevel;
 import org.hisp.dhis.system.notification.Notifier;
 import org.hisp.dhis.user.AuthenticationService;
 import org.hisp.dhis.user.UserDetails;
@@ -257,15 +259,17 @@ private static void logError(String message, Exception ex) {
   }
 
   private JobProgress startRecording(@Nonnull JobConfiguration job, @Nonnull Runnable observer) {
-    JobProgress tracker =
+    SystemSettings settings = settingsProvider.getCurrentSettings();
+    NotificationLevel level =
         job.getJobType().isUsingNotifications()
-            ? new NotifierJobProgress(notifier, job)
-            : JobProgress.noop();
+            ? settings.getNotifierLogLevel()
+            : NotificationLevel.ERROR;
+    JobProgress tracker = new NotifierJobProgress(notifier, job, level);
     boolean logInfoOnDebug =
         job.getSchedulingType() != SchedulingType.ONCE_ASAP
             && job.getLastExecuted() != null
             && Duration.between(job.getLastExecuted().toInstant(), Instant.now()).getSeconds()
-                < settingsProvider.getCurrentSettings().getJobsLogDebugBelowSeconds();
+                < settings.getJobsLogDebugBelowSeconds();
     RecordingJobProgress progress =
         new RecordingJobProgress(messages, job, tracker, true, observer, logInfoOnDebug, false);
     recordingsById.put(job.getUid(), progress);

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/NotifierJobProgress.java

@@ -29,12 +29,12 @@
 
 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.util.concurrent.atomic.AtomicBoolean;
 import javax.annotation.CheckForNull;
 import javax.annotation.Nonnull;
 import lombok.RequiredArgsConstructor;
+import org.hisp.dhis.jsontree.JsonValue;
 import org.hisp.dhis.system.notification.NotificationDataType;
 import org.hisp.dhis.system.notification.NotificationLevel;
 import org.hisp.dhis.system.notification.Notifier;
@@ -47,16 +47,27 @@
  */
 @RequiredArgsConstructor
 public class NotifierJobProgress implements JobProgress {
-  private final Notifier notifier;
 
+  private final Notifier notifier;
   private final JobConfiguration jobId;
-
+  private final NotificationLevel level;
   private final AtomicBoolean hasCleared = new AtomicBoolean();
 
   private int stageItems;
-
   private int stageItem;
 
+  private boolean isLoggedLoop() {
+    return NotificationLevel.LOOP.ordinal() >= level.ordinal();
+  }
+
+  private boolean isLoggedInfo() {
+    return NotificationLevel.INFO.ordinal() >= level.ordinal();
+  }
+
+  private boolean isLoggedError() {
+    return NotificationLevel.ERROR.ordinal() >= level.ordinal();
+  }
+
   @Override
   public void startingProcess(String description, Object... args) {
     String message =
@@ -66,6 +77,7 @@ public void startingProcess(String description, Object... args) {
     if (hasCleared.compareAndSet(false, true)) {
       notifier.clear(jobId);
     }
+    // Note: intentionally no log level check - always log first
     notifier.notify(
         jobId,
         NotificationLevel.INFO,
@@ -77,11 +89,13 @@ public void startingProcess(String description, Object... args) {
 
   @Override
   public void completedProcess(String summary, Object... args) {
+    // Note: intentionally no log level check - always log last
     notifier.notify(jobId, format(summary, args), true);
   }
 
   @Override
   public void failedProcess(@CheckForNull String error, Object... args) {
+    // Note: intentionally no log level check - always log last
     notifier.notify(jobId, NotificationLevel.ERROR, format(error, args), true);
   }
 
@@ -90,28 +104,28 @@ public void startingStage(
       @Nonnull String description, int workItems, @Nonnull FailurePolicy onFailure) {
     stageItems = workItems;
     stageItem = 0;
-    if (isNotEmpty(description)) {
+    if (isLoggedInfo() && isNotEmpty(description)) {
       notifier.notify(jobId, description);
     }
   }
 
   @Override
   public void completedStage(String summary, Object... args) {
-    if (isNotEmpty(summary)) {
+    if (isLoggedInfo() && isNotEmpty(summary)) {
       notifier.notify(jobId, format(summary, args));
     }
   }
 
   @Override
   public void failedStage(@Nonnull String error, Object... args) {
-    if (isNotEmpty(error)) {
+    if (isLoggedError() && isNotEmpty(error)) {
       notifier.notify(jobId, NotificationLevel.ERROR, format(error, args), false);
     }
   }
 
   @Override
   public void startingWorkItem(@Nonnull String description, @Nonnull FailurePolicy onFailure) {
-    if (isNotEmpty(description)) {
+    if (isLoggedLoop() && isNotEmpty(description)) {
       String nOf = "[" + (stageItems > 0 ? stageItem + "/" + stageItems : "" + stageItem) + "] ";
       notifier.notify(jobId, NotificationLevel.LOOP, nOf + description, false);
     }
@@ -120,26 +134,26 @@ public void startingWorkItem(@Nonnull String description, @Nonnull FailurePolicy
 
   @Override
   public void completedWorkItem(String summary, Object... args) {
-    if (isNotEmpty(summary)) {
+    if (isLoggedLoop() && isNotEmpty(summary)) {
       String nOf = "[" + (stageItems > 0 ? stageItem + "/" + stageItems : "" + stageItem) + "] ";
       notifier.notify(jobId, NotificationLevel.LOOP, nOf + format(summary, args), false);
     }
   }
 
   @Override
   public void failedWorkItem(@Nonnull String error, Object... args) {
-    if (isNotEmpty(error)) {
+    if (isLoggedError() && isNotEmpty(error)) {
       notifier.notify(jobId, NotificationLevel.ERROR, format(error, args), false);
     }
   }
 
-  private JsonNode getJobParameterData() {
+  private JsonValue getJobParameterData() {
     JobParameters params = jobId.getJobParameters();
     if (params == null) {
       return null;
     }
     try {
-      return new ObjectMapper().valueToTree(params);
+      return JsonValue.of(new ObjectMapper().writeValueAsString(params));
     } catch (Exception ex) {
       return null;
     }