feat: manage Notifier storage usage [DHIS2-17998] (2.41) (#19807)

* feat: manage Notifier storage usage [DHIS2-17998] (#19738)

* feat: gist for overview lists, limit setting [DHIS2-17998]

* refactor: NotifierStore implemented

* feat: clear and cap API and endpoints

* feat: cap for redis store

* fix: cap and clean consistency, log level filter for scheduling only

* chore: API cleanup, javadoc, some test fixes

* fix: notifier tests

* fix: hide transient empty in-memory stores in the API

* fix: update test assert with new message

* fix: remove notifier stubbing from mock test

* fix: e2e test assert for updated message

* fix: mock test setup, dependencies

* fix: add jackson core back in

* fix: exclude jackson core from dependency check

* fix: sonar issues

* test: notifier API tests for in-memory and redis

* fix: sonar warnings

* chore: fix typo

* chore: prevent null message logging + log cleanup [DHIS2-17998] (#19800)

* fix: maven dependencies + mock tests

* fix: add Long conversion

* fix: e2e test message expectations after change in wording

dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/UID.java

@@ -27,16 +27,68 @@
  */
 package org.hisp.dhis.common;
 
-import lombok.NoArgsConstructor;
+import static java.util.stream.Collectors.toUnmodifiableSet;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import org.hisp.dhis.user.UserDetails;
 
 /**
- * A "virtual" UID type that is "context-sensitive" and points to a UID of the current {@code
+ * UID represents an alphanumeric string of 11 characters starting with a letter.
+ *
+ * <p>A "virtual" UID type that is "context-sensitive" and points to a UID of the current {@code
  * Api.Endpoint}'s {@link org.hisp.dhis.common.OpenApi.EntityType}.
  *
  * <p>In other words by using this type in {@link OpenApi.Param#value()} the annotated parameter
  * becomes a UID string of the controllers' entity type.
  *
  * @author Jan Bernitt
  */
-@NoArgsConstructor
-public final class UID {}
+@Getter
+@EqualsAndHashCode
+public final class UID implements Serializable {
+
+  private final String value;
+
+  private UID(String value) {
+    if (!CodeGenerator.isValidUid(value)) {
+      throw new IllegalArgumentException(
+          "UID must be an alphanumeric string of 11 characters starting with a letter, but was: "
+              + value);
+    }
+    this.value = value;
+  }
+
+  @Override
+  public String toString() {
+    return value;
+  }
+
+  @JsonCreator
+  public static UID of(@Nonnull String value) {
+    return new UID(value);
+  }
+
+  public static UID of(@Nonnull UserDetails currentUser) {
+    return new UID(currentUser.getUid());
+  }
+
+  public static UID of(@CheckForNull UidObject object) {
+    return object == null ? null : new UID(object.getUid());
+  }
+
+  public static Set<String> toValueSet(Collection<UID> uids) {
+    return uids.stream().map(UID::getValue).collect(toUnmodifiableSet());
+  }
+
+  public static List<String> toValueList(Collection<UID> uids) {
+    return uids.stream().map(UID::getValue).toList();
+  }
+}

dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SettingKey.java → dhis-2/dhis-api/src/main/java/org/hisp/dhis/setting/SettingKey.java

@@ -256,7 +256,14 @@ public enum SettingKey {
   ANALYTICS_MAX_PERIOD_YEARS_OFFSET("keyAnalyticsPeriodYearsOffset", -1, Integer.class),
 
   /** Max trackedentityinstance records that can be retrieved from database. */
-  TRACKED_ENTITY_MAX_LIMIT("KeyTrackedEntityInstanceMaxLimit", 50000, Integer.class);
+  TRACKED_ENTITY_MAX_LIMIT("KeyTrackedEntityInstanceMaxLimit", 50000, Integer.class),
+
+  NOTIFIER_LOG_LEVEL("notifierLogLevel", "DEBUG", String.class),
+  NOTIFIER_MAX_MESSAGES_PER_JOB("notifierMaxMessagesPerJob", 500, Integer.class),
+  NOTIFIER_MAX_AGE_DAYS("notifierMaxAgeDays", 7, Integer.class),
+  NOTIFIER_MAX_JOBS_PER_TYPE("notifierMaxJobsPerType", 500, Integer.class),
+  NOTIFIER_GIST_OVERVIEW("notifierGistOverview", true, Boolean.class),
+  NOTIFIER_CLEAN_AFTER_IDLE_TIME("notifierCleanAfterIdleTime", 60_000L, Long.class);
 
   private final String name;
 
@@ -324,6 +331,8 @@ public static Serializable getAsRealClass(String name, String value) {
         return Double.valueOf(value);
       } else if (Integer.class.isAssignableFrom(settingClazz)) {
         return Integer.valueOf(value);
+      } else if (Long.class.isAssignableFrom(settingClazz)) {
+        return Long.valueOf(value);
       } else if (Boolean.class.isAssignableFrom(settingClazz)) {
         return Boolean.valueOf(value);
       } else if (Locale.class.isAssignableFrom(settingClazz)) {

dhis-2/dhis-api/src/main/java/org/hisp/dhis/setting/SystemSettingsProvider.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2004-2025, University of Oslo
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * Neither the name of the HISP project nor the names of its contributors may
+ * be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.hisp.dhis.setting;
+
+import java.io.Serializable;
+
+public interface SystemSettingsProvider {
+
+  <T extends Serializable> T getSystemSetting(SettingKey key, Class<T> type);
+
+  /**
+   * Returns the system setting value for the given key. If no value exists, returns the default
+   * value as defined by the given default value.
+   *
+   * @param key the system setting key.
+   * @return the setting value.
+   */
+  <T extends Serializable> T getSystemSetting(SettingKey key, T defaultValue);
+
+  String getStringSetting(SettingKey key);
+
+  Integer getIntegerSetting(SettingKey key);
+
+  int getIntSetting(SettingKey key);
+
+  Boolean getBooleanSetting(SettingKey key);
+
+  boolean getBoolSetting(SettingKey key);
+}

dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/NotifierJobProgress.java

@@ -29,44 +29,55 @@
 
 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.util.concurrent.atomic.AtomicBoolean;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
 import lombok.RequiredArgsConstructor;
+import org.hisp.dhis.jsontree.JsonValue;
 import org.hisp.dhis.system.notification.NotificationDataType;
 import org.hisp.dhis.system.notification.NotificationLevel;
 import org.hisp.dhis.system.notification.Notifier;
 
 /**
  * A {@link JobProgress} implementation that forwards the tracking to a {@link Notifier}. It has no
- * flow control and should be wrapped in a {@link ControlledJobProgress} for that purpose.
+ * flow control and should be wrapped in a {@link RecordingJobProgress} for that purpose.
  *
- * @see ControlledJobProgress
+ * @see RecordingJobProgress
  */
 @RequiredArgsConstructor
 public class NotifierJobProgress implements JobProgress {
-  private final Notifier notifier;
 
+  private final Notifier notifier;
   private final JobConfiguration jobId;
-
+  private final NotificationLevel level;
   private final AtomicBoolean hasCleared = new AtomicBoolean();
 
   private int stageItems;
-
   private int stageItem;
 
-  @Override
-  public boolean isCancellationRequested() {
-    return false;
+  private boolean isLoggedLoop() {
+    return NotificationLevel.LOOP.ordinal() >= level.ordinal();
+  }
+
+  private boolean isLoggedInfo() {
+    return NotificationLevel.INFO.ordinal() >= level.ordinal();
+  }
+
+  private boolean isLoggedError() {
+    return NotificationLevel.ERROR.ordinal() >= level.ordinal();
   }
 
   @Override
-  public void startingProcess(String description) {
+  public void startingProcess(String description, Object... args) {
     String message =
-        isNotEmpty(description) ? description : jobId.getJobType() + " process started";
+        isNotEmpty(description)
+            ? format(description, args)
+            : jobId.getJobType() + " process started";
     if (hasCleared.compareAndSet(false, true)) {
       notifier.clear(jobId);
     }
+    // Note: intentionally no log level check - always log first
     notifier.notify(
         jobId,
         NotificationLevel.INFO,
@@ -77,69 +88,72 @@ public void startingProcess(String description) {
   }
 
   @Override
-  public void completedProcess(String summary) {
-    notifier.notify(jobId, summary, true);
+  public void completedProcess(String summary, Object... args) {
+    // Note: intentionally no log level check - always log last
+    notifier.notify(jobId, format(summary, args), true);
   }
 
   @Override
-  public void failedProcess(String error) {
-    notifier.notify(jobId, NotificationLevel.ERROR, error, true);
+  public void failedProcess(@CheckForNull String error, Object... args) {
+    // Note: intentionally no log level check - always log last
+    notifier.notify(jobId, NotificationLevel.ERROR, format(error, args), true);
   }
 
   @Override
-  public void startingStage(String description, int workItems, FailurePolicy onFailure) {
+  public void startingStage(
+      @Nonnull String description, int workItems, @Nonnull FailurePolicy onFailure) {
     stageItems = workItems;
     stageItem = 0;
-    if (isNotEmpty(description)) {
+    if (isLoggedInfo() && isNotEmpty(description)) {
       notifier.notify(jobId, description);
     }
   }
 
   @Override
-  public void completedStage(String summary) {
-    if (isNotEmpty(summary)) {
-      notifier.notify(jobId, summary);
+  public void completedStage(String summary, Object... args) {
+    if (isLoggedInfo() && isNotEmpty(summary)) {
+      notifier.notify(jobId, format(summary, args));
     }
   }
 
   @Override
-  public void failedStage(String error) {
-    if (isNotEmpty(error)) {
-      notifier.notify(jobId, NotificationLevel.ERROR, error, false);
+  public void failedStage(@Nonnull String error, Object... args) {
+    if (isLoggedError() && isNotEmpty(error)) {
+      notifier.notify(jobId, NotificationLevel.ERROR, format(error, args), false);
     }
   }
 
   @Override
-  public void startingWorkItem(String description, FailurePolicy onFailure) {
-    if (isNotEmpty(description)) {
+  public void startingWorkItem(@Nonnull String description, @Nonnull FailurePolicy onFailure) {
+    if (isLoggedLoop() && isNotEmpty(description)) {
       String nOf = "[" + (stageItems > 0 ? stageItem + "/" + stageItems : "" + stageItem) + "] ";
       notifier.notify(jobId, NotificationLevel.LOOP, nOf + description, false);
     }
     stageItem++;
   }
 
   @Override
-  public void completedWorkItem(String summary) {
-    if (isNotEmpty(summary)) {
+  public void completedWorkItem(String summary, Object... args) {
+    if (isLoggedLoop() && isNotEmpty(summary)) {
       String nOf = "[" + (stageItems > 0 ? stageItem + "/" + stageItems : "" + stageItem) + "] ";
-      notifier.notify(jobId, NotificationLevel.LOOP, nOf + summary, false);
+      notifier.notify(jobId, NotificationLevel.LOOP, nOf + format(summary, args), false);
     }
   }
 
   @Override
-  public void failedWorkItem(String error) {
-    if (isNotEmpty(error)) {
-      notifier.notify(jobId, NotificationLevel.ERROR, error, false);
+  public void failedWorkItem(@Nonnull String error, Object... args) {
+    if (isLoggedError() && isNotEmpty(error)) {
+      notifier.notify(jobId, NotificationLevel.ERROR, format(error, args), false);
     }
   }
 
-  private JsonNode getJobParameterData() {
+  private JsonValue getJobParameterData() {
     JobParameters params = jobId.getJobParameters();
     if (params == null) {
       return null;
     }
     try {
-      return new ObjectMapper().valueToTree(params);
+      return JsonValue.of(new ObjectMapper().writeValueAsString(params));
     } catch (Exception ex) {
       return null;
     }