diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/UID.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/UID.java index 4f6cb79026cf..e089ba727aa2 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/UID.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/common/UID.java @@ -54,14 +54,14 @@ @Getter @EqualsAndHashCode public final class UID implements Serializable { - private static final String VALID_UID_FORMAT = - "UID must be an alphanumeric string of 11 characters starting with a letter."; private final String value; private UID(String value) { if (!CodeGenerator.isValidUid(value)) { - throw new IllegalArgumentException(VALID_UID_FORMAT); + throw new IllegalArgumentException( + "UID must be an alphanumeric string of 11 characters starting with a letter, but was: " + + value); } this.value = value; } diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java index c6441c0442fd..c8dc04189dba 100644 --- a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/scheduling/JobType.java @@ -197,6 +197,8 @@ public boolean isDefaultExecutedByCreator() { } /** + * @implNote since 2.42 all jobs forward to the {@code Notifier} but those not included here use + * {@link org.hisp.dhis.system.notification.NotificationLevel#ERROR}. * @return true, if {@link JobProgress} events should be forwarded to the {@link * org.eclipse.emf.common.notify.Notifier} API, otherwise false */ diff --git a/dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SettingKey.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/setting/SettingKey.java similarity index 97% rename from dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SettingKey.java rename to dhis-2/dhis-api/src/main/java/org/hisp/dhis/setting/SettingKey.java index 94daf4cbf21c..41fede4bab55 100644 --- a/dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SettingKey.java +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/setting/SettingKey.java @@ -53,6 +53,7 @@ import org.hisp.dhis.scheduling.JobConfiguration; import org.hisp.dhis.security.LoginPageLayout; import org.hisp.dhis.sms.config.SmsConfiguration; +import org.hisp.dhis.system.notification.NotificationLevel; /** * @author Lars Helge Overland @@ -295,7 +296,13 @@ public enum SettingKey { * the app does not exist * */ GLOBAL_SHELL_APP_NAME("globalShellAppName", "global-app-shell", String.class, false, false), - ; + + NOTIFIER_LOG_LEVEL("notifierLogLevel", NotificationLevel.DEBUG, NotificationLevel.class), + NOTIFIER_MAX_MESSAGES_PER_JOB("notifierMaxMessagesPerJob", 500, Integer.class), + NOTIFIER_MAX_AGE_DAYS("notifierMaxAgeDays", 7, Integer.class), + NOTIFIER_MAX_JOBS_PER_TYPE("notifierMaxJobsPerType", 500, Integer.class), + NOTIFIER_GIST_OVERVIEW("notifierGistOverview", true, Boolean.class), + NOTIFIER_CLEAN_AFTER_IDLE_TIME("notifierCleanAfterIdleTime", 60_000L, Long.class); private final String name; @@ -363,6 +370,8 @@ public static Serializable getAsRealClass(String name, String value) { return Double.valueOf(value); } else if (Integer.class.isAssignableFrom(settingClazz)) { return Integer.valueOf(value); + } else if (Long.class.isAssignableFrom(settingClazz)) { + return Long.valueOf(value); } else if (Boolean.class.isAssignableFrom(settingClazz)) { return Boolean.valueOf(value); } else if (Locale.class.isAssignableFrom(settingClazz)) { diff --git a/dhis-2/dhis-api/src/main/java/org/hisp/dhis/setting/SystemSettingsProvider.java b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/setting/SystemSettingsProvider.java new file mode 100644 index 000000000000..10175f8ad2c1 --- /dev/null +++ b/dhis-2/dhis-api/src/main/java/org/hisp/dhis/setting/SystemSettingsProvider.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.setting; + +import java.io.Serializable; + +public interface SystemSettingsProvider { + + T getSystemSetting(SettingKey key, Class type); + + /** + * Returns the system setting value for the given key. If no value exists, returns the default + * value as defined by the given default value. + * + * @param key the system setting key. + * @return the setting value. + */ + T getSystemSetting(SettingKey key, T defaultValue); + + String getStringSetting(SettingKey key); + + Integer getIntegerSetting(SettingKey key); + + int getIntSetting(SettingKey key); + + Boolean getBooleanSetting(SettingKey key); + + boolean getBoolSetting(SettingKey key); +} diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java index 19264d6847a1..a0baf9f94e51 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/DefaultJobSchedulerLoopService.java @@ -52,6 +52,7 @@ import org.hisp.dhis.message.MessageService; import org.hisp.dhis.setting.SettingKey; import org.hisp.dhis.setting.SystemSettingManager; +import org.hisp.dhis.system.notification.NotificationLevel; import org.hisp.dhis.system.notification.Notifier; import org.hisp.dhis.user.AuthenticationService; import org.hisp.dhis.user.UserDetails; @@ -258,10 +259,12 @@ private static void logError(String message, Exception ex) { } private JobProgress startRecording(@Nonnull JobConfiguration job, @Nonnull Runnable observer) { - JobProgress tracker = + NotificationLevel level = job.getJobType().isUsingNotifications() - ? new NotifierJobProgress(notifier, job) - : NoopJobProgress.INSTANCE; + ? systemSettings.getSystemSetting( + SettingKey.NOTIFIER_LOG_LEVEL, NotificationLevel.DEBUG) + : NotificationLevel.ERROR; + JobProgress tracker = new NotifierJobProgress(notifier, job, level); boolean logInfoOnDebug = job.getSchedulingType() != SchedulingType.ONCE_ASAP && job.getLastExecuted() != null diff --git a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/NotifierJobProgress.java b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/NotifierJobProgress.java index d31f5e720c0a..0380cc452fa6 100644 --- a/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/NotifierJobProgress.java +++ b/dhis-2/dhis-services/dhis-service-core/src/main/java/org/hisp/dhis/scheduling/NotifierJobProgress.java @@ -29,10 +29,12 @@ import static org.apache.commons.lang3.StringUtils.isNotEmpty; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; +import org.hisp.dhis.jsontree.JsonValue; import org.hisp.dhis.system.notification.NotificationDataType; import org.hisp.dhis.system.notification.NotificationLevel; import org.hisp.dhis.system.notification.Notifier; @@ -45,16 +47,27 @@ */ @RequiredArgsConstructor public class NotifierJobProgress implements JobProgress { - private final Notifier notifier; + private final Notifier notifier; private final JobConfiguration jobId; - + private final NotificationLevel level; private final AtomicBoolean hasCleared = new AtomicBoolean(); private int stageItems; - private int stageItem; + private boolean isLoggedLoop() { + return NotificationLevel.LOOP.ordinal() >= level.ordinal(); + } + + private boolean isLoggedInfo() { + return NotificationLevel.INFO.ordinal() >= level.ordinal(); + } + + private boolean isLoggedError() { + return NotificationLevel.ERROR.ordinal() >= level.ordinal(); + } + @Override public void startingProcess(String description, Object... args) { String message = @@ -64,6 +77,7 @@ public void startingProcess(String description, Object... args) { if (hasCleared.compareAndSet(false, true)) { notifier.clear(jobId); } + // Note: intentionally no log level check - always log first notifier.notify( jobId, NotificationLevel.INFO, @@ -75,40 +89,43 @@ public void startingProcess(String description, Object... args) { @Override public void completedProcess(String summary, Object... args) { + // Note: intentionally no log level check - always log last notifier.notify(jobId, format(summary, args), true); } @Override - public void failedProcess(String error, Object... args) { + public void failedProcess(@CheckForNull String error, Object... args) { + // Note: intentionally no log level check - always log last notifier.notify(jobId, NotificationLevel.ERROR, format(error, args), true); } @Override - public void startingStage(String description, int workItems, FailurePolicy onFailure) { + public void startingStage( + @Nonnull String description, int workItems, @Nonnull FailurePolicy onFailure) { stageItems = workItems; stageItem = 0; - if (isNotEmpty(description)) { + if (isLoggedInfo() && isNotEmpty(description)) { notifier.notify(jobId, description); } } @Override public void completedStage(String summary, Object... args) { - if (isNotEmpty(summary)) { + if (isLoggedInfo() && isNotEmpty(summary)) { notifier.notify(jobId, format(summary, args)); } } @Override - public void failedStage(String error, Object... args) { - if (isNotEmpty(error)) { + public void failedStage(@Nonnull String error, Object... args) { + if (isLoggedError() && isNotEmpty(error)) { notifier.notify(jobId, NotificationLevel.ERROR, format(error, args), false); } } @Override - public void startingWorkItem(String description, FailurePolicy onFailure) { - if (isNotEmpty(description)) { + public void startingWorkItem(@Nonnull String description, @Nonnull FailurePolicy onFailure) { + if (isLoggedLoop() && isNotEmpty(description)) { String nOf = "[" + (stageItems > 0 ? stageItem + "/" + stageItems : "" + stageItem) + "] "; notifier.notify(jobId, NotificationLevel.LOOP, nOf + description, false); } @@ -117,26 +134,26 @@ public void startingWorkItem(String description, FailurePolicy onFailure) { @Override public void completedWorkItem(String summary, Object... args) { - if (isNotEmpty(summary)) { + if (isLoggedLoop() && isNotEmpty(summary)) { String nOf = "[" + (stageItems > 0 ? stageItem + "/" + stageItems : "" + stageItem) + "] "; notifier.notify(jobId, NotificationLevel.LOOP, nOf + format(summary, args), false); } } @Override - public void failedWorkItem(String error, Object... args) { - if (isNotEmpty(error)) { + public void failedWorkItem(@Nonnull String error, Object... args) { + if (isLoggedError() && isNotEmpty(error)) { notifier.notify(jobId, NotificationLevel.ERROR, format(error, args), false); } } - private JsonNode getJobParameterData() { + private JsonValue getJobParameterData() { JobParameters params = jobId.getJobParameters(); if (params == null) { return null; } try { - return new ObjectMapper().valueToTree(params); + return JsonValue.of(new ObjectMapper().writeValueAsString(params)); } catch (Exception ex) { return null; } diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java index b35cb4f7a627..0d28049560dd 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/DefaultDataValueSetService.java @@ -31,9 +31,6 @@ import static org.hisp.dhis.commons.collection.CollectionUtils.isEmpty; import static org.hisp.dhis.commons.util.StreamUtils.wrapAndCheckCompressionFormat; import static org.hisp.dhis.external.conf.ConfigurationKey.CHANGELOG_AGGREGATE; -import static org.hisp.dhis.system.notification.NotificationLevel.ERROR; -import static org.hisp.dhis.system.notification.NotificationLevel.INFO; -import static org.hisp.dhis.system.notification.NotificationLevel.WARN; import static org.hisp.dhis.system.util.ValidationUtils.dataValueIsZeroAndInsignificant; import com.fasterxml.jackson.databind.ObjectMapper; @@ -107,8 +104,6 @@ import org.hisp.dhis.system.callable.CategoryOptionComboAclCallable; import org.hisp.dhis.system.callable.IdentifiableObjectCallable; import org.hisp.dhis.system.callable.PeriodCallable; -import org.hisp.dhis.system.notification.NotificationLevel; -import org.hisp.dhis.system.notification.Notifier; import org.hisp.dhis.system.util.Clock; import org.hisp.dhis.system.util.CsvUtils; import org.hisp.dhis.system.util.ValidationUtils; @@ -158,8 +153,6 @@ public class DefaultDataValueSetService implements DataValueSetService { private final I18nManager i18nManager; - private final Notifier notifier; - private final InputUtils inputUtils; private final CalendarService calendarService; @@ -572,7 +565,7 @@ public ImportSummary importDataValueSetCsv(InputStream in, ImportOptions options @Override @Transactional public ImportSummary importDataValueSet(DataValueSet dataValueSet, ImportOptions options) { - return importDataValueSet(options, null, () -> new SimpleDataValueSetReader(dataValueSet)); + return importDataValueSet(options, () -> new SimpleDataValueSetReader(dataValueSet)); } @Override @@ -581,7 +574,6 @@ public ImportSummary importDataValueSetXml( InputStream in, ImportOptions options, JobConfiguration id) { return importDataValueSet( options, - id, () -> new XmlDataValueSetReader(XMLFactory.getXMLReader(wrapAndCheckCompressionFormat(in)))); } @@ -591,9 +583,7 @@ public ImportSummary importDataValueSetXml( public ImportSummary importDataValueSetJson( InputStream in, ImportOptions options, JobConfiguration id) { return importDataValueSet( - options, - id, - () -> new JsonDataValueSetReader(wrapAndCheckCompressionFormat(in), jsonMapper)); + options, () -> new JsonDataValueSetReader(wrapAndCheckCompressionFormat(in), jsonMapper)); } @Override @@ -602,7 +592,6 @@ public ImportSummary importDataValueSetCsv( InputStream in, ImportOptions options, JobConfiguration id) { return importDataValueSet( options, - id, () -> new CsvDataValueSetReader( CsvUtils.getReader(wrapAndCheckCompressionFormat(in)), options)); @@ -612,7 +601,7 @@ public ImportSummary importDataValueSetCsv( @Transactional public ImportSummary importDataValueSetPdf( InputStream in, ImportOptions options, JobConfiguration id) { - return importDataValueSet(options, id, () -> new PdfDataValueSetReader(in)); + return importDataValueSet(options, () -> new PdfDataValueSetReader(in)); } @Override @@ -622,35 +611,23 @@ public ImportSummary importDataValueSetPdf(InputStream in, ImportOptions options } private ImportSummary importDataValueSet( - ImportOptions options, JobConfiguration id, Callable createReader) { + ImportOptions options, Callable createReader) { options = ObjectUtils.firstNonNull(options, ImportOptions.getDefaultImportOptions()); - notifier.clear(id); - try (BatchHandler dvBatch = batchHandlerFactory.createBatchHandler(DataValueBatchHandler.class); BatchHandler dvaBatch = batchHandlerFactory.createBatchHandler(DataValueAuditBatchHandler.class); DataValueSetReader reader = createReader.call()) { - ImportSummary summary = importDataValueSet(options, id, reader, dvBatch, dvaBatch); + ImportSummary summary = importDataValueSet(options, reader, dvBatch, dvaBatch); dvBatch.flush(); dvaBatch.flush(); - NotificationLevel notificationLevel = options.getNotificationLevel(INFO); - notifier - .notify(id, notificationLevel, "Import done", true) - .addJobSummary(id, notificationLevel, summary, ImportSummary.class); - return summary; } catch (Exception ex) { log.error(DebugUtils.getStackTrace(ex)); - ImportSummary summary = - new ImportSummary(ImportStatus.ERROR, "The import process failed: " + ex.getMessage()); - notifier - .notify(id, ERROR, "Process failed: " + ex.getMessage(), true) - .addJobSummary(id, summary, ImportSummary.class); - return summary; + return new ImportSummary(ImportStatus.ERROR, "The import process failed: " + ex.getMessage()); } } @@ -673,7 +650,6 @@ private ImportSummary importDataValueSet( */ private ImportSummary importDataValueSet( ImportOptions options, - JobConfiguration id, DataValueSetReader reader, BatchHandler dataValueBatchHandler, BatchHandler auditBatchHandler) { @@ -687,8 +663,6 @@ private ImportSummary importDataValueSet( new Clock(log) .startClock() .logTime("Starting data value import, options: " + context.getImportOptions()); - NotificationLevel notificationLevel = options.getNotificationLevel(INFO); - notifier.notify(id, notificationLevel, "Process started"); // --------------------------------------------------------------------- // Heat caches @@ -706,15 +680,11 @@ private ImportSummary importDataValueSet( if (importValidator.abortDataSetImport(dataValueSet, context, dataSetContext)) { context.getSummary().setDescription("Import process was aborted"); - notifier - .notify(id, WARN, "Import process aborted", true) - .addJobSummary(id, context.getSummary(), ImportSummary.class); return context.getSummary(); } LocalDate completeDate = getCompletionDate(dataValueSet.getCompleteDate()); if (dataSetContext.getDataSet() != null && completeDate != null) { - notifier.notify(id, notificationLevel, "Completing data set"); handleComplete( dataSetContext.getDataSet(), Date.from(completeDate.atStartOfDay().atZone(ZoneId.systemDefault()).toInstant()), @@ -736,7 +706,6 @@ private ImportSummary importDataValueSet( Date now = new Date(); clock.logTime("Validated outer meta-data"); - notifier.notify(id, notificationLevel, "Importing data values"); List values = dataValueSet.getDataValues(); int index = 0; diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/DataValueSetImportJob.java b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/DataValueSetImportJob.java index 40132f913aa0..08b11205cc7a 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/DataValueSetImportJob.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/main/java/org/hisp/dhis/dxf2/datavalueset/tasks/DataValueSetImportJob.java @@ -27,6 +27,8 @@ */ package org.hisp.dhis.dxf2.datavalueset.tasks; +import static org.hisp.dhis.system.notification.NotificationLevel.INFO; + import java.io.IOException; import java.io.InputStream; import lombok.RequiredArgsConstructor; @@ -42,6 +44,8 @@ import org.hisp.dhis.scheduling.JobConfiguration; import org.hisp.dhis.scheduling.JobProgress; import org.hisp.dhis.scheduling.JobType; +import org.hisp.dhis.system.notification.NotificationLevel; +import org.hisp.dhis.system.notification.Notifier; import org.springframework.stereotype.Component; /** @@ -54,6 +58,7 @@ public class DataValueSetImportJob implements Job { private final FileResourceService fileResourceService; private final DataValueSetService dataValueSetService; private final AdxDataService adxDataService; + private final Notifier notifier; @Override public JobType getJobType() { @@ -111,6 +116,9 @@ public void execute(JobConfiguration jobId, JobProgress progress) { count.getUpdated(), count.getDeleted(), count.getIgnored()); + + NotificationLevel level = options == null ? INFO : options.getNotificationLevel(INFO); + notifier.addJobSummary(jobId, level, summary, ImportSummary.class); } catch (IOException ex) { progress.failedProcess(ex); } diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetServiceImportTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetServiceImportTest.java index 0f29277c2d38..8e8b8ecc17e6 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetServiceImportTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/datavalueset/DataValueSetServiceImportTest.java @@ -30,8 +30,6 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -140,10 +138,6 @@ void testImportDataValuesUpdatedSkipNoChange() { when(batchHandlerFactory.createBatchHandler(DataValueAuditBatchHandler.class)) .thenReturn(auditBatchHandler); - when(notifier.clear(any())).thenReturn(notifier); - when(notifier.notify(any(), any(), anyString())).thenReturn(notifier); - when(notifier.notify(any(), any(), anyString(), anyBoolean())).thenReturn(notifier); - DataSet dataSet = createDataSet('A', new MonthlyPeriodType()); dataSet.setUid("pBOMPrpg1QX"); when(identifiableObjectManager.getObject(DataSet.class, IdScheme.UID, "pBOMPrpg1QX")) diff --git a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/TrackerCrudTest.java b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/TrackerCrudTest.java index 65315240c90b..979c44360426 100644 --- a/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/TrackerCrudTest.java +++ b/dhis-2/dhis-services/dhis-service-dxf2/src/test/java/org/hisp/dhis/dxf2/deprecated/tracker/TrackerCrudTest.java @@ -138,7 +138,7 @@ public void setUp() { any(JobConfiguration.class), any(NotificationLevel.class), anyString(), anyBoolean())) .thenReturn(notifier); when(notifier.notify(any(JobConfiguration.class), anyString())).thenReturn(notifier); - when(notifier.clear(any())).thenReturn(notifier); + when(notifier.clear(any(JobConfiguration.class))).thenReturn(notifier); when(defaultTrackedEntityInstanceService.getTrackedEntity(trackedEntityInstanceUid, user)) .thenReturn(new TrackedEntity()); diff --git a/dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SystemSettingManager.java b/dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SystemSettingManager.java index 4bf988078d81..9858b5042e13 100644 --- a/dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SystemSettingManager.java +++ b/dhis-2/dhis-services/dhis-service-setting/src/main/java/org/hisp/dhis/setting/SystemSettingManager.java @@ -40,7 +40,7 @@ * @author Stian Strandli * @author Lars Helge Overland */ -public interface SystemSettingManager { +public interface SystemSettingManager extends SystemSettingsProvider { /** * Saves the given system setting key and value. * @@ -74,6 +74,7 @@ public interface SystemSettingManager { * @param key the system setting key. * @return the setting value. */ + @Override default T getSystemSetting(SettingKey key, Class type) { if (type != key.getClazz()) { throw new IllegalArgumentException( @@ -84,15 +85,6 @@ default T getSystemSetting(SettingKey key, Class typ return type.cast(getSystemSetting(key, key.getDefaultValue())); } - /** - * Returns the system setting value for the given key. If no value exists, returns the default - * value as defined by the given default value. - * - * @param key the system setting key. - * @return the setting value. - */ - T getSystemSetting(SettingKey key, T defaultValue); - /** * Returns the translation for given setting key and locale or empty Optional if no translation is * available or setting key is not translatable. @@ -138,22 +130,27 @@ default T getSystemSetting(SettingKey key, Class typ // Typed methods // ------------------------------------------------------------------------- + @Override default String getStringSetting(SettingKey key) { return getSystemSetting(key, String.class); } + @Override default Integer getIntegerSetting(SettingKey key) { return getSystemSetting(key, Integer.class); } + @Override default int getIntSetting(SettingKey key) { return getSystemSetting(key, Integer.class); } + @Override default Boolean getBooleanSetting(SettingKey key) { return getSystemSetting(key, Boolean.class); } + @Override default boolean getBoolSetting(SettingKey key) { return Boolean.TRUE.equals(getSystemSetting(key, Boolean.class)); } diff --git a/dhis-2/dhis-support/dhis-support-system/pom.xml b/dhis-2/dhis-support/dhis-support-system/pom.xml index f1e4fcef329b..32982af8bb9d 100644 --- a/dhis-2/dhis-support/dhis-support-system/pom.xml +++ b/dhis-2/dhis-support/dhis-support-system/pom.xml @@ -39,6 +39,10 @@ org.hisp.dhis dhis-support-external + + org.hisp.dhis + json-tree + @@ -285,6 +289,11 @@ hamcrest test + + org.testcontainers + testcontainers + test + diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/configuration/NotifierConfiguration.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/configuration/NotifierConfiguration.java index 8fe118d1c88b..30dda80283aa 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/configuration/NotifierConfiguration.java +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/configuration/NotifierConfiguration.java @@ -30,9 +30,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.hisp.dhis.condition.RedisDisabledCondition; import org.hisp.dhis.condition.RedisEnabledCondition; -import org.hisp.dhis.system.notification.InMemoryNotifier; +import org.hisp.dhis.setting.SystemSettingsProvider; +import org.hisp.dhis.system.notification.DefaultNotifier; +import org.hisp.dhis.system.notification.InMemoryNotifierStore; import org.hisp.dhis.system.notification.Notifier; -import org.hisp.dhis.system.notification.RedisNotifier; +import org.hisp.dhis.system.notification.NotifierStore; +import org.hisp.dhis.system.notification.RedisNotifierStore; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; @@ -53,13 +56,17 @@ public class NotifierConfiguration { @SuppressWarnings("unchecked") @Bean("notifier") @Conditional(RedisEnabledCondition.class) - public Notifier redisNotifier(ObjectMapper objectMapper) { - return new RedisNotifier((RedisTemplate) redisTemplate, objectMapper); + public Notifier redisNotifier( + ObjectMapper objectMapper, SystemSettingsProvider settingsProvider) { + NotifierStore store = new RedisNotifierStore((RedisTemplate) redisTemplate); + return new DefaultNotifier(store, objectMapper, settingsProvider, System::currentTimeMillis); } @Bean("notifier") @Conditional(RedisDisabledCondition.class) - public Notifier inMemoryNotifier() { - return new InMemoryNotifier(); + public Notifier inMemoryNotifier( + ObjectMapper objectMapper, SystemSettingsProvider settingsProvider) { + return new DefaultNotifier( + new InMemoryNotifierStore(), objectMapper, settingsProvider, System::currentTimeMillis); } } diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/DefaultNotifier.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/DefaultNotifier.java new file mode 100644 index 000000000000..59dee8ec66cb --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/DefaultNotifier.java @@ -0,0 +1,335 @@ +/* + * Copyright (c) 2004-2022, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.system.notification; + +import static java.lang.System.currentTimeMillis; +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.toCollection; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Date; +import java.util.Deque; +import java.util.EnumMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BiFunction; +import java.util.function.LongSupplier; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +import org.hisp.dhis.common.UID; +import org.hisp.dhis.jsontree.JsonValue; +import org.hisp.dhis.scheduling.JobConfiguration; +import org.hisp.dhis.scheduling.JobType; +import org.hisp.dhis.setting.SettingKey; +import org.hisp.dhis.setting.SystemSettingsProvider; + +/** + * Implements the {@link Notifier} API on top of a {@link NotifierStore}. + * + *

Incoming {@link Notification} messages are decoupled from the caller thread by pushing them + * into a {@link BlockingQueue}. If the queue is full the message is dropped after a short timeout + * to not block the source thread. A worker thread constantly takes massages from the queue and + * pushes them into the {@link NotifierStore.NotificationStore}. This has the big advantage that + * only one thread ever writes to the store preventing any issues caused by concurrent writes. Also, + * this removes the burden from the source thread to be slowed down by the cost of storing the + * messages. Last but not least it also makes sure that any error from persisting messages does not + * affect the source thread. + * + *

Please note that this implementation does not filter on {@link NotificationLevel} other than + * {@link NotificationLevel#OFF}. The {@link org.hisp.dhis.setting.SettingKey#NOTIFIER_LOG_LEVEL} is + * applied in the {@link org.hisp.dhis.scheduling.JobProgress} that forwards to the {@link Notifier} + * API. This is for backwards compatibility of usages of the {@link Notifier} API outside of job + * execution and to keep the {@link NotificationLevel} consistent over a job run. + * + * @since 2.42 + * @author Jan Bernitt + */ +@Slf4j +public class DefaultNotifier implements Notifier { + + private final NotifierStore store; + private final ObjectMapper jsonMapper; + private final SystemSettingsProvider settingsProvider; + private final LongSupplier clock; + private final BlockingQueue> pushToStore; + private final AtomicBoolean cleaning = new AtomicBoolean(); + + private int maxMessagesPerJob; + private long cleanAfterIdleTime; + private long settingsSince; + + public DefaultNotifier( + NotifierStore store, + ObjectMapper jsonMapper, + SystemSettingsProvider settingsProvider, + LongSupplier clock) { + this.store = store; + this.jsonMapper = jsonMapper; + this.settingsProvider = settingsProvider; + this.clock = clock; + this.maxMessagesPerJob = + settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_MESSAGES_PER_JOB); + this.cleanAfterIdleTime = + settingsProvider.getSystemSetting(SettingKey.NOTIFIER_CLEAN_AFTER_IDLE_TIME, Long.class); + this.settingsSince = currentTimeMillis(); + this.pushToStore = new ArrayBlockingQueue<>(2048); + Executors.newSingleThreadExecutor().execute(this::asyncPushToStore); + } + + @Override + public boolean isIdle() { + return pushToStore.isEmpty() && !cleaning.get(); + } + + private void asyncPushToStore() { + long lastTime = 0; + while (true) { + try { + Entry e = pushToStore.poll(cleanAfterIdleTime, TimeUnit.MILLISECONDS); + while (e != null + && e.getValue().getLevel() == NotificationLevel.LOOP + && !pushToStore.isEmpty()) { + e = pushToStore.take(); // skip pushing LOOP entries which are already outdated + } + if (e != null) { + Notification n = e.getValue(); + // make sure notifications are at least 1ms apart + // also, make sure the time is actually reflecting the insert order + if (n.getTime().getTime() <= lastTime) n.setTime(new Date(lastTime + 1)); + asyncPushToStore(e.getKey(), n); + lastTime = n.getTime().getTime(); + } else { + // when there hasn't been any notifications lately + // the poll times out; it is a good time to run some cleanup + asyncAutomaticCleanup(); + } + } catch (InterruptedException ex) { + log.warn("Notification lost due interruption."); + Thread.currentThread().interrupt(); + } catch (Exception ex) { + log.warn("Notification lost due to: " + ex.getMessage()); + } + } + } + + private void asyncPushToStore(UID job, Notification n) { + NotifierStore.NotificationStore list = store.notifications(n.getCategory(), job); + + boolean limit = true; + Notification newest = list.getNewest(); + if (newest != null && newest.getLevel() == NotificationLevel.LOOP) { + list.removeNewest(); + limit = false; // we deleted one so there is room for one + } + if (limit) { + int size = list.size(); + int maxSize = getMaxMessagesPerJob(); + if (size + 1 > maxSize) { + list.removeOldest(size + 1 - maxSize); + } + } + list.add(n); + logNotificationAdded(n); + } + + private static void logNotificationAdded(Notification n) { + String message = n.getMessage(); + if (message == null || message.isEmpty()) return; + switch (n.getLevel()) { + case LOOP, DEBUG -> log.debug(message); + case INFO -> log.info(message); + case WARN -> log.warn(message); + case ERROR -> log.error(message); + } + } + + private void asyncAutomaticCleanup() { + cleaning.set(true); + try { + store.capMaxAge(settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_AGE_DAYS)); + store.capMaxCount(settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_JOBS_PER_TYPE)); + } finally { + cleaning.set(false); + } + } + + /** This is potentially called so often that it is cached here refreshing it every 10 seconds. */ + private int getMaxMessagesPerJob() { + long now = currentTimeMillis(); + if (now - settingsSince > 10_000) { + maxMessagesPerJob = settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_MESSAGES_PER_JOB); + cleanAfterIdleTime = + settingsProvider.getSystemSetting(SettingKey.NOTIFIER_CLEAN_AFTER_IDLE_TIME, Long.class); + settingsSince = now; + } + return maxMessagesPerJob; + } + + @Override + public Notifier notify( + JobConfiguration id, + @Nonnull NotificationLevel level, + String message, + boolean completed, + NotificationDataType dataType, + JsonValue data) { + if (id == null || level.isOff()) return this; + + Date now = new Date(clock.getAsLong()); + Notification n = + new Notification(level, id.getJobType(), now, message, completed, dataType, data); + + try { + if (!pushToStore.offer(Map.entry(UID.of(id.getUid()), n), 50, TimeUnit.MILLISECONDS)) + log.warn("Notification lost due to timeout: " + n); + } catch (InterruptedException e) { + log.warn("Notification lost due to interruption: " + n); + Thread.currentThread().interrupt(); + } + return this; + } + + @Override + public Map>> getNotifications(Boolean gist) { + Map>> res = new EnumMap<>(JobType.class); + for (JobType jobType : JobType.values()) { + Map> byJobId = getNotificationsByJobType(jobType, gist); + if (!byJobId.isEmpty()) res.put(jobType, byJobId); + } + return res; + } + + @Override + public Deque getNotificationsByJobId(JobType jobType, String jobId) { + return getAllNotificationsByJobId(jobType, UID.of(jobId)); + } + + @Nonnull + private Deque getAllNotificationsByJobId(JobType jobType, UID job) { + return store + .notifications(jobType, job) + .listNewestFirst() + .collect(toCollection(LinkedList::new)); + } + + private Deque getGistNotificationsByJobId(JobType jobType, UID job) { + Deque res = new LinkedList<>(); + NotifierStore.NotificationStore notifications = store.notifications(jobType, job); + Notification newest = notifications.getNewest(); + if (newest == null) return res; + // newest goes first + res.addFirst(newest); + Notification oldest = notifications.getOldest(); + if (oldest != null && !newest.getId().equals(oldest.getId())) res.addLast(oldest); + return res; + } + + @Override + public Map> getNotificationsByJobType(JobType jobType, Boolean gist) { + if (gist == null) gist = settingsProvider.getBoolSetting(SettingKey.NOTIFIER_GIST_OVERVIEW); + BiFunction> read = + gist ? this::getGistNotificationsByJobId : this::getAllNotificationsByJobId; + Map> res = new LinkedHashMap<>(); + store.notifications(jobType).stream() + .sorted(comparing(NotifierStore.NotificationStore::ageTimestamp).reversed()) + .forEach(s -> res.put(s.job().getValue(), read.apply(s.type(), s.job()))); + return res; + } + + @Override + public void clear() { + store.clear(); + } + + @Override + public void clear(@Nonnull JobType type) { + store.clear(type); + } + + @Override + public void clear(@Nonnull JobType type, @Nonnull UID job) { + store.clear(type, job); + } + + @Override + public void capMaxAge(int maxAge) { + store.capMaxAge(maxAge); + } + + @Override + public void capMaxCount(int maxCount) { + store.capMaxCount(maxCount); + } + + @Override + public void capMaxAge(int maxAge, @Nonnull JobType type) { + store.capMaxAge(maxAge, type); + } + + @Override + public void capMaxCount(int maxCount, @Nonnull JobType type) { + store.capMaxCount(maxCount, type); + } + + @Override + public Notifier addJobSummary( + JobConfiguration id, NotificationLevel level, T summary, Class type) { + if (id == null || level == null || level.isOff() || !type.equals(summary.getClass())) + return this; + + try { + store + .summary(id.getJobType(), UID.of(id.getUid())) + .set(JsonValue.of(jsonMapper.writeValueAsString(summary))); + } catch (Exception ex) { + log.warn("Summary lost due to: " + ex.getMessage()); + } + return this; + } + + @Override + public Map getJobSummariesForJobType(JobType jobType) { + Map res = new LinkedHashMap<>(); + store.summaries(jobType).stream() + .sorted(comparing(NotifierStore.SummaryStore::ageTimestamp).reversed()) + .forEach(s -> res.put(s.job().getValue(), s.get())); + return res; + } + + @Override + public JsonValue getJobSummaryByJobId(JobType jobType, String jobId) { + return store.summary(jobType, UID.of(jobId)).get(); + } +} diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/InMemoryNotifier.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/InMemoryNotifier.java deleted file mode 100644 index 08747e29a424..000000000000 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/InMemoryNotifier.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2004-2022, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.system.notification; - -import com.fasterxml.jackson.databind.JsonNode; -import java.util.Date; -import java.util.Deque; -import java.util.Map; -import javax.annotation.Nonnull; -import lombok.extern.slf4j.Slf4j; -import org.hisp.dhis.scheduling.JobConfiguration; -import org.hisp.dhis.scheduling.JobType; - -/** - * @author Lars Helge Overland - */ -@Slf4j -public class InMemoryNotifier implements Notifier { - public static final int MAX_POOL_TYPE_SIZE = 500; - - private final NotificationMap notificationMap = new NotificationMap(MAX_POOL_TYPE_SIZE); - - @Override - public Notifier notify( - JobConfiguration id, - @Nonnull NotificationLevel level, - String message, - boolean completed, - NotificationDataType dataType, - JsonNode data) { - if (id != null && !level.isOff()) { - Notification notification = - new Notification(level, id.getJobType(), new Date(), message, completed, dataType, data); - - notificationMap.add(id, notification); - - NotificationLoggerUtil.log(log, level, message); - } - - return this; - } - - @Override - public Map>> getNotifications() { - return notificationMap.getNotifications(); - } - - @Override - public Deque getNotificationsByJobId(JobType jobType, String jobId) { - return notificationMap.getNotificationsByJobId(jobType, jobId); - } - - @Override - public Map> getNotificationsByJobType(JobType jobType) { - return notificationMap.getNotificationsWithType(jobType); - } - - @Override - public Notifier clear(JobConfiguration id) { - if (id != null) { - notificationMap.clear(id); - } - - return this; - } - - @Override - public Notifier addJobSummary( - JobConfiguration id, NotificationLevel level, T jobSummary, Class jobSummaryType) { - if (id != null && !(level != null && level.isOff())) { - notificationMap.addSummary(id, jobSummary); - } - - return this; - } - - @Override - public Map getJobSummariesForJobType(JobType jobType) { - return notificationMap.getJobSummariesForJobType(jobType); - } - - @Override - public Object getJobSummaryByJobId(JobType jobType, String jobId) { - return notificationMap.getSummary(jobType, jobId); - } -} diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/InMemoryNotifierStore.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/InMemoryNotifierStore.java new file mode 100644 index 000000000000..bad600f1e24f --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/InMemoryNotifierStore.java @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.system.notification; + +import static java.lang.System.currentTimeMillis; +import static java.util.function.Predicate.not; + +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import org.hisp.dhis.common.UID; +import org.hisp.dhis.jsontree.JsonValue; +import org.hisp.dhis.scheduling.JobType; + +/** + * A {@link NotifierStore} that stores values in memory. + * + * @implNote The implementation does need to use concurrent collections because there can be + * concurrent reads (in particular iteration) while writes are running. + * @since 2.42 + * @author Jan Bernitt + */ +public final class InMemoryNotifierStore implements NotifierStore { + + private final Map> summaryStores = new ConcurrentHashMap<>(); + private final Map> notificationStores = + new ConcurrentHashMap<>(); + + @Nonnull + @Override + public NotificationStore notifications(@Nonnull JobType type, @Nonnull UID job) { + return notificationStores + .computeIfAbsent(type, k -> new ConcurrentHashMap<>()) + .computeIfAbsent( + job, k -> new InMemoryNotificationStore(type, job, new ConcurrentLinkedDeque<>())); + } + + @Nonnull + @Override + public SummaryStore summary(@Nonnull JobType type, @Nonnull UID job) { + return summaryStores + .computeIfAbsent(type, k -> new ConcurrentHashMap<>()) + .computeIfAbsent(job, k -> new InMemorySummaryStore(type, job, new AtomicReference<>())); + } + + @Nonnull + @Override + public List notifications(@Nonnull JobType type) { + Map byId = notificationStores.get(type); + return byId == null + ? List.of() + : byId.values().stream().filter(not(NotificationStore::isEmpty)).toList(); + } + + @Nonnull + @Override + public List summaries(@Nonnull JobType type) { + Map byId = summaryStores.get(type); + return byId == null + ? List.of() + : byId.values().stream().filter(SummaryStore::isPresent).toList(); + } + + @Override + public void clear() { + notificationStores.clear(); + summaryStores.clear(); + } + + @Override + public void clear(@Nonnull JobType type) { + notificationStores.remove(type); + summaryStores.remove(type); + } + + @Override + public void clear(@Nonnull JobType type, @Nonnull UID job) { + Map map = notificationStores.get(type); + if (map != null) map.remove(job); + map = summaryStores.get(type); + if (map != null) map.remove(job); + } + + private record InMemoryNotificationStore(JobType type, UID job, Deque collection) + implements NotificationStore { + + @Override + public boolean isEmpty() { + return collection.isEmpty(); + } + + @Override + public int size() { + return collection.size(); + } + + @Override + public void removeNewest() { + collection.pollFirst(); + } + + @Override + public void removeOldest(int n) { + Notification n0 = collection.pollLast(); + for (int i = 0; i < n; i++) collection.pollLast(); + if (n0 != null) collection.addLast(n0); + } + + @Override + public void add(@Nonnull Notification n) { + collection.addFirst(n); + } + + @CheckForNull + @Override + public Notification getNewest() { + return collection.peekFirst(); + } + + @CheckForNull + @Override + public Notification getOldest() { + return collection.peekLast(); + } + + @Nonnull + @Override + public Stream listNewestFirst() { + return collection.stream(); + } + } + + private record Summary(long ageTimestamp, JsonValue value) {} + + private record InMemorySummaryStore(JobType type, UID job, AtomicReference

summary) + implements SummaryStore { + + @Override + public long ageTimestamp() { + Summary s = summary.get(); + return s == null ? 0L : s.ageTimestamp(); + } + + @CheckForNull + @Override + public JsonValue get() { + Summary s = summary.get(); + return s == null ? null : s.value(); + } + + @Override + public void set(@Nonnull JsonValue summary) { + this.summary.set(new Summary(currentTimeMillis(), summary)); + } + } +} diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/Notification.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/Notification.java index d54bb2606bec..e8efc4be9153 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/Notification.java +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/Notification.java @@ -28,7 +28,6 @@ package org.hisp.dhis.system.notification; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement; import java.util.Date; @@ -38,6 +37,7 @@ import lombok.ToString; import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.common.DxfNamespaces; +import org.hisp.dhis.jsontree.JsonValue; import org.hisp.dhis.scheduling.JobType; /** @@ -54,7 +54,7 @@ public class Notification implements Comparable { @ToString.Include private JobType category; - @ToString.Include private Date time; + @Nonnull @ToString.Include private Date time; @ToString.Include private String message; @@ -62,7 +62,7 @@ public class Notification implements Comparable { private NotificationDataType dataType; - private JsonNode data; + private JsonValue data; // ------------------------------------------------------------------------- // Constructors @@ -70,16 +70,17 @@ public class Notification implements Comparable { public Notification() { this.uid = CodeGenerator.generateUid(); + this.time = new Date(); } public Notification( NotificationLevel level, JobType category, - Date time, + @Nonnull Date time, String message, boolean completed, NotificationDataType dataType, - JsonNode data) { + JsonValue data) { this.uid = CodeGenerator.generateUid(); this.level = level; this.category = category; @@ -118,6 +119,7 @@ public JobType getCategory() { return category; } + @Nonnull @JsonProperty @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) public Date getTime() { @@ -144,7 +146,7 @@ public NotificationDataType getDataType() { @JsonProperty @JacksonXmlProperty(namespace = DxfNamespaces.DXF_2_0) - public JsonNode getData() { + public JsonValue getData() { return data; } diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/NotificationMap.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/NotificationMap.java deleted file mode 100644 index 8054579a2dbc..000000000000 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/NotificationMap.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2004-2022, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.system.notification; - -import static java.util.Arrays.stream; -import static java.util.Collections.unmodifiableMap; -import static java.util.stream.Collectors.toMap; - -import java.util.Deque; -import java.util.EnumMap; -import java.util.LinkedList; -import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.function.ToIntFunction; -import java.util.function.UnaryOperator; -import lombok.RequiredArgsConstructor; -import org.hisp.dhis.scheduling.JobConfiguration; -import org.hisp.dhis.scheduling.JobType; - -/** - * Keeps an ordered list of {@link Notification}s and/or summary {@link Object}s per {@link JobType} - * and {@link JobConfiguration} UID. - * - *

For each {@link Pool} the capacity of entries is capped at a fixed maximum capacity. - * - *

If maximum capacity is reached and another entry is added for that {@link JobType}'s {@link - * Pool} the overall oldest entry is removed and the new one added. This means entries of another - * {@link JobConfiguration} can be removed as long as they belong to the same {@link JobType}. - * - * @author Henning Håkonsen - * @author Jan Bernitt (thread-safety) - */ -public class NotificationMap { - private final Map>> notificationsByJobType = - new EnumMap<>(JobType.class); - - private final Map> summariesByJobType = new EnumMap<>(JobType.class); - - @RequiredArgsConstructor - static final class Pool { - private final int capacity; - - /** - * Remembers the job IDs in the order messages came in, so we know what job ID has the oldest - * message. When {@link #capacity} is exceeded that value is responsible for clearing memory. - */ - private final Deque jobIdsInOrder = new ConcurrentLinkedDeque<>(); - - private final Map valuesByJobId = new ConcurrentHashMap<>(); - - synchronized void remove(String jobId) { - jobIdsInOrder.removeIf(jobId::equals); - valuesByJobId.remove(jobId); - } - - synchronized void add( - String jobId, UnaryOperator limit, ToIntFunction size, UnaryOperator adder) { - if (jobIdsInOrder.size() >= capacity) { - String jobIdToShorten = jobIdsInOrder.removeLast(); - valuesByJobId.compute(jobIdToShorten, (key, values) -> limit.apply(values)); - } - int sizeBefore = size.applyAsInt(valuesByJobId.get(jobId)); - int sizeAfter = - size.applyAsInt(valuesByJobId.compute(jobId, (key, values) -> adder.apply(values))); - if (sizeAfter > sizeBefore) { - jobIdsInOrder.addFirst(jobId); - } - } - } - - NotificationMap(int capacity) { - stream(JobType.values()) - .forEach( - jobType -> { - notificationsByJobType.put(jobType, new Pool<>(capacity)); - summariesByJobType.put(jobType, new Pool<>(capacity)); - }); - } - - public Map>> getNotifications() { - return notificationsByJobType.entrySet().stream() - .collect(toMap(Entry::getKey, e -> e.getValue().valuesByJobId)); - } - - public Deque getNotificationsByJobId(JobType jobType, String jobId) { - Deque res = notificationsByJobType.get(jobType).valuesByJobId.get(jobId); - // return a defensive copy - return res == null ? new LinkedList<>() : new LinkedList<>(res); - } - - public Map> getNotificationsWithType(JobType jobType) { - return unmodifiableMap(notificationsByJobType.get(jobType).valuesByJobId); - } - - public void add(JobConfiguration configuration, Notification notification) { - String jobId = configuration.getUid(); - if (jobId == null) { - return; - } - notificationsByJobType - .get(configuration.getJobType()) - .add( - jobId, - NotificationMap::withLimit, - notifications -> notifications == null ? 0 : notifications.size(), - notifications -> withAdded(notifications, notification)); - } - - public void addSummary(JobConfiguration configuration, Object summary) { - String jobId = configuration.getUid(); - if (jobId == null) { - return; - } - summariesByJobType - .get(configuration.getJobType()) - .add( - jobId, - currentSummary -> null, - currentSummary -> currentSummary == null ? 0 : 1, - currentSummary -> summary); - } - - public Object getSummary(JobType jobType, String jobId) { - return summariesByJobType.get(jobType).valuesByJobId.get(jobId); - } - - public Map getJobSummariesForJobType(JobType jobType) { - return unmodifiableMap(summariesByJobType.get(jobType).valuesByJobId); - } - - public void clear(JobConfiguration configuration) { - JobType jobType = configuration.getJobType(); - String jobId = configuration.getUid(); - notificationsByJobType.get(jobType).remove(jobId); - summariesByJobType.get(jobType).remove(jobId); - } - - private static Deque withAdded( - Deque notifications, Notification item) { - if (notifications == null) { - notifications = new ConcurrentLinkedDeque<>(); - } - Notification mostRecent = notifications.peekFirst(); - if (mostRecent != null && mostRecent.getLevel() == NotificationLevel.LOOP) { - notifications.pollFirst(); - } - notifications.addFirst(item); - return notifications; - } - - private static Deque withLimit(Deque notifications) { - if (notifications != null) { - Notification mostRecent = notifications.peekFirst(); - if (mostRecent != null && mostRecent.getLevel() == NotificationLevel.LOOP) { - notifications.pollFirst(); - } else { - notifications.pollLast(); - } - if (notifications.isEmpty()) { - return null; - } - } - return notifications; - } -} diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/Notifier.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/Notifier.java index c580b6378b5b..b35e27800710 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/Notifier.java +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/Notifier.java @@ -27,10 +27,12 @@ */ package org.hisp.dhis.system.notification; -import com.fasterxml.jackson.databind.JsonNode; import java.util.Deque; import java.util.Map; +import javax.annotation.CheckForNull; import javax.annotation.Nonnull; +import org.hisp.dhis.common.UID; +import org.hisp.dhis.jsontree.JsonValue; import org.hisp.dhis.scheduling.JobConfiguration; import org.hisp.dhis.scheduling.JobType; @@ -39,6 +41,7 @@ * @author Jan Bernitt (pulled up default methods) */ public interface Notifier { + default Notifier notify(JobConfiguration id, String message) { return notify(id, NotificationLevel.INFO, message, false); } @@ -62,7 +65,7 @@ Notifier notify( String message, boolean completed, NotificationDataType dataType, - JsonNode data); + JsonValue data); default Notifier update(JobConfiguration id, String message) { return update(id, NotificationLevel.INFO, message, false); @@ -84,22 +87,113 @@ default Notifier update( return this; } - Map>> getNotifications(); + /** + * @param gist when true, only the first and last message are included for each job. When {@code + * null} the {@link org.hisp.dhis.setting.SettingKey#NOTIFIER_GIST_OVERVIEW} is used. + * @return a map with notifications for all job types and jobs + */ + Map>> getNotifications(@CheckForNull Boolean gist); Deque getNotificationsByJobId(JobType jobType, String jobId); - Map> getNotificationsByJobType(JobType jobType); - - Notifier clear(JobConfiguration id); - - default Notifier addJobSummary(JobConfiguration id, T jobSummary, Class jobSummaryType) { - return addJobSummary(id, NotificationLevel.INFO, jobSummary, jobSummaryType); + /** + * @param jobType include jobs of this type in the result + * @param gist when true, only the first and last message are included for each job. When {@code + * null} the {@link org.hisp.dhis.setting.SettingKey#NOTIFIER_GIST_OVERVIEW} is used. + * @return a map with notifications for all jobs of the provided type + */ + Map> getNotificationsByJobType( + JobType jobType, @CheckForNull Boolean gist); + + default Notifier addJobSummary(JobConfiguration id, T summary, Class type) { + return addJobSummary(id, NotificationLevel.INFO, summary, type); } Notifier addJobSummary( - JobConfiguration id, NotificationLevel level, T jobSummary, Class jobSummaryType); - - Map getJobSummariesForJobType(JobType jobType); - - Object getJobSummaryByJobId(JobType jobType, String jobId); + JobConfiguration id, NotificationLevel level, T summary, Class type); + + Map getJobSummariesForJobType(JobType jobType); + + JsonValue getJobSummaryByJobId(JobType jobType, String jobId); + + /** + * @since 2.42 + * @return true, when currently no messages are being processed and no automatic cleanup is + * running in the background. + */ + boolean isIdle(); + + /* + Cleanup API + */ + + /** + * Removes all data for the specified job (both notifications and summary) + * + * @param type of the job to clear + * @param job ID of the job to clear + * @since 2.42 + */ + void clear(@Nonnull JobType type, @Nonnull UID job); + + /** + * Removes all data for all jobs of the specified type. + * + * @param type of the jobs to clear + * @since 2.42 + */ + void clear(@Nonnull JobType type); + + /** + * Removes all data (of all jobs and job types). + * + * @since 2.42 + */ + void clear(); + + /** + * Removes all data for jobs of the given type unless they are younger than the given max age in + * days + * + * @param maxAge keep the data for jobs from the most recent days + * @param type of jobs to check + * @since 2.42 + */ + void capMaxAge(int maxAge, @Nonnull JobType type); + + /** + * Removes all data unless the job is younger than the given max age in days + * + * @param maxAge keep the data for jobs from the most recent days + * @since 2.42 + */ + void capMaxAge(int maxAge); + + /** + * Removes all data for jobs of the given type except for the most recent n ones. + * + * @param maxCount number to keep (most recent first) + * @param type of jobs to check + * @since 2.42 + */ + void capMaxCount(int maxCount, @Nonnull JobType type); + + /** + * Removes all data for jobs except for the most recent n ones for each type. + * + * @param maxCount number to keep (most recent first) + * @since 2.42 + */ + void capMaxCount(int maxCount); + + /** + * For backwards compatibility (not having to update all callers) + * + * @param config the job to clear all data for + * @return itself for chaining + */ + default Notifier clear(@CheckForNull JobConfiguration config) { + if (config != null) clear(config.getJobType(), UID.of(config.getUid())); + return this; + } } diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/NotifierStore.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/NotifierStore.java new file mode 100644 index 000000000000..776f885a6921 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/NotifierStore.java @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.system.notification; + +import static java.lang.System.currentTimeMillis; +import static java.util.Comparator.comparingLong; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import org.hisp.dhis.common.UID; +import org.hisp.dhis.jsontree.JsonValue; +import org.hisp.dhis.scheduling.JobType; + +/** + * Persistence API for {@link Notification}s. + * + * @author Jan Bernitt + * @since 2.42 + */ +public interface NotifierStore { + + @Nonnull + NotificationStore notifications(@Nonnull JobType type, @Nonnull UID job); + + @Nonnull + List notifications(@Nonnull JobType type); + + @Nonnull + SummaryStore summary(@Nonnull JobType type, @Nonnull UID job); + + @Nonnull + List summaries(@Nonnull JobType type); + + /** + * Removes all data for the specified job (both notifications and summary) + * + * @param type of the job to clear + * @param job ID of the job to clear + */ + void clear(@Nonnull JobType type, @Nonnull UID job); + + /** + * Removes all data for all jobs of the specified type. + * + * @param type of the jobs to clear + */ + void clear(@Nonnull JobType type); + + /** Removes all data (of all jobs and job types). */ + void clear(); + + /** + * Removes all data for jobs except for the most recent n ones for each type. + * + * @param maxCount number to keep (most recent first) + */ + default void capMaxCount(int maxCount) { + if (maxCount <= 0) { + clear(); + return; + } + Stream.of(JobType.values()).forEach(type -> capMaxCount(maxCount, type)); + } + + /** + * Removes all data unless the job is younger than the given max age in days + * + * @param maxAge keep the data for jobs from the most recent days + */ + default void capMaxAge(int maxAge) { + Stream.of(JobType.values()).forEach(type -> capMaxAge(maxAge, type)); + } + + /** + * Removes all data for jobs of the given type except for the most recent n ones. + * + * @param type of jobs to check + * @param maxCount number to keep (most recent first) + */ + default void capMaxCount(int maxCount, @Nonnull JobType type) { + if (maxCount <= 0) { + clear(type); + return; + } + List stores = notifications(type); + if (stores.size() <= maxCount) return; + int remove = stores.size() - maxCount; + stores.stream() + .sorted(comparingLong(NotificationStore::ageTimestamp)) + .limit(remove) + .forEach(s -> clear(s.type(), s.job())); + } + + /** + * Removes all data for jobs of the given type unless they are younger than the given max age in + * days + * + * @param type of jobs to check + * @param maxAge keep the data for jobs from the most recent days + */ + default void capMaxAge(int maxAge, @Nonnull JobType type) { + List stores = notifications(type); + long removeBefore = currentTimeMillis() - TimeUnit.DAYS.toMillis(maxAge); + stores.stream() + .filter(s -> s.ageTimestamp() < removeBefore) + .forEach(s -> clear(s.type(), s.job())); + } + + /** The common API for {@link NotificationStore} and {@link SummaryStore}. */ + sealed interface PerJobStore { + + /** + * @return the {@link JobType} the store deals with + */ + JobType type(); + + /** + * @return the ID of the {@link org.hisp.dhis.scheduling.JobConfiguration} the store contains + * data for + */ + UID job(); + + /** + * @return the most recent timestamp the data in this store was touched + */ + long ageTimestamp(); + } + + /** + * API to interact with the collection for a single list of {@link Notification}s for a specific + * {@link JobType} and job by its {@link UID}. The collection is sorted by the {@link + * Notification#getTime()} value. + * + * @implSpec Implementations must handle reading and writing methods being used concurrently, but + * writes ({@link #add(Notification)}) will never be called concurrently + */ + non-sealed interface NotificationStore extends PerJobStore { + + default boolean isEmpty() { + return size() == 0; + } + + /** + * @return number of {@link Notification}s in the collection of this store + */ + int size(); + + /** Removes the {@link Notification} most recently added */ + void removeNewest(); + + /** + * This does however retain the very first message. + * + * @param n number of messages to remove from the start (oldest) (but not the 1st) + */ + void removeOldest(int n); + + /** + * Adds a new {@link Notification} which is expected to be younger in terms of {@link + * Notification#getTime()} as any entry already added before. + * + * @param n the entry to add + */ + void add(@Nonnull Notification n); + + /** + * @return the entry most recently added, or null if the store is empty + */ + @CheckForNull + Notification getNewest(); + + /** + * @return the entry added first (when the store was empty) or null if it still is empty + */ + @CheckForNull + Notification getOldest(); + + /** + * @return a stream of the store entries starting with the one most recently added and ending + * with the one added first, when the store was empty + */ + @Nonnull + Stream listNewestFirst(); + + /** + * @return the timestamp to use when comparing the age of this store to the age of other stores + */ + @Override + default long ageTimestamp() { + Notification newest = getNewest(); + return newest == null ? 0L : newest.getTime().getTime(); + } + } + + /** + * API for a store containing the summary value associated with a specific {@link + * org.hisp.dhis.scheduling.JobConfiguration}. + */ + non-sealed interface SummaryStore extends PerJobStore { + + default boolean isPresent() { + return get() != null; + } + + /** + * @return the stored summary value, or {@code null} when no such summary was set, or when it + * did expire (got cleaned) + */ + @CheckForNull + JsonValue get(); + + /** + * Set a new summary for a {@link org.hisp.dhis.scheduling.JobConfiguration} run. + * + * @param summary the value to store + */ + void set(@Nonnull JsonValue summary); + } +} diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/RedisNotifier.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/RedisNotifier.java deleted file mode 100644 index 021cf49767b0..000000000000 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/RedisNotifier.java +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright (c) 2004-2022, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.system.notification; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.Deque; -import java.util.EnumMap; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import javax.annotation.Nonnull; -import lombok.extern.slf4j.Slf4j; -import org.hisp.dhis.scheduling.JobConfiguration; -import org.hisp.dhis.scheduling.JobType; -import org.springframework.data.redis.core.RedisTemplate; - -/** - * Notifier implementation backed by redis. It holds 2 types of data. Notifications and Summaries. - * Since order of the Notifications and Summaries are important, (to limit the maximum number of - * objects held), we use a combination of "Sorted Sets" , "HashMaps" and "Values" (data structures - * in redis) to have a similar behaviour as InMemoryNotifier. - * - * @author Ameen Mohamed - */ -@Slf4j -public class RedisNotifier implements Notifier { - private static final String NOTIFIER_ERROR = "Redis Notifier error:%s"; - - private final RedisTemplate redisTemplate; - - private static final String NOTIFICATIONS_KEY_PREFIX = "notifications:"; - - private static final String NOTIFICATION_ORDER_KEY_PREFIX = "notification:order:"; - - private static final String SUMMARIES_KEY_PREFIX = "summaries:"; - - private static final String SUMMARIES_KEY_ORDER_PREFIX = "summary:order:"; - - private static final String SUMMARY_TYPE_PREFIX = "summary:type:"; - - private static final String COLON = ":"; - - private static final int MAX_POOL_TYPE_SIZE = 500; - - private final ObjectMapper jsonMapper; - - public RedisNotifier(RedisTemplate redisTemplate, ObjectMapper jsonMapper) { - this.redisTemplate = redisTemplate; - this.jsonMapper = jsonMapper; - } - - // ------------------------------------------------------------------------- - // Notifier implementation backed by Redis - // ------------------------------------------------------------------------- - - @Override - public Notifier notify( - JobConfiguration id, - @Nonnull NotificationLevel level, - String message, - boolean completed, - NotificationDataType dataType, - JsonNode data) { - if (id != null && !level.isOff()) { - Notification notification = - new Notification(level, id.getJobType(), new Date(), message, completed, dataType, data); - - String notificationKey = generateNotificationKey(id.getJobType(), id.getUid()); - String notificationOrderKey = generateNotificationOrderKey(id.getJobType()); - - Date now = new Date(); - - try { - Long zCard = redisTemplate.boundZSetOps(notificationOrderKey).zCard(); - if (zCard != null && zCard >= MAX_POOL_TYPE_SIZE) { - Set deleteKeys = redisTemplate.boundZSetOps(notificationOrderKey).range(0, 0); - if (deleteKeys != null) { - redisTemplate.delete(deleteKeys); - } - redisTemplate.boundZSetOps(notificationOrderKey).removeRange(0, 0); - } - - redisTemplate - .boundZSetOps(notificationKey) - .add(jsonMapper.writeValueAsString(notification), now.getTime()); - redisTemplate.boundZSetOps(notificationOrderKey).add(id.getUid(), now.getTime()); - } catch (JsonProcessingException ex) { - log.warn(String.format(NOTIFIER_ERROR, ex.getMessage())); - } - - NotificationLoggerUtil.log(log, level, message); - } - return this; - } - - @Override - public Map>> getNotifications() { - Map>> notifications = new EnumMap<>(JobType.class); - for (JobType jobType : JobType.values()) { - notifications.put(jobType, getNotificationsByJobType(jobType)); - } - return notifications; - } - - @Override - public Deque getNotificationsByJobId(JobType jobType, String jobId) { - Set notifications = - redisTemplate.boundZSetOps(generateNotificationKey(jobType, jobId)).range(0, -1); - if (notifications == null) return new LinkedList<>(); - List res = new ArrayList<>(); - notifications.forEach( - notification -> - executeLogErrors( - () -> res.add(jsonMapper.readValue(notification, Notification.class)))); - Collections.sort(res); - return new LinkedList<>(res); - } - - @Override - public Map> getNotificationsByJobType(JobType jobType) { - Set keys = - redisTemplate.boundZSetOps(generateNotificationOrderKey(jobType)).range(0, -1); - if (keys == null || keys.isEmpty()) return Map.of(); - LinkedHashMap> res = new LinkedHashMap<>(); - keys.forEach(jobId -> res.put(jobId, getNotificationsByJobId(jobType, jobId))); - return res; - } - - @Override - public Notifier clear(JobConfiguration id) { - if (id != null) { - redisTemplate.delete(generateNotificationKey(id.getJobType(), id.getUid())); - redisTemplate.boundHashOps(generateSummaryKey(id.getJobType())).delete(id.getUid()); - redisTemplate.boundZSetOps(generateNotificationOrderKey(id.getJobType())).remove(id.getUid()); - redisTemplate.boundZSetOps(generateSummaryOrderKey(id.getJobType())).remove(id.getUid()); - } - - return this; - } - - @Override - public Notifier addJobSummary( - JobConfiguration id, NotificationLevel level, T jobSummary, Class jobSummaryType) { - if (id != null - && !(level != null && level.isOff()) - && jobSummary.getClass().equals(jobSummaryType)) { - String summaryKey = generateSummaryKey(id.getJobType()); - try { - String existingSummaryTypeStr = - redisTemplate.boundValueOps(generateSummaryTypeKey(id.getJobType())).get(); - if (existingSummaryTypeStr == null) { - redisTemplate - .boundValueOps(generateSummaryTypeKey(id.getJobType())) - .set(jobSummaryType.getName()); - } else { - Class existingSummaryType = Class.forName(existingSummaryTypeStr); - if (!existingSummaryType.equals(jobSummaryType)) { - return this; - } - } - - String summaryOrderKey = generateSummaryOrderKey(id.getJobType()); - Long zCard = redisTemplate.boundZSetOps(summaryOrderKey).zCard(); - if (zCard != null && zCard >= MAX_POOL_TYPE_SIZE) { - Set summaryKeysToBeDeleted = - redisTemplate.boundZSetOps(summaryOrderKey).range(0, 0); - redisTemplate.boundZSetOps(summaryOrderKey).removeRange(0, 0); - if (summaryKeysToBeDeleted != null) { - summaryKeysToBeDeleted.forEach(d -> redisTemplate.boundHashOps(summaryKey).delete(d)); - } - } - redisTemplate - .boundHashOps(summaryKey) - .put(id.getUid(), jsonMapper.writeValueAsString(jobSummary)); - Date now = new Date(); - - redisTemplate.boundZSetOps(summaryOrderKey).add(id.getUid(), now.getTime()); - - } catch (JsonProcessingException | ClassNotFoundException ex) { - log.warn(String.format(NOTIFIER_ERROR, ex.getMessage())); - } - } - return this; - } - - @Override - public Map getJobSummariesForJobType(JobType jobType) { - Map jobSummariesForType = new LinkedHashMap<>(); - try { - String existingSummaryTypeStr = - redisTemplate.boundValueOps(generateSummaryTypeKey(jobType)).get(); - if (existingSummaryTypeStr == null) { - return jobSummariesForType; - } - - Class existingSummaryType = Class.forName(existingSummaryTypeStr); - Map serializedSummaryMap = - redisTemplate.boundHashOps(generateSummaryKey(jobType)).entries(); - if (serializedSummaryMap != null) { - serializedSummaryMap.forEach( - (k, v) -> - executeLogErrors( - () -> - jobSummariesForType.put( - (String) k, jsonMapper.readValue((String) v, existingSummaryType)))); - } - } catch (ClassNotFoundException ex) { - log.warn(String.format(NOTIFIER_ERROR, ex.getMessage())); - } - - return jobSummariesForType; - } - - @Override - public Object getJobSummaryByJobId(JobType jobType, String jobId) { - String existingSummaryTypeStr = - redisTemplate.boundValueOps(generateSummaryTypeKey(jobType)).get(); - if (existingSummaryTypeStr == null) { - return null; - } - try { - Class existingSummaryType = Class.forName(existingSummaryTypeStr); - Object serializedSummary = redisTemplate.boundHashOps(generateSummaryKey(jobType)).get(jobId); - - return serializedSummary != null - ? jsonMapper.readValue((String) serializedSummary, existingSummaryType) - : null; - } catch (IOException | ClassNotFoundException ex) { - log.warn(String.format(NOTIFIER_ERROR, ex.getMessage())); - } - return null; - } - - private static String generateNotificationKey(JobType jobType, String jobUid) { - return NOTIFICATIONS_KEY_PREFIX + jobType.toString() + COLON + jobUid; - } - - private static String generateNotificationOrderKey(JobType jobType) { - return NOTIFICATION_ORDER_KEY_PREFIX + jobType.toString(); - } - - private static String generateSummaryKey(JobType jobType) { - return SUMMARIES_KEY_PREFIX + jobType.toString(); - } - - private static String generateSummaryOrderKey(JobType jobType) { - return SUMMARIES_KEY_ORDER_PREFIX + jobType.toString(); - } - - private static String generateSummaryTypeKey(JobType jobType) { - return SUMMARY_TYPE_PREFIX + jobType.toString(); - } - - private interface Operation { - - void run() throws Exception; - } - - private static void executeLogErrors(Operation operation) { - try { - operation.run(); - } catch (Exception ex) { - log.warn(String.format(NOTIFIER_ERROR, ex.getMessage())); - } - } -} diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/RedisNotifierStore.java b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/RedisNotifierStore.java new file mode 100644 index 000000000000..e076bb1a14b0 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/RedisNotifierStore.java @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.system.notification; + +import static java.lang.System.currentTimeMillis; + +import java.util.Date; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; +import org.hisp.dhis.common.UID; +import org.hisp.dhis.jsontree.Json; +import org.hisp.dhis.jsontree.JsonMixed; +import org.hisp.dhis.jsontree.JsonNumber; +import org.hisp.dhis.jsontree.JsonObject; +import org.hisp.dhis.jsontree.JsonString; +import org.hisp.dhis.jsontree.JsonValue; +import org.hisp.dhis.scheduling.JobType; +import org.hisp.dhis.util.DateUtils; +import org.springframework.data.redis.core.BoundHashOperations; +import org.springframework.data.redis.core.BoundZSetOperations; +import org.springframework.data.redis.core.RedisOperations; + +/** + * Provides Redis backed implementation of the {@link NotifierStore}. + * + * @since 2.42 + * @author Jan Bernitt + */ +@RequiredArgsConstructor +public class RedisNotifierStore implements NotifierStore { + + private final RedisOperations redis; + + @Nonnull + @Override + public NotificationStore notifications(@Nonnull JobType type, @Nonnull UID job) { + return notifications(type, notificationsKeyOf(type.name(), job.getValue())); + } + + @Nonnull + @Override + public List notifications(@Nonnull JobType type) { + Set keys = redis.keys(notificationsKeyOf(type.name(), "*")); + if (keys == null || keys.isEmpty()) return List.of(); + return keys.stream().map(key -> notifications(type, key)).toList(); + } + + @Nonnull + private NotificationStore notifications(@Nonnull JobType type, String key) { + UID job = UID.of(key.substring(key.lastIndexOf(':') + 1)); + return new RedisNotificationStore(type, job, redis.boundZSetOps(key)); + } + + @Nonnull + @Override + public SummaryStore summary(@Nonnull JobType type, @Nonnull UID job) { + return new RedisSummaryStore(type, job, redis.boundHashOps(summariesKeyOf(type.name()))); + } + + @Nonnull + @Override + public List summaries(@Nonnull JobType type) { + BoundHashOperations table = + redis.boundHashOps(summariesKeyOf(type.name())); + Set keys = table.keys(); + if (keys == null || keys.isEmpty()) return List.of(); + return keys.stream().map(job -> summary(type, UID.of(job))).toList(); + } + + @Override + public void clear() { + clear(notificationsKeyOf("*", null)); + clear(summariesKeyOf("*")); + } + + @Override + public void clear(@Nonnull JobType type) { + clear(notificationsKeyOf(type.name(), "*")); + clear(summariesKeyOf(type.name())); + } + + @Override + public void clear(@Nonnull JobType type, @Nonnull UID job) { + redis.delete(notificationsKeyOf(type.name(), job.getValue())); + redis.boundHashOps(summariesKeyOf(type.name())).delete(job.getValue()); + } + + private void clear(String pattern) { + Set keys = redis.keys(pattern); + if (keys == null || keys.isEmpty()) return; + redis.delete(keys); + } + + /** + * Stores {@link Notification}s (as JSON) in a redis ZSET with the {@link Notification#getTime()} + * used as score. + */ + private record RedisNotificationStore( + JobType type, UID job, BoundZSetOperations collection) + implements NotificationStore { + + @Override + public int size() { + Long size = collection.zCard(); + return size == null ? 0 : size.intValue(); + } + + @Override + public void removeNewest() { + collection.removeRange(-1, -1); + } + + @Override + public void removeOldest(int n) { + collection.removeRange(1, n); // keep the first + } + + @Override + public void add(@Nonnull Notification n) { + collection.add(toJson(n), n.getTime().getTime()); + } + + @CheckForNull + @Override + public Notification getNewest() { + return getEntry(-1); // last in zset + } + + @CheckForNull + @Override + public Notification getOldest() { + return getEntry(0); // first in zset + } + + @CheckForNull + private Notification getEntry(int index) { + Set values = collection.range(index, index); + if (values == null || values.isEmpty()) return null; + return fromJson(type, values.stream().findFirst().orElse(null)); + } + + @Nonnull + @Override + public Stream listNewestFirst() { + Set newestFirst = collection.reverseRange(0, -1); + if (newestFirst == null) return Stream.empty(); + return newestFirst.stream().map(n -> fromJson(type, n)); + } + + private Notification fromJson(JobType jobType, String json) { + if (json == null || json.isEmpty()) return null; + JsonObject src = JsonMixed.of(json); + Notification dest = new Notification(); + dest.setCategory(jobType); + JsonString level = src.getString("level"); + dest.setLevel( + level.isUndefined() ? NotificationLevel.INFO : level.parsed(NotificationLevel::valueOf)); + String id = src.getString("id").string(); + dest.setUid(id); + dest.setCompleted(src.getBoolean("completed").booleanValue(false)); + JsonValue time = src.get("time"); + dest.setTime( + time.isNumber() + ? new Date(time.as(JsonNumber.class).longValue()) + : DateUtils.parseDate(time.as(JsonString.class).string())); + dest.setMessage(src.getString("message").string()); + dest.setDataType(src.getString("dataType").parsed(NotificationDataType::valueOf)); + JsonValue data = src.get("data"); + if (!data.isUndefined()) dest.setData(data); + return dest; + } + + /** + * The idea is to make the JSON as small as possible. Therefore, uid is excluded (duplicate of + * id), time is added as timestamp number, level is only added then not INFO, completed is only + * added when true, data and dataType are only added when set. The category is skipped entirely + * as it can be recovered from the redis key/query. + * + * @param n a notification to serialize + * @return the notification as minimal JSON + */ + private String toJson(Notification n) { + return Json.object( + obj -> { + obj.addString("id", n.getId()) + .addNumber("time", n.getTime().getTime()) + .addString("message", n.getMessage()); + if (n.getLevel() != NotificationLevel.INFO) + obj.addString("level", n.getLevel().name()); + if (n.isCompleted()) obj.addBoolean("completed", true); + if (n.getDataType() != null) { + obj.addString("dataType", n.getDataType().name()); + if (n.getData() != null && !n.getData().isUndefined()) + obj.addMember("data", n.getData().node()); + } + }) + .toJson(); + } + } + + /** + * Stores summary objects as JSON ina redis H table. All summaries of a {@link JobType} are stored + * in the same table with the keys being the {@link UID} of the job. + */ + private record RedisSummaryStore( + JobType type, UID job, BoundHashOperations table) + implements SummaryStore { + + @Override + public long ageTimestamp() { + String json = table.get(job.getValue()); + if (json == null) return 0L; + JsonNumber ageTimestamp = JsonMixed.of(json).getNumber("ageTimestamp"); + return ageTimestamp.isUndefined() ? 0L : ageTimestamp.longValue(); + } + + @CheckForNull + @Override + public JsonValue get() { + String json = table.get(job.getValue()); + if (json == null) return null; + JsonMixed root = JsonMixed.of(json); + if (root.get("ageTimestamp").isUndefined()) return root; + return root.get("value"); + } + + @Override + public void set(@Nonnull JsonValue summary) { + table.put( + job().getValue(), + Json.object( + obj -> + obj.addMember("value", summary.node()) + .addNumber("ageTimestamp", currentTimeMillis())) + .toJson()); + } + } + + private static String notificationsKeyOf(@Nonnull String jobType, @CheckForNull String jobUid) { + return jobUid == null + ? "notifications:%s".formatted(jobType) + : "notifications:%s:%s".formatted(jobType, jobUid); + } + + private static String summariesKeyOf(@Nonnull String jobType) { + return "summaries:%s".formatted(jobType); + } +} diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/FakeRedis.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/FakeRedis.java new file mode 100644 index 000000000000..e7752bedd751 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/FakeRedis.java @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.system.notification; + +import static java.util.stream.Collectors.toUnmodifiableSet; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.quality.Strictness.LENIENT; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.Map; +import java.util.Set; +import java.util.function.LongSupplier; +import java.util.stream.Stream; +import org.hisp.dhis.setting.SystemSettingsProvider; +import org.mockito.MockSettings; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.springframework.data.redis.core.BoundHashOperations; +import org.springframework.data.redis.core.BoundZSetOperations; +import org.springframework.data.redis.core.RedisOperations; + +/** + * Implementation of {@link RedisOperations} that can be used to simulate redis during tests. + * + *

It only supports the operations actually used by the {@link RedisNotifierStore}. + * + *

Since the redis API interface are huge {@link Mockito} is used not as a mocking to but as a + * way of wiring API calls to handwritten implementation methods. + * + * @author Jan Bernitt + */ +record FakeRedis( + Map zSets, + Map hTables, + RedisOperations api) { + + static Notifier notifier(SystemSettingsProvider settings, LongSupplier clock) { + return new DefaultNotifier( + new RedisNotifierStore(new FakeRedis().api()), new ObjectMapper(), settings, clock); + } + + FakeRedis() { + this(new HashMap<>(), new HashMap<>(), redisOps()); + } + + FakeRedis { + when(api.keys(anyString())).thenAnswer(this::keys); + when(api.boundZSetOps(anyString())).thenAnswer(this::boundZSetOps); + when(api.boundHashOps(anyString())).thenAnswer(this::boundHashOps); + when(api.delete(anyString())).thenAnswer(this::delete); + when(api.delete(anyCollection())).thenAnswer(this::deleteAll); + } + + private Set keys(InvocationOnMock i) { + String pattern = i.getArgument(0, String.class); + return Stream.concat( + zSets.entrySet().stream() + .filter(e -> !e.getValue().entries.isEmpty() && patternMatches(pattern, e.getKey())) + .map(Map.Entry::getKey), + hTables.entrySet().stream() + .filter(e -> !e.getValue().entries.isEmpty() && patternMatches(pattern, e.getKey())) + .map(Map.Entry::getKey)) + .collect(toUnmodifiableSet()); + } + + private static boolean patternMatches(String pattern, String key) { + return key.matches(pattern.replace("*", ".*")); + } + + private boolean delete(InvocationOnMock i) { + String key = i.getArgument(0, String.class); + boolean zSetDeleted = zSets.containsKey(key) && !zSets.remove(key).entries.isEmpty(); + boolean hTableDeleted = hTables.containsKey(key) && !hTables.remove(key).entries.isEmpty(); + return zSetDeleted || hTableDeleted; + } + + private long deleteAll(InvocationOnMock i) { + @SuppressWarnings("unchecked") + Collection keys = i.getArgument(0, Collection.class); + int size = zSets.size() + hTables.size(); + keys.forEach(zSets::remove); + keys.forEach(hTables::remove); + // Note: the size is off if the set/table were empty + // but handling that seems unnecessary at this point + return size - (zSets.size() + hTables.size()); + } + + private BoundZSetOperations boundZSetOps(InvocationOnMock i) { + String key = i.getArgument(0, String.class); + return zSets.computeIfAbsent(key, k -> new FakeZSet(new LinkedList<>(), boundZSetOps())).api(); + } + + private BoundHashOperations boundHashOps(InvocationOnMock i) { + String key = i.getArgument(0, String.class); + return hTables + .computeIfAbsent(key, k -> new FakeHashTable(new HashMap<>(), boundHashOps())) + .api(); + } + + record FakeZSet(LinkedList entries, BoundZSetOperations api) { + + FakeZSet { + when(api.zCard()).thenAnswer(i -> (long) entries.size()); + when(api.removeRange(anyLong(), anyLong())).thenAnswer(this::removeRange); + when(api.add(anyString(), anyDouble())).thenAnswer(this::add); + when(api.range(anyLong(), anyLong())).then(this::range); + when(api.reverseRange(anyLong(), anyLong())).then(this::reverseRange); + } + + private long removeRange(InvocationOnMock i) { + int start = i.getArgument(0, Long.class).intValue(); + int end = i.getArgument(1, Long.class).intValue(); + if (start < 0) start = entries.size() + start; + if (end < 0) end = entries.size() + end; + int len = end - start + 1; + for (int j = 0; j < len; j++) entries.remove(start); + return len; + } + + private boolean add(InvocationOnMock i) { + String value = i.getArgument(0, String.class); + // Note: the score is not supported since we only append with increased score + // always adding last is the same as sorting by score + entries.addLast(value); + return true; + } + + private Set range(InvocationOnMock i) { + int start = i.getArgument(0, Long.class).intValue(); + int end = i.getArgument(1, Long.class).intValue(); + return range(start, end); + } + + private Set range(int start, int end) { + int size = entries.size(); + if (start < 0) start = size + start; + if (end < 0) end = size + end; + if (start >= size || start < 0) return Set.of(); + if (end >= size) end = size - 1; + return new LinkedHashSet<>(entries.subList(start, end + 1)); + } + + private Set reverseRange(InvocationOnMock i) { + int start = i.getArgument(0, Long.class).intValue(); + int end = i.getArgument(1, Long.class).intValue(); + LinkedList tmp = new LinkedList<>(); + range(-end - 1, -start - 1).forEach(tmp::addFirst); + return new LinkedHashSet<>(tmp); + } + } + + record FakeHashTable( + Map entries, BoundHashOperations api) { + + FakeHashTable { + doAnswer( + i -> { + put(i); + return null; + }) + .when(api) + .put(anyString(), anyString()); + when(api.get(anyString())).thenAnswer(this::get); + } + + private void put(InvocationOnMock i) { + String key = i.getArgument(0, String.class); + String value = i.getArgument(1, String.class); + entries.put(key, value); + } + + private String get(InvocationOnMock i) { + String key = i.getArgument(0, String.class); + return entries.get(key); + } + } + + private static final MockSettings SETTINGS = Mockito.withSettings().strictness(LENIENT); + + @SuppressWarnings("unchecked") + private static RedisOperations redisOps() { + return mock(RedisOperations.class, SETTINGS); + } + + @SuppressWarnings("unchecked") + private static BoundHashOperations boundHashOps() { + return mock(BoundHashOperations.class, SETTINGS); + } + + @SuppressWarnings("unchecked") + private static BoundZSetOperations boundZSetOps() { + return mock(BoundZSetOperations.class, SETTINGS); + } +} diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/FakeRedisTest.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/FakeRedisTest.java new file mode 100644 index 000000000000..0f0499e1ae2e --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/FakeRedisTest.java @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.system.notification; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Set; +import org.junit.jupiter.api.Test; +import org.springframework.data.redis.core.BoundHashOperations; +import org.springframework.data.redis.core.BoundZSetOperations; +import org.springframework.data.redis.core.RedisOperations; + +/** + * Tests basic correctness of the {@link FakeRedis}. + * + * @author Jan Bernitt + * @since 2.42 + */ +class FakeRedisTest { + + private final RedisOperations redis = new FakeRedis().api(); + + @Test + void testKeys_readingDoesNotAddKeys() { + BoundZSetOperations set = redis.boundZSetOps("test"); + assertEquals(0L, set.zCard()); + assertEquals(Set.of(), redis.keys("*")); + assertEquals(Set.of(), redis.keys("test")); + } + + @Test + void testKeys_zSet() { + BoundZSetOperations set1 = redis.boundZSetOps("test:x"); + set1.add("1", 1); + assertEquals(Set.of("test:x"), redis.keys("*")); + assertEquals(Set.of("test:x"), redis.keys("test:*")); + assertEquals(Set.of("test:x"), redis.keys("test:x")); + + BoundZSetOperations set2 = redis.boundZSetOps("foo:bar:7"); + set2.add("2", 2); + assertEquals(Set.of("test:x", "foo:bar:7"), redis.keys("*")); + assertEquals(Set.of("foo:bar:7"), redis.keys("foo:*")); + assertEquals(Set.of("foo:bar:7"), redis.keys("foo:bar:*")); + assertEquals(Set.of("foo:bar:7"), redis.keys("foo:bar:7")); + assertEquals(Set.of(), redis.keys("foo:bar:9")); + } + + @Test + void testKeys_hashTable() { + BoundHashOperations table1 = redis.boundHashOps("table"); + table1.put("a", "b"); + assertEquals(Set.of("table"), redis.keys("*")); + assertEquals(Set.of("table"), redis.keys("table")); + + BoundHashOperations table2 = redis.boundHashOps("summaries:id"); + table2.put("c", "d"); + assertEquals(Set.of("table", "summaries:id"), redis.keys("*")); + assertEquals(Set.of("table"), redis.keys("table")); + assertEquals(Set.of("summaries:id"), redis.keys("summaries:*")); + assertEquals(Set.of("summaries:id"), redis.keys("summaries:id")); + } + + @Test + @SuppressWarnings("DataFlowIssue") + void testDelete_NonExistingHasNoEffect() { + BoundZSetOperations set1 = redis.boundZSetOps("empty:s"); + assertEquals(0L, set1.zCard()); + assertFalse(redis.delete("empty:s")); + assertFalse(redis.delete("x")); + + BoundHashOperations table1 = redis.boundHashOps("empty:t"); + assertNull(table1.get("x")); + assertFalse(redis.delete("empty:t")); + assertFalse(redis.delete("x")); + } + + @Test + @SuppressWarnings("DataFlowIssue") + void testDelete_byKey() { + BoundZSetOperations set1 = redis.boundZSetOps("my:set"); + set1.add("val", 42); + set1.add("val2", 43); + assertEquals(Set.of("my:set"), redis.keys("*")); + assertTrue(redis.delete("my:set")); + assertEquals(Set.of(), redis.keys("*")); + } + + @Test + void testDelete_byKeys() { + BoundZSetOperations set1 = redis.boundZSetOps("my:set"); + set1.add("val", 42); + set1.add("val2", 43); + BoundHashOperations table1 = redis.boundHashOps("my:table"); + table1.put("a", "b"); + + assertEquals(Set.of("my:set", "my:table"), redis.keys("my:*")); + assertEquals(2, redis.delete(Set.of("my:set", "my:table"))); + assertEquals(Set.of(), redis.keys("*")); + } + + @Test + void testZCard() { + BoundZSetOperations set1 = redis.boundZSetOps("my:set"); + assertEquals(0L, set1.zCard()); + + set1.add("1", 1); + assertEquals(1L, set1.zCard()); + + set1.add("2", 2); + assertEquals(2L, set1.zCard()); + + set1.add("1", 3); + assertEquals(3L, set1.zCard()); + + set1.removeRange(0, 0); + assertEquals(2L, set1.zCard()); + } + + @Test + @SuppressWarnings("DataFlowIssue") + void testRange() { + BoundZSetOperations set1 = redis.boundZSetOps("my:set"); + set1.add("1", 1); + set1.add("2", 2); + set1.add("3", 3); + set1.add("4", 4); + + assertEquals(List.of("1", "2", "3", "4"), List.copyOf(set1.range(0, -1))); + assertEquals(List.of("4"), List.copyOf(set1.range(-1, -1))); + assertEquals(List.of("1"), List.copyOf(set1.range(0, 0))); + assertEquals(List.of("2", "3"), List.copyOf(set1.range(1, 2))); + assertEquals(List.of(), List.copyOf(set1.range(10, 20))); + assertEquals(List.of("3", "4"), List.copyOf(set1.range(2, 20))); + } + + @Test + @SuppressWarnings("DataFlowIssue") + void testReverseRange() { + BoundZSetOperations set1 = redis.boundZSetOps("my:set"); + set1.add("1", 1); + set1.add("2", 2); + set1.add("3", 3); + set1.add("4", 4); + + assertEquals(List.of("4", "3", "2", "1"), List.copyOf(set1.reverseRange(0, -1))); + assertEquals(List.of("1"), List.copyOf(set1.reverseRange(-1, -1))); + assertEquals(List.of("4"), List.copyOf(set1.reverseRange(0, 0))); + assertEquals(List.of("3", "2"), List.copyOf(set1.reverseRange(1, 2))); + } + + @Test + @SuppressWarnings("DataFlowIssue") + void testRemoveRange() { + BoundZSetOperations set1 = redis.boundZSetOps("my:set"); + set1.add("1", 1); + set1.add("2", 2); + set1.add("3", 3); + set1.add("4", 4); + + assertEquals(1L, set1.removeRange(1, 1)); + assertEquals(List.of("1", "3", "4"), List.copyOf(set1.range(0, -1))); + assertEquals(1L, set1.removeRange(-1, -1)); + assertEquals(List.of("1", "3"), List.copyOf(set1.range(0, -1))); + assertEquals(1L, set1.removeRange(0, 0)); + assertEquals(List.of("3"), List.copyOf(set1.range(0, -1))); + } + + @Test + void testPut() { + BoundHashOperations table1 = redis.boundHashOps("my:table"); + + table1.put("a", "b"); + assertEquals("b", table1.get("a")); + + table1.put("b", "x"); + assertEquals("x", table1.get("b")); + } +} diff --git a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/NotificationLoggerUtil.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/InMemoryNotifierTest.java similarity index 72% rename from dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/NotificationLoggerUtil.java rename to dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/InMemoryNotifierTest.java index 1419fe23c581..21ba2a2eddb2 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/notification/NotificationLoggerUtil.java +++ b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/InMemoryNotifierTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2022, University of Oslo + * Copyright (c) 2004-2025, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -27,30 +27,21 @@ */ package org.hisp.dhis.system.notification; -import javax.annotation.Nonnull; -import org.slf4j.Logger; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.hisp.dhis.setting.SystemSettingsProvider; /** - * @author Luca Cambi + * Tests the {@link Notifier} API for the {@link InMemoryNotifierStore} implementation. + * + *

The actual tests are in {@link NotifierStoreTest} as they are used for both stores. + * + * @author Jan Bernitt */ -public class NotificationLoggerUtil { - public static void log(Logger logger, @Nonnull NotificationLevel level, String message) { - switch (level) { - case LOOP: - case DEBUG: - logger.debug(message); - break; - case INFO: - logger.info(message); - break; - case WARN: - logger.warn(message); - break; - case ERROR: - logger.error(message); - break; - case OFF: - break; - } +class InMemoryNotifierTest extends NotifierStoreTest { + + @Override + void setUpNotifier(SystemSettingsProvider settings) { + notifier = + new DefaultNotifier(new InMemoryNotifierStore(), new ObjectMapper(), settings, clock); } } diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotificationLoggerUtilTest.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotificationLoggerUtilTest.java deleted file mode 100644 index 9ac4d89e1555..000000000000 --- a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotificationLoggerUtilTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2004-2022, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.system.notification; - -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.slf4j.Logger; - -/** - * @author Luca Cambi - */ -@ExtendWith(MockitoExtension.class) -class NotificationLoggerUtilTest { - - @Mock Logger logger; - - private static String logMessage = "logMessage"; - - @Test - void shouldLogDebugLevel() { - - NotificationLoggerUtil.log(logger, NotificationLevel.DEBUG, logMessage); - - verify(logger).debug(logMessage); - verify(logger, times(0)).info(logMessage); - verify(logger, times(0)).warn(logMessage); - verify(logger, times(0)).error(logMessage); - } - - @Test - void shouldLogInfoLevel() { - - NotificationLoggerUtil.log(logger, NotificationLevel.INFO, logMessage); - - verify(logger, times(0)).debug(logMessage); - verify(logger).info(logMessage); - verify(logger, times(0)).warn(logMessage); - verify(logger, times(0)).error(logMessage); - } - - @Test - void shouldLogWarnLevel() { - - NotificationLoggerUtil.log(logger, NotificationLevel.WARN, logMessage); - - verify(logger, times(0)).debug(logMessage); - verify(logger, times(0)).info(logMessage); - verify(logger).warn(logMessage); - verify(logger, times(0)).error(logMessage); - } - - @Test - void shouldLogErrorLevel() { - - NotificationLoggerUtil.log(logger, NotificationLevel.ERROR, logMessage); - - verify(logger, times(0)).debug(logMessage); - verify(logger, times(0)).info(logMessage); - verify(logger, times(0)).warn(logMessage); - verify(logger).error(logMessage); - } -} diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotificationMapTest.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotificationMapTest.java deleted file mode 100644 index 3af5b206b21a..000000000000 --- a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotificationMapTest.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2004-2022, University of Oslo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * Neither the name of the HISP project nor the names of its contributors may - * be used to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ -package org.hisp.dhis.system.notification; - -import static org.hisp.dhis.scheduling.JobType.DATAVALUE_IMPORT; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.Date; -import java.util.Deque; -import java.util.Map; -import org.hisp.dhis.scheduling.JobConfiguration; -import org.junit.jupiter.api.Test; - -class NotificationMapTest { - private final NotificationMap notifications = new NotificationMap(50); - - @Test - void testFirstSummaryToBeCreatedIsTheFirstOneToBeRemoved() { - final int maxSize = 50; - // Fill the map with jobs - JobConfiguration config = new JobConfiguration(null, DATAVALUE_IMPORT, "userId"); - addSummaryEntries(config, maxSize); - // Add one more - config.setUid(String.valueOf(maxSize)); - notifications.addSummary(config, maxSize); - // Check that oldest job is not in the map anymore - assertFalse(notifications.getJobSummariesForJobType(config.getJobType()).containsKey("0")); - // Add one more - config.setUid(String.valueOf(maxSize + 1)); - notifications.addSummary(config, maxSize + 1); - // Check that oldest job is not in the map anymore - assertFalse(notifications.getJobSummariesForJobType(config.getJobType()).containsKey("1")); - } - - @Test - void testFirstNotificationToBeCreatedIsTheFirstOneToBeRemoved() { - final int maxSize = 50; - // Fill the map with jobs - JobConfiguration config = new JobConfiguration(null, DATAVALUE_IMPORT, "userId"); - config.setUid("1"); - addNotificationEntries(config, maxSize); - - Map> typeNotification = - notifications.getNotificationsWithType(config.getJobType()); - Deque jobNotifications = typeNotification.get(config.getUid()); - assertNotNull(jobNotifications); - assertEquals(maxSize, jobNotifications.size()); - // Add one more - notifications.add(config, newNotification(config, maxSize)); - // Check that oldest job is not in the map anymore - assertFalse(jobNotifications.stream().anyMatch(n -> "0".equals(n.getMessage()))); - assertTrue(jobNotifications.stream().anyMatch(n -> (maxSize + "").equals(n.getMessage()))); - assertEquals(maxSize, jobNotifications.size()); - // Add one more - notifications.add(config, newNotification(config, maxSize + 1)); - // Check that oldest job is not in the map anymore - assertFalse(jobNotifications.stream().anyMatch(n -> "1".equals(n.getMessage()))); - assertTrue( - jobNotifications.stream().anyMatch(n -> ((maxSize + 1) + "").equals(n.getMessage()))); - assertEquals(maxSize, jobNotifications.size()); - } - - private void addSummaryEntries(JobConfiguration config, int n) { - for (int i = 0; i < n; i++) { - config.setUid(String.valueOf(i)); - notifications.addSummary(config, i); - } - } - - private void addNotificationEntries(JobConfiguration config, int n) { - for (int i = 0; i < n; i++) { - notifications.add(config, newNotification(config, i)); - } - } - - private Notification newNotification(JobConfiguration config, int no) { - return new Notification( - NotificationLevel.INFO, config.getJobType(), new Date(), "" + no, false, null, null); - } -} diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotifierStoreTest.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotifierStoreTest.java new file mode 100644 index 000000000000..cc4bddf0a921 --- /dev/null +++ b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotifierStoreTest.java @@ -0,0 +1,418 @@ +/* + * Copyright (c) 2004-2025, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.hisp.dhis.system.notification; + +import static java.lang.System.currentTimeMillis; +import static java.time.Duration.ofMillis; +import static java.time.Duration.ofSeconds; +import static java.util.concurrent.TimeUnit.DAYS; +import static java.util.concurrent.TimeUnit.HOURS; +import static org.hisp.dhis.scheduling.JobType.ANALYTICS_TABLE; +import static org.hisp.dhis.scheduling.JobType.DATA_INTEGRITY; +import static org.hisp.dhis.system.notification.NotificationLevel.LOOP; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; +import static org.testcontainers.shaded.org.awaitility.Awaitility.waitAtMost; + +import java.util.Deque; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongSupplier; +import org.hisp.dhis.common.UID; +import org.hisp.dhis.scheduling.JobConfiguration; +import org.hisp.dhis.scheduling.JobType; +import org.hisp.dhis.setting.SettingKey; +import org.hisp.dhis.setting.SystemSettingsProvider; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.quality.Strictness; + +/** + * Tests for the {@link Notifier} API that should run both for the {@link InMemoryNotifierStore} and + * the {@link RedisNotifierStore}. + * + * @author Jan Bernitt + */ +abstract class NotifierStoreTest { + + protected Notifier notifier; + protected LongSupplier clock = System::currentTimeMillis; + protected SystemSettingsProvider settingsProvider = + mock(SystemSettingsProvider.class, withSettings().strictness(Strictness.LENIENT)); + + /** + * Expected to init the {@link #notifier} field with a new {@link Notifier} instance using the + * provided {@link SystemSettingsProvider}. + * + * @param settings the settings to use + */ + abstract void setUpNotifier(SystemSettingsProvider settings); + + @BeforeEach + void setUp() { + // as default + when(settingsProvider.getSystemSetting(SettingKey.NOTIFIER_LOG_LEVEL, NotificationLevel.DEBUG)) + .thenReturn(NotificationLevel.DEBUG); + when(settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_MESSAGES_PER_JOB)).thenReturn(500); + when(settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_AGE_DAYS)).thenReturn(7); + when(settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_JOBS_PER_TYPE)).thenReturn(500); + when(settingsProvider.getBoolSetting(SettingKey.NOTIFIER_GIST_OVERVIEW)).thenReturn(true); + when(settingsProvider.getSystemSetting(SettingKey.NOTIFIER_CLEAN_AFTER_IDLE_TIME, Long.class)) + .thenReturn(60_000L); + setUpNotifier(settingsProvider); + } + + /** + * When messages are added they have to make it though a sync queue to become visible in the + * store, therefor one has to wait a bit before making asserts on the store content. + */ + private void awaitIdle() { + waitAtMost(ofSeconds(5)).pollInterval(ofMillis(25)).until(notifier::isIdle); + } + + @Test + void testNotify() { + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg1"); + notifier.notify(job1, "msg2"); + notifier.notify(job1, "msg3"); + awaitIdle(); + assertEquals(3, notifier.getNotificationsByJobId(DATA_INTEGRITY, job1.getUid()).size()); + } + + @Test + void testGetNotifications() { + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, "msg22"); + JobConfiguration job3 = job(3, ANALYTICS_TABLE); + notifier.notify(job3, "msg31"); + notifier.notify(job3, "msg32"); + notifier.notify(job3, "msg33"); + + awaitIdle(); + + Map>> notifications = notifier.getNotifications(false); + assertEquals(2, notifications.size()); + Map> dataIntegrity = notifications.get(DATA_INTEGRITY); + assertEquals(Set.of(job1.getUid(), job2.getUid()), dataIntegrity.keySet()); + assertMessages(List.of("msg11"), dataIntegrity.get(job1.getUid())); + assertMessages(List.of("msg22", "msg21"), dataIntegrity.get(job2.getUid())); + Map> analyticsTable = notifications.get(ANALYTICS_TABLE); + assertEquals(Set.of(job3.getUid()), analyticsTable.keySet()); + assertMessages(List.of("msg33", "msg32", "msg31"), analyticsTable.get(job3.getUid())); + + notifications = notifier.getNotifications(true); + assertMessages( + List.of("msg33", "msg31"), notifications.get(ANALYTICS_TABLE).get(job3.getUid())); + } + + @Test + void testGetNotificationsByJobId() { + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, "msg22"); + JobConfiguration job3 = job(3, ANALYTICS_TABLE); + notifier.notify(job3, "msg31"); + notifier.notify(job3, "msg32"); + notifier.notify(job3, "msg33"); + + awaitIdle(); + + assertMessages( + List.of("msg11"), notifier.getNotificationsByJobId(DATA_INTEGRITY, job1.getUid())); + assertMessages( + List.of("msg22", "msg21"), notifier.getNotificationsByJobId(DATA_INTEGRITY, job2.getUid())); + assertMessages(List.of(), notifier.getNotificationsByJobId(DATA_INTEGRITY, job3.getUid())); + assertMessages( + List.of("msg33", "msg32", "msg31"), + notifier.getNotificationsByJobId(ANALYTICS_TABLE, job3.getUid())); + } + + @Test + void testGetNotificationsByJobType() { + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, "msg22"); + JobConfiguration job3 = job(3, ANALYTICS_TABLE); + notifier.notify(job3, "msg31"); + notifier.notify(job3, "msg32"); + notifier.notify(job3, "msg33"); + + awaitIdle(); + + Map> notifications = + notifier.getNotificationsByJobType(DATA_INTEGRITY, false); + assertEquals(2, notifications.size()); + assertMessages(List.of("msg11"), notifications.get(job1.getUid())); + assertMessages(List.of("msg22", "msg21"), notifications.get(job2.getUid())); + + notifications = notifier.getNotificationsByJobType(DATA_INTEGRITY, true); + assertMessages(List.of("msg11"), notifications.get(job1.getUid())); + assertMessages(List.of("msg22", "msg21"), notifications.get(job2.getUid())); + + notifications = notifier.getNotificationsByJobType(ANALYTICS_TABLE, false); + assertEquals(1, notifications.size()); + assertMessages(List.of("msg33", "msg32", "msg31"), notifications.get(job3.getUid())); + + notifications = notifier.getNotificationsByJobType(ANALYTICS_TABLE, true); + assertEquals(1, notifications.size()); + assertMessages(List.of("msg33", "msg31"), notifications.get(job3.getUid())); + } + + @Test + void testClear() { + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, "msg22"); + JobConfiguration job3 = job(3, ANALYTICS_TABLE); + notifier.notify(job3, "msg31"); + notifier.notify(job3, "msg32"); + notifier.notify(job3, "msg33"); + + awaitIdle(); + + assertEquals(2, notifier.getNotifications(false).size()); + notifier.clear(); + assertEquals(0, notifier.getNotifications(false).size()); + assertEquals(0, notifier.getNotificationsByJobType(DATA_INTEGRITY, false).size()); + assertEquals(0, notifier.getNotificationsByJobType(ANALYTICS_TABLE, false).size()); + assertEquals(0, notifier.getNotificationsByJobId(DATA_INTEGRITY, job1.getUid()).size()); + assertEquals(0, notifier.getNotificationsByJobId(DATA_INTEGRITY, job2.getUid()).size()); + assertEquals(0, notifier.getNotificationsByJobId(ANALYTICS_TABLE, job3.getUid()).size()); + } + + @Test + void testClear_byType() { + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, "msg22"); + JobConfiguration job3 = job(3, ANALYTICS_TABLE); + notifier.notify(job3, "msg31"); + notifier.notify(job3, "msg32"); + notifier.notify(job3, "msg33"); + + awaitIdle(); + + assertEquals(3, notifier.getNotificationsByJobId(ANALYTICS_TABLE, job3.getUid()).size()); + notifier.clear(ANALYTICS_TABLE); + assertEquals(0, notifier.getNotificationsByJobId(ANALYTICS_TABLE, job3.getUid()).size()); + + assertEquals(2, notifier.getNotificationsByJobType(DATA_INTEGRITY, true).size()); + notifier.clear(DATA_INTEGRITY); + assertEquals(0, notifier.getNotificationsByJobType(DATA_INTEGRITY, true).size()); + } + + @Test + void testClear_byJob() { + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, "msg22"); + + awaitIdle(); + + notifier.clear(DATA_INTEGRITY, UID.of(job2.getUid())); + assertEquals(1, notifier.getNotificationsByJobType(DATA_INTEGRITY, false).size()); + assertEquals(0, notifier.getNotificationsByJobId(DATA_INTEGRITY, job2.getUid()).size()); + } + + @Test + void testCapMaxAge() { + long t0 = currentTimeMillis(); + AtomicLong manualClock = new AtomicLong(t0); + clock = manualClock::get; + setUpNotifier(settingsProvider); // update to use clock + + manualClock.set(t0 - DAYS.toMillis(5)); + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + + manualClock.set(t0 - HOURS.toMillis(55)); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, "msg22"); + + manualClock.set(t0 - DAYS.toMillis(1)); + JobConfiguration job3 = job(3, ANALYTICS_TABLE); + notifier.notify(job3, "msg31"); + notifier.notify(job3, "msg32"); + notifier.notify(job3, "msg33"); + + awaitIdle(); + + manualClock.set(currentTimeMillis()); // must be actual now + notifier.capMaxAge(3); + assertEquals(0, notifier.getNotificationsByJobId(DATA_INTEGRITY, job1.getUid()).size()); + assertEquals(2, notifier.getNotificationsByJobId(DATA_INTEGRITY, job2.getUid()).size()); + assertEquals(3, notifier.getNotificationsByJobId(ANALYTICS_TABLE, job3.getUid()).size()); + notifier.capMaxAge(2, DATA_INTEGRITY); + assertEquals(0, notifier.getNotificationsByJobType(DATA_INTEGRITY, false).size()); + assertEquals(3, notifier.getNotificationsByJobId(ANALYTICS_TABLE, job3.getUid()).size()); + assertEquals(1, notifier.getNotificationsByJobType(ANALYTICS_TABLE, false).size()); + notifier.capMaxAge(2); + assertEquals(3, notifier.getNotificationsByJobId(ANALYTICS_TABLE, job3.getUid()).size()); + } + + @Test + void testCapMaxCount() { + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, "msg22"); + JobConfiguration job3 = job(3, DATA_INTEGRITY); + notifier.notify(job3, "msg31"); + notifier.notify(job3, "msg32"); + notifier.notify(job3, "msg33"); + JobConfiguration job4 = job(4, ANALYTICS_TABLE); + notifier.notify(job4, "msg41"); + notifier.notify(job4, "msg42"); + notifier.notify(job4, "msg43"); + notifier.notify(job4, "msg44"); + + awaitIdle(); + + notifier.capMaxCount(2); + assertEquals(2, notifier.getNotificationsByJobType(DATA_INTEGRITY, true).size()); + assertEquals(1, notifier.getNotificationsByJobType(ANALYTICS_TABLE, false).size()); + assertEquals( + Set.of(job2.getUid(), job3.getUid()), + notifier.getNotificationsByJobType(DATA_INTEGRITY, true).keySet()); + } + + @Test + void testGetNotifierMaxMessagesPerJob() { + when(settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_MESSAGES_PER_JOB)).thenReturn(3); + setUpNotifier(settingsProvider); + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, "msg22"); + notifier.notify(job2, "msg23"); + notifier.notify(job2, "msg24"); + JobConfiguration job3 = job(3, ANALYTICS_TABLE); + notifier.notify(job3, "msg31"); + notifier.notify(job3, "msg32"); + notifier.notify(job3, "msg33"); + notifier.notify(job3, "msg34"); + notifier.notify(job3, "msg35"); + + awaitIdle(); + + assertMessages( + List.of("msg11"), notifier.getNotificationsByJobId(DATA_INTEGRITY, job1.getUid())); + assertMessages( + List.of("msg24", "msg23", "msg21"), + notifier.getNotificationsByJobId(DATA_INTEGRITY, job2.getUid())); + assertMessages( + List.of("msg35", "msg34", "msg31"), + notifier.getNotificationsByJobId(ANALYTICS_TABLE, job3.getUid())); + } + + @Test + void testNotify_loopLevelMessagesAreReplaced() { + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, LOOP, "msg22"); + notifier.notify(job2, LOOP, "msg23"); + notifier.notify(job2, "msg24"); + JobConfiguration job3 = job(3, ANALYTICS_TABLE); + notifier.notify(job3, "msg31"); + notifier.notify(job3, "msg32"); + notifier.notify(job3, LOOP, "msg33"); + notifier.notify(job3, "msg34"); + notifier.notify(job3, "msg35"); + + awaitIdle(); + + assertMessages( + List.of("msg11"), notifier.getNotificationsByJobId(DATA_INTEGRITY, job1.getUid())); + assertMessages( + List.of("msg24", "msg21"), notifier.getNotificationsByJobId(DATA_INTEGRITY, job2.getUid())); + assertMessages( + List.of("msg35", "msg34", "msg32", "msg31"), + notifier.getNotificationsByJobId(ANALYTICS_TABLE, job3.getUid())); + } + + @Test + void testCleanAfterIdleTime() throws InterruptedException { + when(settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_JOBS_PER_TYPE)).thenReturn(2); + when(settingsProvider.getSystemSetting(SettingKey.NOTIFIER_CLEAN_AFTER_IDLE_TIME, Long.class)) + .thenReturn(100L); + setUpNotifier(settingsProvider); + JobConfiguration job1 = job(1, DATA_INTEGRITY); + notifier.notify(job1, "msg11"); + JobConfiguration job2 = job(2, DATA_INTEGRITY); + notifier.notify(job2, "msg21"); + notifier.notify(job2, LOOP, "msg22"); + notifier.notify(job2, LOOP, "msg23"); + notifier.notify(job2, "msg24"); + JobConfiguration job3 = job(3, DATA_INTEGRITY); + notifier.notify(job3, "msg31"); + notifier.notify(job3, "msg32"); + notifier.notify(job3, LOOP, "msg33"); + notifier.notify(job3, "msg34"); + notifier.notify(job3, "msg35"); + + awaitIdle(); + + // wait to clean-up kicks in after 100ms + Thread.sleep(200); + + awaitIdle(); + + assertEquals(2, notifier.getNotificationsByJobType(DATA_INTEGRITY, false).size()); + } + + private static JobConfiguration job(int serial, JobType type) { + return new JobConfiguration("job" + serial, type); + } + + private static void assertMessages(List expected, Deque actual) { + assertEquals(expected, actual.stream().map(Notification::getMessage).toList()); + } +} diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotifierTest.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotifierTest.java index f6c9ce6b98f1..fa038b979ed9 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotifierTest.java +++ b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/NotifierTest.java @@ -27,204 +27,185 @@ */ package org.hisp.dhis.system.notification; +import static java.time.Duration.ofSeconds; import static org.hisp.dhis.scheduling.JobType.ANALYTICS_TABLE; import static org.hisp.dhis.scheduling.JobType.DATAVALUE_IMPORT; import static org.hisp.dhis.scheduling.JobType.METADATA_IMPORT; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; +import static org.testcontainers.shaded.org.awaitility.Awaitility.waitAtMost; +import com.fasterxml.jackson.databind.ObjectMapper; import java.util.Deque; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; -import org.hisp.dhis.DhisConvenienceTest; +import org.hisp.dhis.jsontree.Json; +import org.hisp.dhis.jsontree.JsonValue; import org.hisp.dhis.scheduling.JobConfiguration; import org.hisp.dhis.scheduling.JobType; -import org.hisp.dhis.user.User; +import org.hisp.dhis.setting.SettingKey; +import org.hisp.dhis.setting.SystemSettingsProvider; import org.junit.jupiter.api.Test; +import org.mockito.quality.Strictness; /** * @author Lars Helge Overland */ -class NotifierTest extends DhisConvenienceTest { - private Notifier notifier = new InMemoryNotifier(); - - private final User user = makeUser("A"); - - private final JobConfiguration dataValueImportJobConfig; - - private final JobConfiguration analyticsTableJobConfig; - - private final JobConfiguration metadataImportJobConfig; - - private final JobConfiguration dataValueImportSecondJobConfig; - - private final JobConfiguration dataValueImportThirdJobConfig; - - private final JobConfiguration dataValueImportFourthConfig; +class NotifierTest { + + private final Notifier notifier = + new DefaultNotifier( + new InMemoryNotifierStore(), + new ObjectMapper(), + settingsProvider(), + System::currentTimeMillis); + + private final JobConfiguration analyticsTable; + private final JobConfiguration metadataImport; + + private final JobConfiguration dataImport1; + private final JobConfiguration dataImport2; + private final JobConfiguration dataImport3; + private final JobConfiguration dataImport4; + + private static SystemSettingsProvider settingsProvider() { + SystemSettingsProvider settingsProvider = + mock(SystemSettingsProvider.class, withSettings().strictness(Strictness.LENIENT)); + when(settingsProvider.getSystemSetting(SettingKey.NOTIFIER_LOG_LEVEL, NotificationLevel.DEBUG)) + .thenReturn(NotificationLevel.DEBUG); + when(settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_MESSAGES_PER_JOB)).thenReturn(500); + when(settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_AGE_DAYS)).thenReturn(7); + when(settingsProvider.getIntSetting(SettingKey.NOTIFIER_MAX_JOBS_PER_TYPE)).thenReturn(500); + when(settingsProvider.getBoolSetting(SettingKey.NOTIFIER_GIST_OVERVIEW)).thenReturn(true); + when(settingsProvider.getSystemSetting(SettingKey.NOTIFIER_CLEAN_AFTER_IDLE_TIME, Long.class)) + .thenReturn(60_000L); + return settingsProvider; + } public NotifierTest() { - dataValueImportJobConfig = new JobConfiguration(null, DATAVALUE_IMPORT, user.getUid()); - dataValueImportJobConfig.setUid("dvi1"); - analyticsTableJobConfig = new JobConfiguration(null, ANALYTICS_TABLE, user.getUid()); - analyticsTableJobConfig.setUid("at1"); - metadataImportJobConfig = new JobConfiguration(null, METADATA_IMPORT, user.getUid()); - metadataImportJobConfig.setUid("mdi1"); - dataValueImportSecondJobConfig = new JobConfiguration(null, DATAVALUE_IMPORT, user.getUid()); - dataValueImportSecondJobConfig.setUid("dvi2"); - dataValueImportThirdJobConfig = new JobConfiguration(null, DATAVALUE_IMPORT, user.getUid()); - dataValueImportThirdJobConfig.setUid("dvi3"); - dataValueImportFourthConfig = new JobConfiguration(null, DATAVALUE_IMPORT, user.getUid()); - dataValueImportFourthConfig.setUid("dvi4"); - JobConfiguration dataValueImportFifthConfig = - new JobConfiguration(null, DATAVALUE_IMPORT, user.getUid()); - dataValueImportFifthConfig.setUid("dvi5"); + analyticsTable = new JobConfiguration(null, ANALYTICS_TABLE); + metadataImport = new JobConfiguration(null, METADATA_IMPORT); + dataImport1 = new JobConfiguration(null, DATAVALUE_IMPORT); + dataImport2 = new JobConfiguration(null, DATAVALUE_IMPORT); + dataImport3 = new JobConfiguration(null, DATAVALUE_IMPORT); + dataImport4 = new JobConfiguration(null, DATAVALUE_IMPORT); } @Test void testGetNotifications() { - notifier.notify(dataValueImportJobConfig, "Import started"); - notifier.notify(dataValueImportJobConfig, "Import working"); - notifier.notify(dataValueImportJobConfig, "Import done"); - notifier.notify(analyticsTableJobConfig, "Process started"); - notifier.notify(analyticsTableJobConfig, "Process done"); - Map>> notificationsMap = notifier.getNotifications(); - assertNotNull(notificationsMap); - assertEquals( - 3, - notifier - .getNotificationsByJobId( - dataValueImportJobConfig.getJobType(), dataValueImportJobConfig.getUid()) - .size()); - assertEquals( - 2, - notifier - .getNotificationsByJobId( - analyticsTableJobConfig.getJobType(), analyticsTableJobConfig.getUid()) - .size()); - assertEquals( - 0, - notifier - .getNotificationsByJobId( - metadataImportJobConfig.getJobType(), metadataImportJobConfig.getUid()) - .size()); - notifier.clear(dataValueImportJobConfig); - notifier.clear(analyticsTableJobConfig); - notifier.notify(dataValueImportJobConfig, "Import started"); - notifier.notify(dataValueImportJobConfig, "Import working"); - notifier.notify(dataValueImportJobConfig, "Import done"); - notifier.notify(analyticsTableJobConfig, "Process started"); - notifier.notify(analyticsTableJobConfig, "Process done"); - assertEquals( - 3, - notifier - .getNotificationsByJobId( - dataValueImportJobConfig.getJobType(), dataValueImportJobConfig.getUid()) - .size()); - assertEquals( - 2, - notifier - .getNotificationsByJobId( - analyticsTableJobConfig.getJobType(), analyticsTableJobConfig.getUid()) - .size()); - notifier.clear(dataValueImportJobConfig); - assertEquals( - 0, - notifier - .getNotificationsByJobId( - dataValueImportJobConfig.getJobType(), dataValueImportJobConfig.getUid()) - .size()); - assertEquals( - 2, - notifier - .getNotificationsByJobId( - analyticsTableJobConfig.getJobType(), analyticsTableJobConfig.getUid()) - .size()); - notifier.clear(analyticsTableJobConfig); - assertEquals( - 0, - notifier - .getNotificationsByJobId( - dataValueImportJobConfig.getJobType(), dataValueImportJobConfig.getUid()) - .size()); - assertEquals( - 0, - notifier - .getNotificationsByJobId( - analyticsTableJobConfig.getJobType(), analyticsTableJobConfig.getUid()) - .size()); - notifier.notify(dataValueImportSecondJobConfig, "Process done"); - notifier.notify(dataValueImportJobConfig, "Import started"); - notifier.notify(dataValueImportJobConfig, "Import working"); - notifier.notify(dataValueImportJobConfig, "Import in progress"); - notifier.notify(dataValueImportJobConfig, "Import done"); - notifier.notify(analyticsTableJobConfig, "Process started"); - notifier.notify(analyticsTableJobConfig, "Process done"); + notifier.notify(dataImport1, "Import started"); + notifier.notify(dataImport1, "Import working"); + notifier.notify(dataImport1, "Import done"); + notifier.notify(analyticsTable, "Process started"); + notifier.notify(analyticsTable, "Process done"); + awaitIdle(); + assertNotNull(notifier.getNotifications(false)); + assertEquals(3, getNotificationsCount(DATAVALUE_IMPORT, dataImport1.getUid())); + assertEquals(2, getNotificationsCount(ANALYTICS_TABLE, analyticsTable.getUid())); + assertEquals(0, getNotificationsCount(METADATA_IMPORT, metadataImport.getUid())); + + notifier.clear(dataImport1); + notifier.clear(analyticsTable); + notifier.notify(dataImport1, "Import started"); + notifier.notify(dataImport1, "Import working"); + notifier.notify(dataImport1, "Import done"); + notifier.notify(analyticsTable, "Process started"); + notifier.notify(analyticsTable, "Process done"); + awaitIdle(); + assertEquals(3, getNotificationsCount(DATAVALUE_IMPORT, dataImport1.getUid())); + assertEquals(2, getNotificationsCount(ANALYTICS_TABLE, analyticsTable.getUid())); + notifier.clear(dataImport1); + assertEquals(0, getNotificationsCount(DATAVALUE_IMPORT, dataImport1.getUid())); + assertEquals(2, getNotificationsCount(ANALYTICS_TABLE, analyticsTable.getUid())); + notifier.clear(analyticsTable); + assertEquals(0, getNotificationsCount(DATAVALUE_IMPORT, dataImport1.getUid())); + assertEquals(0, getNotificationsCount(ANALYTICS_TABLE, analyticsTable.getUid())); + + notifier.notify(dataImport2, "Process done"); + notifier.notify(dataImport1, "Import started"); + notifier.notify(dataImport1, "Import working"); + notifier.notify(dataImport1, "Import in progress"); + notifier.notify(dataImport1, "Import done"); + notifier.notify(analyticsTable, "Process started"); + notifier.notify(analyticsTable, "Process done"); + awaitIdle(); Deque notifications = - notifier.getNotificationsByJobType(DATAVALUE_IMPORT).get(dataValueImportJobConfig.getUid()); + getNotifications(DATAVALUE_IMPORT).get(dataImport1.getUid()); assertNotNull(notifications); assertEquals(4, notifications.size()); - notifier.notify(dataValueImportThirdJobConfig, "Completed1"); - Map> notificationsByJobType = - notifier.getNotificationsByJobType(DATAVALUE_IMPORT); - assertNotNull(notificationsByJobType); - assertEquals(3, notificationsByJobType.size()); - assertEquals(4, notificationsByJobType.get(dataValueImportJobConfig.getUid()).size()); - assertEquals(1, notificationsByJobType.get(dataValueImportSecondJobConfig.getUid()).size()); - assertEquals(1, notificationsByJobType.get(dataValueImportThirdJobConfig.getUid()).size()); - assertEquals( - "Completed1", - notificationsByJobType.get(dataValueImportThirdJobConfig.getUid()).getFirst().getMessage()); - notifier.notify(dataValueImportFourthConfig, "Completed2"); - notificationsByJobType = notifier.getNotificationsByJobType(DATAVALUE_IMPORT); - assertNotNull(notificationsByJobType); - assertEquals(4, notificationsByJobType.get(dataValueImportJobConfig.getUid()).size()); - assertEquals(1, notificationsByJobType.get(dataValueImportSecondJobConfig.getUid()).size()); - assertEquals(1, notificationsByJobType.get(dataValueImportThirdJobConfig.getUid()).size()); - assertEquals(1, notificationsByJobType.get(dataValueImportFourthConfig.getUid()).size()); - assertEquals( - "Completed2", - notificationsByJobType.get(dataValueImportFourthConfig.getUid()).getFirst().getMessage()); + + notifier.notify(dataImport3, "Completed1"); + awaitIdle(); + Map> byJobId = getNotifications(DATAVALUE_IMPORT); + assertNotNull(byJobId); + assertEquals(3, byJobId.size()); + assertEquals(4, byJobId.get(dataImport1.getUid()).size()); + assertEquals(1, byJobId.get(dataImport2.getUid()).size()); + assertEquals(1, byJobId.get(dataImport3.getUid()).size()); + assertEquals("Completed1", byJobId.get(dataImport3.getUid()).getFirst().getMessage()); + + notifier.notify(dataImport4, "Completed2"); + awaitIdle(); + byJobId = getNotifications(DATAVALUE_IMPORT); + assertNotNull(byJobId); + assertEquals(4, byJobId.get(dataImport1.getUid()).size()); + assertEquals(1, byJobId.get(dataImport2.getUid()).size()); + assertEquals(1, byJobId.get(dataImport3.getUid()).size()); + assertEquals(1, byJobId.get(dataImport4.getUid()).size()); + assertEquals("Completed2", byJobId.get(dataImport4.getUid()).getFirst().getMessage()); + } + + private void awaitIdle() { + waitAtMost(ofSeconds(5)).until(notifier::isIdle); + } + + private Map> getNotifications(JobType type) { + return notifier.getNotificationsByJobType(type, false); + } + + private int getNotificationsCount(JobType type, String uid) { + return notifier.getNotificationsByJobId(type, uid).size(); } @Test void testGetSummary() { - notifier.addJobSummary(dataValueImportJobConfig, "somethingid1", String.class); - notifier.addJobSummary(analyticsTableJobConfig, "somethingid2", String.class); - notifier.addJobSummary(dataValueImportSecondJobConfig, "somethingid4", String.class); - notifier.addJobSummary(metadataImportJobConfig, "somethingid3", String.class); - Map jobSummariesForAnalyticsType = + notifier.addJobSummary(dataImport1, "somethingid1", String.class); + notifier.addJobSummary(analyticsTable, "somethingid2", String.class); + notifier.addJobSummary(dataImport2, "somethingid4", String.class); + notifier.addJobSummary(metadataImport, "somethingid3", String.class); + Map jobSummariesForAnalyticsType = notifier.getJobSummariesForJobType(DATAVALUE_IMPORT); assertNotNull(jobSummariesForAnalyticsType); assertEquals(2, jobSummariesForAnalyticsType.size()); - Map jobSummariesForMetadataImportType = + Map jobSummariesForMetadataImportType = notifier.getJobSummariesForJobType(METADATA_IMPORT); assertNotNull(jobSummariesForMetadataImportType); assertEquals(1, jobSummariesForMetadataImportType.size()); assertEquals( - "somethingid3", jobSummariesForMetadataImportType.get(metadataImportJobConfig.getUid())); - Object summary = - notifier.getJobSummaryByJobId( - dataValueImportJobConfig.getJobType(), dataValueImportJobConfig.getUid()); + Json.of("somethingid3"), jobSummariesForMetadataImportType.get(metadataImport.getUid())); + Object summary = notifier.getJobSummaryByJobId(DATAVALUE_IMPORT, dataImport1.getUid()); assertNotNull(summary); - assertEquals("somethingid1", summary, "True"); - notifier.addJobSummary(dataValueImportThirdJobConfig, "summarry3", String.class); + assertEquals(Json.of("somethingid1"), summary, "True"); + notifier.addJobSummary(dataImport3, "summarry3", String.class); jobSummariesForAnalyticsType = notifier.getJobSummariesForJobType(DATAVALUE_IMPORT); assertNotNull(jobSummariesForAnalyticsType); assertEquals(3, jobSummariesForAnalyticsType.size()); - notifier.addJobSummary(dataValueImportFourthConfig, "summarry4", String.class); + notifier.addJobSummary(dataImport4, "summarry4", String.class); jobSummariesForAnalyticsType = notifier.getJobSummariesForJobType(DATAVALUE_IMPORT); assertNotNull(jobSummariesForAnalyticsType); assertEquals(4, jobSummariesForAnalyticsType.size()); } @Test - void testInsertingNotificationsInSameJobConcurrently() throws InterruptedException { + void testInsertingNotificationsInSameJobConcurrently() { ExecutorService e = Executors.newFixedThreadPool(5); - JobConfiguration jobConfig = createJobConfig(-1); + JobConfiguration jobConfig = new JobConfiguration(null, METADATA_IMPORT); notifier.notify(jobConfig, "somethingid"); IntStream.range(0, 100) .forEach(i -> e.execute(() -> notifier.notify(jobConfig, "somethingid" + i))); @@ -232,38 +213,14 @@ void testInsertingNotificationsInSameJobConcurrently() throws InterruptedExcepti .forEach( i -> { for (Notification notification : - notifier.getNotificationsByJobType(METADATA_IMPORT).get(jobConfig.getUid())) { + getNotifications(METADATA_IMPORT).get(jobConfig.getUid())) { // Iterate over notifications when new notification are added assertNotNull(notification.getUid()); } }); awaitTermination(e); - assertEquals( - 101, notifier.getNotificationsByJobType(METADATA_IMPORT).get(jobConfig.getUid()).size()); - } - - @Test - void testInsertingNotificationJobConcurrently() { - notifier.notify(createJobConfig(-1), "zero"); - ExecutorService e = Executors.newFixedThreadPool(5); - IntStream.range(0, 1000) - .forEach( - i -> { - e.execute( - () -> { - notifier.notify(createJobConfig(i), "somethingid" + i); - }); - }); - awaitTermination(e); - int actualSize = notifier.getNotificationsByJobType(METADATA_IMPORT).size(); - int delta = actualSize - 500; - assertTrue(delta <= 5, "delta should not be larger than number of workers but was: " + delta); - } - - private JobConfiguration createJobConfig(int i) { - JobConfiguration jobConfig = new JobConfiguration(null, METADATA_IMPORT, user.getUid()); - jobConfig.setUid("jobId" + i); - return jobConfig; + awaitIdle(); + assertEquals(101, getNotifications(METADATA_IMPORT).get(jobConfig.getUid()).size()); } public void awaitTermination(ExecutorService threadPool) { diff --git a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/RedisNotifierTest.java b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/RedisNotifierTest.java index c668d6074422..68cabeb67ff8 100644 --- a/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/RedisNotifierTest.java +++ b/dhis-2/dhis-support/dhis-support-system/src/test/java/org/hisp/dhis/system/notification/RedisNotifierTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2023, University of Oslo + * Copyright (c) 2004-2025, University of Oslo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -27,124 +27,19 @@ */ package org.hisp.dhis.system.notification; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.Deque; -import java.util.HashSet; -import java.util.Set; -import org.hisp.dhis.DhisConvenienceTest; -import org.hisp.dhis.scheduling.JobType; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.data.redis.core.BoundZSetOperations; -import org.springframework.data.redis.core.RedisTemplate; +import org.hisp.dhis.setting.SystemSettingsProvider; /** - * @author David Mackessy + * Tests the {@link Notifier} API with the {@link RedisNotifierStore} implementation when Redis is + * replaced by a {@link FakeRedis} which keep the data in memory. + * + *

The actual tests are in {@link NotifierStoreTest} as they are used for both stores. + * + * @author Jan Bernitt */ -@ExtendWith(MockitoExtension.class) -class RedisNotifierTest extends DhisConvenienceTest { - - @Mock private RedisTemplate redisTemplate; - - @Mock private BoundZSetOperations boundZSetOperations; - - @InjectMocks private RedisNotifier notifier; - - /** - * When Redis is enabled, the APP UI relies on specific ordered data to know when generating - * Resource/Analytics tables has completed. The APP UI only knows the job is complete once it sees - * that the first element in the returned data is marked as 'completed'. This test confirms that - * the returned data always has the latest {@link Notification} as the first element. - */ - @Test - void getNotificationsByJobIdTest_OrderedByTime() { - notifier = new RedisNotifier(redisTemplate, new ObjectMapper()); - JobType jobType = JobType.ANALYTICS_TABLE; - String jobId = "job1d1"; - Set dataFromRedis = new HashSet<>(); - - // 3 Notifications, each with a different 'time' value - // the 'completed' Notification has the latest time - String notificationMiddle = - "{\"uid\":\"ju8WSUHJKHO\",\"level\": \"INFO\",\"category\":\"ANALYTICS_TABLE\",\"time\":\"2023-07-05T10:16:33.554\",\"message\":\"1 Analytics tables updated\",\"completed\":false}"; - String notificationLatest = - "{\"uid\":\"zM8zxPLTKaY\",\"level\":\"INFO\",\"category\":\"ANALYTICS_TABLE\",\"time\":\"2023-07-05T10:16:33.555\",\"message\":\"2 Drop SQL views\",\"completed\":true}"; - String notificationEarliest = - "{\"uid\":\"aM8zxPLTKaY\",\"level\":\"INFO\",\"category\":\"ANALYTICS_TABLE\",\"time\":\"2023-07-05T10:16:33.553\",\"message\":\"3 Drop SQL views\",\"completed\":false}"; - - // add notifications unordered - dataFromRedis.add(notificationMiddle); - dataFromRedis.add(notificationEarliest); - dataFromRedis.add(notificationLatest); - - when(redisTemplate.boundZSetOps(any())).thenReturn(boundZSetOperations); - when(redisTemplate.boundZSetOps(any()).range(0, -1)).thenReturn(dataFromRedis); - - // Notifications should be returned in an ordered Queue from this call - Deque result = notifier.getNotificationsByJobId(jobType, jobId); - - assertFalse(result.isEmpty()); - // check first Notification is completed - Notification peek = result.peek(); - assertTrue(peek.isCompleted()); - - // confirm the ordering of each Notification - Notification latestNotification = result.removeFirst(); - Notification middleNotification = result.removeFirst(); - Notification earliestNotification = result.removeFirst(); - assertTrue(latestNotification.getTime().after(middleNotification.getTime())); - assertTrue(latestNotification.getTime().after(earliestNotification.getTime())); - assertTrue(middleNotification.getTime().after(earliestNotification.getTime())); - } - - @Test - void getNotificationsByJobIdTest_OrderedByTimeAndCompleted() { - notifier = new RedisNotifier(redisTemplate, new ObjectMapper()); - JobType jobType = JobType.ANALYTICS_TABLE; - String jobId = "job1d1"; - Set dataFromRedis = new HashSet<>(); - - // 3 Notifications, 2 with the same 'time' value (1 of which is 'complete') - String notificationEarliest = - "{\"uid\":\"ju8WSUHJKHO\",\"level\": \"INFO\",\"category\":\"ANALYTICS_TABLE\",\"time\":\"2023-07-05T10:16:33.554\",\"message\":\"1 Analytics tables updated\",\"completed\":false}"; - String notificationLatestComplete = - "{\"uid\":\"zM8zxPLTKaY\",\"level\":\"INFO\",\"category\":\"ANALYTICS_TABLE\",\"time\":\"2023-07-05T10:16:33.555\",\"message\":\"2 Drop SQL views\",\"completed\":true}"; - String notificationLatestNotComplete = - "{\"uid\":\"aM8zxPLTKaY\",\"level\":\"INFO\",\"category\":\"ANALYTICS_TABLE\",\"time\":\"2023-07-05T10:16:33.555\",\"message\":\"3 Drop SQL views\",\"completed\":false}"; - - // add notifications unordered - dataFromRedis.add(notificationEarliest); - dataFromRedis.add(notificationLatestComplete); - dataFromRedis.add(notificationLatestNotComplete); - - when(redisTemplate.boundZSetOps(any())).thenReturn(boundZSetOperations); - when(redisTemplate.boundZSetOps(any()).range(0, -1)).thenReturn(dataFromRedis); - - // Notifications should be returned in an ordered Queue from this call - Deque result = notifier.getNotificationsByJobId(jobType, jobId); - - assertFalse(result.isEmpty()); - // check first Notification is completed - Notification peek = result.peek(); - assertTrue(peek.isCompleted()); - - // confirm the ordering of each Notification and that first two times are equal - Notification latestNotificationComplete = result.removeFirst(); - Notification latestNotificationCompleteNotComplete = result.removeFirst(); - Notification earliestNotification = result.removeFirst(); - assertEquals( - latestNotificationComplete.getTime(), latestNotificationCompleteNotComplete.getTime()); - assertTrue(latestNotificationComplete.getTime().after(earliestNotification.getTime())); - assertTrue( - latestNotificationCompleteNotComplete.getTime().after(earliestNotification.getTime())); +class RedisNotifierTest extends NotifierStoreTest { + @Override + void setUpNotifier(SystemSettingsProvider settings) { + notifier = FakeRedis.notifier(settings, clock); } } diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/aggregate/DataImportTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/aggregate/DataImportTest.java index f99f544a5bd3..5dfa9367cee6 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/aggregate/DataImportTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/aggregate/DataImportTest.java @@ -124,9 +124,9 @@ void dataValuesCanBeImportedAsync() { systemActions .waitUntilTaskCompleted("DATAVALUE_IMPORT", taskId) .validate() - .body("message", hasItem(containsString("Process started"))) - .body("message", hasItem(containsString("Importing data values"))) - .body("message", hasItem(containsString("Import done"))); + .body("message", hasItem(containsString("Data value set import"))) + .body("message", hasItem(containsString("Importing data..."))) + .body("message", hasItem(containsString("Import complete with status SUCCESS"))); // validate task summaries were created ApiResponse taskSummariesResponse = @@ -170,7 +170,7 @@ void asyncImportEmptyJsonFileSummaryAvailable() { hasItem( containsString( "Import complete with status ERROR, 0 created, 0 updated, 0 deleted, 0 ignored"))) - .body("message", hasItem(containsString("No content to map due to end-of-input"))); + .body("message", hasItem(containsString("Import complete with status ERROR"))); // then a task summary should be available with an error message ApiResponse taskSummariesResponse = diff --git a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/merge/IndicatorTypeMergeTest.java b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/merge/IndicatorTypeMergeTest.java index 3a69f70abb95..bfd0e7ad31dc 100644 --- a/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/merge/IndicatorTypeMergeTest.java +++ b/dhis-2/dhis-test-e2e/src/test/java/org/hisp/dhis/merge/IndicatorTypeMergeTest.java @@ -76,7 +76,7 @@ void testInvalidSourceUid() { .body( "message", equalTo( - "JSON parse error: Cannot construct instance of `org.hisp.dhis.common.UID`, problem: UID must be an alphanumeric string of 11 characters starting with a letter.")); + "JSON parse error: Cannot construct instance of `org.hisp.dhis.common.UID`, problem: UID must be an alphanumeric string of 11 characters starting with a letter, but was: invalid")); } @Test @@ -97,7 +97,7 @@ void testInvalidTargetUid() { .body( "message", equalTo( - "JSON parse error: Cannot construct instance of `org.hisp.dhis.common.UID`, problem: UID must be an alphanumeric string of 11 characters starting with a letter.")); + "JSON parse error: Cannot construct instance of `org.hisp.dhis.common.UID`, problem: UID must be an alphanumeric string of 11 characters starting with a letter, but was: invalid")); } @Test diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemControllerTest.java index 8b9d968d5609..a4075c22261c 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/SystemControllerTest.java @@ -48,27 +48,8 @@ class SystemControllerTest extends DhisControllerConvenienceTest { @Test void testGetTasksJson() { JsonObject tasks = GET("/system/tasks").content(HttpStatus.OK); - assertObjectMembers( - tasks, - "CONTINUOUS_ANALYTICS_TABLE", - "DATA_SYNC", - "TRACKER_PROGRAMS_DATA_SYNC", - "EVENT_PROGRAMS_DATA_SYNC", - "FILE_RESOURCE_CLEANUP", - "IMAGE_PROCESSING", - "META_DATA_SYNC", - "SMS_SEND", - "SEND_SCHEDULED_MESSAGE", - "PROGRAM_NOTIFICATIONS", - "VALIDATION_RESULTS_NOTIFICATION", - "CREDENTIALS_EXPIRY_ALERT", - "MONITORING", - "PUSH_ANALYSIS", - "PREDICTOR", - "DATA_STATISTICS", - "DATA_INTEGRITY", - "RESOURCE_TABLE", - "ANALYTICS_TABLE"); + assertTrue(tasks.isObject()); + tasks.values().forEach(m -> assertTrue(m.isObject(), m + " is not an object")); } @Test @@ -81,7 +62,8 @@ void testGetTasksExtendedJson() { @Test void testGetTaskJsonByUid() { JsonArray task = - GET("/system/tasks/{jobType}/{jobId}", "META_DATA_SYNC", "xyz").content(HttpStatus.OK); + GET("/system/tasks/{jobType}/{jobId}", "META_DATA_SYNC", "a1234567890") + .content(HttpStatus.OK); assertTrue(task.isArray()); assertEquals(0, task.size()); } @@ -95,7 +77,8 @@ void testGetTaskSummaryExtendedJson() { @Test void testGetTaskSummaryJson() { - JsonObject summary = GET("/system/taskSummaries/META_DATA_SYNC/xyz").content(HttpStatus.OK); + JsonObject summary = + GET("/system/taskSummaries/META_DATA_SYNC/a1234567890").content(HttpStatus.OK); assertTrue(summary.isObject()); assertEquals(0, summary.size()); } @@ -116,11 +99,4 @@ void testGetSystemInfo_NonSuperUser() { assertNull(info.getString("javaVersion").string()); assertNotNull(info.getString("serverDate").string()); } - - private static void assertObjectMembers(JsonObject root, String... members) { - for (String member : members) { - JsonObject memberObj = root.getObject(member); - assertTrue(memberObj.isObject(), member + " is not an object"); - } - } } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemController.java index 2acf84f6ba4a..6a7cb50e5f3b 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/SystemController.java @@ -27,7 +27,6 @@ */ package org.hisp.dhis.webapi.controller; -import static java.util.Collections.emptyMap; import static org.hisp.dhis.webapi.utils.ContextUtils.setNoStore; import static org.springframework.http.CacheControl.noStore; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; @@ -47,13 +46,16 @@ import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import lombok.Data; import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.common.DhisApiVersion; import org.hisp.dhis.common.OpenApi; +import org.hisp.dhis.common.UID; import org.hisp.dhis.fieldfiltering.FieldFilterParams; import org.hisp.dhis.fieldfiltering.FieldFilterService; import org.hisp.dhis.i18n.I18n; import org.hisp.dhis.i18n.I18nManager; +import org.hisp.dhis.jsontree.JsonValue; import org.hisp.dhis.scheduling.JobConfiguration; import org.hisp.dhis.scheduling.JobConfigurationService; import org.hisp.dhis.scheduling.JobStatus; @@ -79,6 +81,7 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; @@ -167,40 +170,71 @@ public void getUidCsv( // Tasks // ------------------------------------------------------------------------- + @Data + public static class DeleteTasksParams { + private Integer maxCount; + + private Integer maxAge; + } + + @DeleteMapping("/tasks") + @ResponseStatus(HttpStatus.NO_CONTENT) + public void deleteNotifications(DeleteTasksParams params) { + Integer maxCount = params.getMaxCount(); + Integer maxAge = params.getMaxAge(); + if (maxAge != null) notifier.capMaxAge(maxAge); + if (maxCount != null) notifier.capMaxCount(maxCount); + if (maxCount == null && maxAge == null) notifier.clear(); + } + + @DeleteMapping("/tasks/{jobType}") + @ResponseStatus(HttpStatus.NO_CONTENT) + public void deleteNotificationsByJobType( + @PathVariable("jobType") JobType jobType, DeleteTasksParams params) { + Integer maxAge = params.getMaxAge(); + Integer maxCount = params.getMaxCount(); + if (maxAge != null) notifier.capMaxAge(maxAge, jobType); + if (maxCount != null) notifier.capMaxCount(maxCount, jobType); + if (maxCount == null && maxAge == null) notifier.clear(jobType); + } + + @DeleteMapping("/tasks/{jobType}/{jobId}") + @ResponseStatus(HttpStatus.NO_CONTENT) + public void deleteNotificationsByJobId( + @PathVariable("jobType") JobType jobType, + @PathVariable("jobId") @OpenApi.Param(value = {UID.class, JobConfiguration.class}) + UID jobId) { + notifier.clear(jobType, jobId); + } + @GetMapping(value = "/tasks", produces = APPLICATION_JSON_VALUE) - public ResponseEntity>>> getTasksJson() { - return ResponseEntity.ok().cacheControl(noStore()).body(notifier.getNotifications()); + public ResponseEntity>>> getTasksAllJobTypes( + @RequestParam(required = false) Boolean gist) { + return ResponseEntity.ok().cacheControl(noStore()).body(notifier.getNotifications(gist)); } @GetMapping(value = "/tasks/{jobType}", produces = APPLICATION_JSON_VALUE) - public ResponseEntity>> getTasksExtendedJson( - @PathVariable("jobType") String jobType) { + public ResponseEntity>> getTasksByJobType( + @PathVariable("jobType") JobType jobType, @RequestParam(required = false) Boolean gist) { Map> notifications = - jobType == null - ? emptyMap() - : notifier.getNotificationsByJobType(JobType.valueOf(jobType.toUpperCase())); + jobType == null ? Map.of() : notifier.getNotificationsByJobType(jobType, gist); return ResponseEntity.ok().cacheControl(noStore()).body(notifications); } @GetMapping(value = "/tasks/{jobType}/{jobId}", produces = APPLICATION_JSON_VALUE) public ResponseEntity> getTaskJsonByUid( - @PathVariable("jobType") String jobType, @PathVariable("jobId") String jobId) { - if (jobType == null) { - return ResponseEntity.ok().cacheControl(noStore()).body(List.of()); - } - - Deque notifications = - notifier.getNotificationsByJobId(JobType.valueOf(jobType.toUpperCase()), jobId); + @PathVariable("jobType") JobType jobType, + @PathVariable("jobId") @OpenApi.Param(value = {UID.class, JobConfiguration.class}) + UID jobId) { + Deque notifications = notifier.getNotificationsByJobId(jobType, jobId.getValue()); - if (notifications.isEmpty()) { - return ResponseEntity.ok().cacheControl(noStore()).body(List.of()); - } + if (notifications.isEmpty()) return ResponseEntity.ok().cacheControl(noStore()).body(List.of()); if (!notifications.getFirst().isCompleted()) { - JobConfiguration job = jobConfigurationService.getJobConfigurationByUid(jobId); + JobConfiguration job = jobConfigurationService.getJobConfigurationByUid(jobId.getValue()); if (job == null || job.getJobStatus() != JobStatus.RUNNING) { - notifier.clear(getJobSafe(job, JobType.valueOf(jobType.toUpperCase()), jobId)); + notifier.clear(getJobSafe(job, jobType, jobId.getValue())); Notification notification = notifications.getFirst(); notification.setCompleted(true); return ResponseEntity.ok().cacheControl(noStore()).body(List.of(notification)); @@ -214,30 +248,20 @@ public ResponseEntity> getTaskJsonByUid( // ------------------------------------------------------------------------- @GetMapping(value = "/taskSummaries/{jobType}", produces = APPLICATION_JSON_VALUE) - public ResponseEntity> getTaskSummaryExtendedJson( - @PathVariable("jobType") String jobType) { - if (jobType != null) { - Map summary = - notifier.getJobSummariesForJobType(JobType.valueOf(jobType.toUpperCase())); - if (summary != null) { - return ResponseEntity.ok().cacheControl(noStore()).body(summary); - } - } + public ResponseEntity> getTaskSummaryExtendedJson( + @PathVariable("jobType") JobType jobType) { + Map summary = notifier.getJobSummariesForJobType(jobType); + if (summary != null) return ResponseEntity.ok().cacheControl(noStore()).body(summary); return ResponseEntity.ok().cacheControl(noStore()).build(); } - @OpenApi.Response(ObjectNode.class) @GetMapping(value = "/taskSummaries/{jobType}/{jobId}", produces = APPLICATION_JSON_VALUE) - public ResponseEntity getTaskSummaryJson( - @PathVariable("jobType") String jobType, @PathVariable("jobId") String jobId) { - if (jobType != null) { - Object summary = notifier.getJobSummaryByJobId(JobType.valueOf(jobType.toUpperCase()), jobId); - - if (summary != null) { - return ResponseEntity.ok().cacheControl(noStore()).body(summary); - } - } - + public ResponseEntity getTaskSummaryJson( + @PathVariable("jobType") JobType jobType, + @PathVariable("jobId") @OpenApi.Param(value = {UID.class, JobConfiguration.class}) + UID jobId) { + JsonValue summary = notifier.getJobSummaryByJobId(jobType, jobId.getValue()); + if (summary != null) return ResponseEntity.ok().cacheControl(noStore()).body(summary); return ResponseEntity.ok().cacheControl(noStore()).build(); } diff --git a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportController.java b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportController.java index 0113a82ea559..c4621f28b3f3 100644 --- a/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportController.java +++ b/dhis-2/dhis-web-api/src/main/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportController.java @@ -32,13 +32,13 @@ import static org.hisp.dhis.webapi.utils.ContextUtils.setNoStore; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Deque; import java.util.List; -import java.util.Optional; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.RequiredArgsConstructor; @@ -48,6 +48,7 @@ import org.hisp.dhis.dxf2.webmessage.WebMessage; import org.hisp.dhis.feedback.ConflictException; import org.hisp.dhis.feedback.NotFoundException; +import org.hisp.dhis.jsontree.JsonValue; import org.hisp.dhis.scheduling.JobConfiguration; import org.hisp.dhis.scheduling.JobConfigurationService; import org.hisp.dhis.scheduling.JobSchedulerService; @@ -255,11 +256,16 @@ public ImportReport getJobReport( @PathVariable String uid, @RequestParam(defaultValue = "errors", required = false) TrackerBundleReportMode reportMode, HttpServletResponse response) - throws HttpStatusCodeException, NotFoundException { + throws HttpStatusCodeException, NotFoundException, ConflictException { setNoStore(response); - return Optional.ofNullable(notifier.getJobSummaryByJobId(JobType.TRACKER_IMPORT_JOB, uid)) - .map(report -> trackerImportService.buildImportReport((ImportReport) report, reportMode)) - .orElseThrow(() -> new NotFoundException("Summary for job " + uid + " does not exist")); + JsonValue report = notifier.getJobSummaryByJobId(JobType.TRACKER_IMPORT_JOB, uid); + if (report == null) throw new NotFoundException("Summary for job " + uid + " does not exist"); + try { + return trackerImportService.buildImportReport( + jsonMapper.readValue(report.toJson(), ImportReport.class), reportMode); + } catch (JsonProcessingException e) { + throw new ConflictException("Failed to convert the import report: " + report); + } } } diff --git a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportControllerTest.java b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportControllerTest.java index 9aa57d7e9de1..665f17aee9b7 100644 --- a/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportControllerTest.java +++ b/dhis-2/dhis-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerImportControllerTest.java @@ -50,6 +50,7 @@ import org.hisp.dhis.common.CodeGenerator; import org.hisp.dhis.commons.jackson.config.JacksonObjectMapperConfig; import org.hisp.dhis.feedback.NotFoundException; +import org.hisp.dhis.jsontree.JsonValue; import org.hisp.dhis.render.DefaultRenderService; import org.hisp.dhis.render.RenderService; import org.hisp.dhis.scheduling.JobConfigurationService; @@ -323,9 +324,8 @@ void verifyShouldFindJobReport() throws Exception { new HashMap<>()); // When - when(notifier.getJobSummaryByJobId(TRACKER_IMPORT_JOB, uid)).thenReturn(importReport); - - when(trackerImportService.buildImportReport(any(), any())).thenReturn(importReport); + JsonValue report = JsonValue.of(new ObjectMapper().writeValueAsString(importReport)); + when(notifier.getJobSummaryByJobId(TRACKER_IMPORT_JOB, uid)).thenReturn(report); // Then String contentAsString = @@ -336,14 +336,12 @@ void verifyShouldFindJobReport() throws Exception { .contentType(MediaType.APPLICATION_JSON) .accept(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()) - .andExpect(jsonPath("$.message").doesNotExist()) .andExpect(content().contentType("application/json")) .andReturn() .getResponse() .getContentAsString(); verify(notifier).getJobSummaryByJobId(TRACKER_IMPORT_JOB, uid); - verify(trackerImportService).buildImportReport(any(), any()); try { renderService.fromJson(contentAsString, ImportReport.class); diff --git a/dhis-2/pom.xml b/dhis-2/pom.xml index 497a3f227d59..a1fe70c46bac 100644 --- a/dhis-2/pom.xml +++ b/dhis-2/pom.xml @@ -2120,6 +2120,7 @@ jasperreports.version=${jasperreports.version} io.debezium:debezium-connector-postgres org.springframework.security:spring-security-core com.fasterxml.jackson.core:jackson-databind + com.fasterxml.jackson.core:jackson-core