From a88e7c52f4c748d4d7873a9204c4c716d7ddf591 Mon Sep 17 00:00:00 2001 From: Tianci Shen Date: Mon, 8 Jan 2024 12:06:17 -0800 Subject: [PATCH] feat: Add logic to clean up logs by last modified date for TimeBasedArchiveRemover Signed-off-by: Tianci Shen --- .../SizeAndTimeBasedRollingPolicy.java | 11 ++++ .../core/rolling/TimeBasedRollingPolicy.java | 13 ++++ .../core/rolling/helper/ArchiveRemover.java | 2 + .../SizeAndTimeBasedArchiveRemover.java | 4 ++ .../helper/TimeBasedArchiveRemover.java | 61 +++++++++++++++++-- ...meBasedRollingWithArchiveRemoval_Test.java | 42 ++++++++++--- 6 files changed, 121 insertions(+), 12 deletions(-) diff --git a/logback-core/src/main/java/ch/qos/logback/core/rolling/SizeAndTimeBasedRollingPolicy.java b/logback-core/src/main/java/ch/qos/logback/core/rolling/SizeAndTimeBasedRollingPolicy.java index 4bbf9b81cc..d7e0d9cf02 100755 --- a/logback-core/src/main/java/ch/qos/logback/core/rolling/SizeAndTimeBasedRollingPolicy.java +++ b/logback-core/src/main/java/ch/qos/logback/core/rolling/SizeAndTimeBasedRollingPolicy.java @@ -14,12 +14,15 @@ package ch.qos.logback.core.rolling; import ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP.Usage; +import ch.qos.logback.core.util.Duration; import ch.qos.logback.core.util.FileSize; public class SizeAndTimeBasedRollingPolicy extends TimeBasedRollingPolicy { FileSize maxFileSize; + Duration checkIncrement = null; + @Override public void start() { SizeAndTimeBasedFNATP sizeAndTimeBasedFNATP = new SizeAndTimeBasedFNATP(Usage.EMBEDDED); @@ -30,6 +33,10 @@ public void start() { addInfo("Archive files will be limited to [" + maxFileSize + "] each."); } + if (checkIncrement != null) { + sizeAndTimeBasedFNATP.setCheckIncrement(checkIncrement); + } + sizeAndTimeBasedFNATP.setMaxFileSize(maxFileSize); timeBasedFileNamingAndTriggeringPolicy = sizeAndTimeBasedFNATP; @@ -47,6 +54,10 @@ public void setMaxFileSize(FileSize aMaxFileSize) { this.maxFileSize = aMaxFileSize; } + public void setCheckIncrement(Duration checkIncrement) { + this.checkIncrement = checkIncrement; + } + @Override public String toString() { return "c.q.l.core.rolling.SizeAndTimeBasedRollingPolicy@" + this.hashCode(); diff --git a/logback-core/src/main/java/ch/qos/logback/core/rolling/TimeBasedRollingPolicy.java b/logback-core/src/main/java/ch/qos/logback/core/rolling/TimeBasedRollingPolicy.java index e75b4bf83b..5cb9426239 100644 --- a/logback-core/src/main/java/ch/qos/logback/core/rolling/TimeBasedRollingPolicy.java +++ b/logback-core/src/main/java/ch/qos/logback/core/rolling/TimeBasedRollingPolicy.java @@ -61,6 +61,7 @@ public class TimeBasedRollingPolicy extends RollingPolicyBase implements Trig TimeBasedFileNamingAndTriggeringPolicy timeBasedFileNamingAndTriggeringPolicy; boolean cleanHistoryOnStart = false; + boolean cleanLogsByLastModifiedDate = false; public void start() { // set the LR for our utility object @@ -109,6 +110,7 @@ public void start() { archiveRemover = timeBasedFileNamingAndTriggeringPolicy.getArchiveRemover(); archiveRemover.setMaxHistory(maxHistory); archiveRemover.setTotalSizeCap(totalSizeCap.getSize()); + archiveRemover.setCleanLogsByLastModifiedDate(cleanLogsByLastModifiedDate); if (cleanHistoryOnStart) { addInfo("Cleaning on start up"); Instant now = Instant.ofEpochMilli(timeBasedFileNamingAndTriggeringPolicy.getCurrentTime()); @@ -271,6 +273,17 @@ public void setCleanHistoryOnStart(boolean cleanHistoryOnStart) { this.cleanHistoryOnStart = cleanHistoryOnStart; } + + /** + * Should archive removal use a file's last modified date to determine deletion? + * Default is false. + * + * @param cleanLogsByLastModifiedDate + */ + public void setCleanLogsByLastModifiedDate(boolean cleanLogsByLastModifiedDate){ + this.cleanLogsByLastModifiedDate = cleanLogsByLastModifiedDate; + } + @Override public String toString() { return "c.q.l.core.rolling.TimeBasedRollingPolicy@" + this.hashCode(); diff --git a/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/ArchiveRemover.java b/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/ArchiveRemover.java index 835081c034..4de0346e07 100644 --- a/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/ArchiveRemover.java +++ b/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/ArchiveRemover.java @@ -31,5 +31,7 @@ public interface ArchiveRemover extends ContextAware { void setTotalSizeCap(long totalSizeCap); + void setCleanLogsByLastModifiedDate(boolean cleanLogsByLastModifiedDate); + Future cleanAsynchronously(Instant now); } \ No newline at end of file diff --git a/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/SizeAndTimeBasedArchiveRemover.java b/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/SizeAndTimeBasedArchiveRemover.java index e4cadcdabc..edfb689a33 100755 --- a/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/SizeAndTimeBasedArchiveRemover.java +++ b/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/SizeAndTimeBasedArchiveRemover.java @@ -29,6 +29,10 @@ public SizeAndTimeBasedArchiveRemover(FileNamePattern fileNamePattern, RollingCa super(fileNamePattern, rc); } + File getParentDir(Instant cleanupCutoff) { + return getParentDir(new File(fileNamePattern.convertMultipleArguments(cleanupCutoff, 0))); + } + protected File[] getFilesInPeriod(Instant instantOfPeriodToClean) { File archive0 = new File(fileNamePattern.convertMultipleArguments(instantOfPeriodToClean, 0)); File parentDir = getParentDir(archive0); diff --git a/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/TimeBasedArchiveRemover.java b/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/TimeBasedArchiveRemover.java index 357e333790..b7758f2602 100644 --- a/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/TimeBasedArchiveRemover.java +++ b/logback-core/src/main/java/ch/qos/logback/core/rolling/helper/TimeBasedArchiveRemover.java @@ -36,6 +36,7 @@ public class TimeBasedArchiveRemover extends ContextAwareBase implements Archive final RollingCalendar rc; private int maxHistory = CoreConstants.UNBOUNDED_HISTORY; private long totalSizeCap = CoreConstants.UNBOUNDED_TOTAL_SIZE_CAP; + private boolean cleanLogsByLastModifiedDate = false; final boolean parentClean; long lastHeartBeat = UNINITIALIZED; @@ -70,10 +71,54 @@ public void clean(Instant now) { addInfo("Multiple periods, i.e. " + periodsElapsed + " periods, seem to have elapsed. This is expected at application start."); } - for (int i = 0; i < periodsElapsed; i++) { - int offset = getPeriodOffsetForDeletionTarget() - i; - Instant instantOfPeriodToClean = rc.getEndOfNextNthPeriod(now, offset); - cleanPeriod(instantOfPeriodToClean); + + if (cleanLogsByLastModifiedDate) { + // Delete old logs based on date the file was last modified + cleanLogsByDateModified(now); + } else { + // Delete old logs based on expected file name + for (int i = 0; i < periodsElapsed; i++) { + int offset = getPeriodOffsetForDeletionTarget() - i; + Instant instantOfPeriodToClean = rc.getEndOfNextNthPeriod(now, offset); + cleanPeriod(instantOfPeriodToClean); + } + } + } + + /** + * Iterates through log files and deletes files outside the rollover window + * Expects the file name to occur before the date specifier + * Does not work well with file patterns that have auxiliary date specifiers + * + * @param now + */ + private void cleanLogsByDateModified(Instant now) { + File filePattern = new File(fileNamePattern.getPattern()); + String fileNameBeforeDateSpecifier = filePattern.getName().split("\\%d\\{.+\\}")[0]; + Instant cleanupCutoff = rc.getEndOfNextNthPeriod(now, getPeriodOffsetForDeletionTarget()); + + File parentDir; + parentDir = getParentDir(cleanupCutoff); + if (parentDir == null) { + addError("Cannot get parent directory"); + return; + } + + File[] matchedFiles; + matchedFiles = parentDir.listFiles((dir, name) -> name.contains(fileNameBeforeDateSpecifier)); + if (matchedFiles == null) { + addError("Failed to find relevant log files"); + return; + } + + for (File file : matchedFiles) { + Instant lastModifiedDate = Instant.ofEpochMilli(file.lastModified()); + if (cleanupCutoff.isAfter(lastModifiedDate)) { + checkAndDeleteFile(file); + } + } + if (parentClean && matchedFiles.length > 0) { + removeFolderIfEmpty(parentDir); } } @@ -148,6 +193,10 @@ protected void descendingSort(File[] matchingFileArray, Instant instant) { // nothing to do in super class } + File getParentDir(Instant cleanupCutoff) { + return getParentDir(new File(fileNamePattern.convert(cleanupCutoff))); + } + File getParentDir(File file) { File absolute = file.getAbsoluteFile(); File parentDir = absolute.getParentFile(); @@ -241,6 +290,10 @@ public void setTotalSizeCap(long totalSizeCap) { this.totalSizeCap = totalSizeCap; } + public void setCleanLogsByLastModifiedDate(boolean cleanLogsByLastModifiedDate) { + this.cleanLogsByLastModifiedDate = cleanLogsByLastModifiedDate; + } + public String toString() { return "c.q.l.core.rolling.helper.TimeBasedArchiveRemover"; } diff --git a/logback-core/src/test/java/ch/qos/logback/core/rolling/TimeBasedRollingWithArchiveRemoval_Test.java b/logback-core/src/test/java/ch/qos/logback/core/rolling/TimeBasedRollingWithArchiveRemoval_Test.java index 00a1199190..17c87e008c 100755 --- a/logback-core/src/test/java/ch/qos/logback/core/rolling/TimeBasedRollingWithArchiveRemoval_Test.java +++ b/logback-core/src/test/java/ch/qos/logback/core/rolling/TimeBasedRollingWithArchiveRemoval_Test.java @@ -16,9 +16,11 @@ import static ch.qos.logback.core.CoreConstants.DAILY_DATE_PATTERN; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; import java.io.File; import java.io.FileFilter; +import java.io.IOException; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -338,10 +340,10 @@ public void dailyChronologSizeBasedRolloverWithSecondPhase() { checkDirPatternCompliance(maxHistory + 1); } - void logTwiceAndStop(long currentTime, String fileNamePattern, int maxHistory, long durationInMillis) { + void logTwiceAndStop(long currentTime, String fileNamePattern, int maxHistory, long durationInMillis, boolean cleanLogsByLastModifiedDate) { ConfigParameters params = new ConfigParameters(currentTime).fileNamePattern(fileNamePattern) .maxHistory(maxHistory); - buildRollingFileAppender(params, DO_CLEAN_HISTORY_ON_START); + buildRollingFileAppender(params, DO_CLEAN_HISTORY_ON_START, cleanLogsByLastModifiedDate); rfa.doAppend("Hello ----------------------------------------------------------" + new Date(currentTime)); currentTime += durationInMillis / 2; add(tbrp.compressionFuture); @@ -359,7 +361,7 @@ public void cleanHistoryOnStartWithHourPattern() { String fileNamePattern = randomOutputDir + "clean-%d{" + DAILY_HOUR_PATTERN + "}.txt"; int maxHistory = 3; for (int i = 0; i <= 5; i++) { - logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR); + logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR, DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE); simulatedTime += MILLIS_IN_HOUR; } checkFileCount(expectedCountWithoutFolders(maxHistory)); @@ -379,7 +381,7 @@ public void cleanHistoryOnStartWithHourPatternWithCollisions() { String fileNamePattern = randomOutputDir + "clean-%d{HH}.txt"; int maxHistory = 3; for (int i = 0; i <= 5; i++) { - logTwiceAndStop(now, fileNamePattern, maxHistory, MILLIS_IN_DAY); + logTwiceAndStop(now, fileNamePattern, maxHistory, MILLIS_IN_DAY, DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE); now = now + MILLIS_IN_HOUR; } checkFileCount(expectedCountWithoutFolders(maxHistory)); @@ -391,7 +393,7 @@ public void cleanHistoryOnStartWithDayPattern() { String fileNamePattern = randomOutputDir + "clean-%d{" + DAILY_DATE_PATTERN + "}.txt"; int maxHistory = 3; for (int i = 0; i <= 5; i++) { - logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_DAY); + logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_DAY, DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE); simulatedTime += MILLIS_IN_DAY; } checkFileCount(expectedCountWithoutFolders(maxHistory)); @@ -403,12 +405,32 @@ public void cleanHistoryOnStartWithHourDayPattern() { String fileNamePattern = randomOutputDir + "clean-%d{yyyy-MM-dd-HH}.txt"; int maxHistory = 3; for (int i = 0; i <= 5; i++) { - logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR); + logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR, DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE); simulatedTime += MILLIS_IN_HOUR; } checkFileCount(expectedCountWithoutFolders(maxHistory)); } + @Test + public void cleanLogsByLastModifiedDateWithHourDayPattern() throws IOException { + long simulatedTime = WED_2016_03_23_T_230705_CET; + String fileNamePattern = randomOutputDir + "clean-%d{yyyy-MM-dd-HH}.txt"; + int maxHistory = 3; + logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR, DO_CLEAN_LOGS_BY_LAST_MODIFIED_DATE); + simulatedTime += MILLIS_IN_HOUR; + File fileToDelete = new File(randomOutputDir + "clean-0000-00-00-00.txt"); + if (!fileToDelete.exists()) { + assertTrue(fileToDelete.createNewFile()); + } + assertTrue(fileToDelete.setLastModified(0)); + for (int i = 0; i <= 2; i++) { + logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR, DO_CLEAN_LOGS_BY_LAST_MODIFIED_DATE); + simulatedTime += MILLIS_IN_HOUR; + } + checkFileCount(expectedCountWithoutFolders(maxHistory)); + assertFalse(fileToDelete.exists()); + } + int expectedCountWithoutFolders(int maxHistory) { return maxHistory + 1; } @@ -422,7 +444,7 @@ int expectedCountWithFolders(int maxHistory, boolean withExtraFolder) { return result; } - void buildRollingFileAppender(ConfigParameters cp, boolean cleanHistoryOnStart) { + void buildRollingFileAppender(ConfigParameters cp, boolean cleanHistoryOnStart, boolean cleanLogsByLastModifiedDate) { rfa.setContext(context); rfa.setEncoder(encoder); tbrp.setContext(context); @@ -431,6 +453,7 @@ void buildRollingFileAppender(ConfigParameters cp, boolean cleanHistoryOnStart) tbrp.setTotalSizeCap(new FileSize(cp.sizeCap)); tbrp.setParent(rfa); tbrp.setCleanHistoryOnStart(cleanHistoryOnStart); + tbrp.setCleanLogsByLastModifiedDate(cleanLogsByLastModifiedDate); tbrp.timeBasedFileNamingAndTriggeringPolicy = tbfnatp; tbrp.timeBasedFileNamingAndTriggeringPolicy.setCurrentTime(cp.simulatedTime); tbrp.start(); @@ -440,10 +463,13 @@ void buildRollingFileAppender(ConfigParameters cp, boolean cleanHistoryOnStart) boolean DO_CLEAN_HISTORY_ON_START = true; boolean DO_NOT_CLEAN_HISTORY_ON_START = false; + boolean DO_CLEAN_LOGS_BY_LAST_MODIFIED_DATE = true; + boolean DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE = false; + long logOverMultiplePeriods(ConfigParameters cp) { - buildRollingFileAppender(cp, DO_NOT_CLEAN_HISTORY_ON_START); + buildRollingFileAppender(cp, DO_NOT_CLEAN_HISTORY_ON_START, DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE); int runLength = cp.simulatedNumberOfPeriods * ticksPerPeriod; int startInactivityIndex = cp.startInactivity * ticksPerPeriod;