Merge pull request #84 from adobe/83-solve-issue-with-huge-log-files

Fixed #83 implemented more careful log handling
adobe · Mar 18, 2024 · 76469c3 · 76469c3
2 parents 29ab2e8 + 43ad73d
commit 76469c3
Show file tree

Hide file tree

Showing 5 changed files with 70 additions and 23 deletions.
diff --git a/ReleaseNotes.md b/ReleaseNotes.md
@@ -5,7 +5,9 @@
 * [#72 Provide the stack trace of the root cause of the errors](https://github.com/adobe/bridgeService/issues/72). With issue #9 we discovered that the stack trace should be that of the original cause.
 * [#78 Trimming error messages](https://github.com/adobe/bridgeService/issues/78). We now trim the error messages.
 * [#80 Removing stack trace for certain errors](https://github.com/adobe/bridgeService/issues/80). We removed the stack trace for errors, where the method or class cannot be found or accessed.
-* We have now introduced a dedicated exception or when a public Method in a Java class with no scope definition. This created linkage errors. 
+* [#83 Adapting log files for production](https://github.com/adobe/bridgeService/issues/83). We now limit the log files so not too much log data is stored. More details to be found in [the Technical Doc](docs/Technical.md).
+* Moved the log level for the loading of the classes to 'trace' instead of 'debug'.
+* We have now introduced a dedicated exception for when a public Method in a Java class with no scope definition. This created linkage errors. 
 
 ## 2.11.14
 * [#61 Allowing Map/JSON return type](https://github.com/adobe/bridgeService/issues/61). Until recently we were not able to correctly accept map and JSON Objects.

diff --git a/docs/Technical.md b/docs/Technical.md
@@ -4,11 +4,11 @@ This document deals with the general technical elements of the project.
 ## Error Handling
 We currently manage the following error types.
 
-| Error Code | Exceptions                                                                         | Description                                                |
-|------------|------------------------------------------------------------------------------------|------------------------------------------------------------|
-| 404        | NonExistentJavaObjectException , JsonProcessingException, AmbiguousMethodException | Errors due to client side manipulations, and payloads.     |
-| 408        | IBSTimeOutException                                                                | When the underlying java call takes too long to execute.   |
-| 500        | IBSConfigurationException, IBSRunTimeException, TargetJavaMethodCallException      | Errors happening when calling the underlying Java classes. |
+| Error Code | Exceptions                                                                                                           | Description                                                |
+|------------|----------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------|
+| 404        | NonExistentJavaObjectException , JsonProcessingException, AmbiguousMethodException, JavaObjectInaccessibleException  | Errors due to client side manipulations, and payloads.     |
+| 408        | IBSTimeOutException                                                                                                  | When the underlying java call takes too long to execute.   |
+| 500        | IBSConfigurationException, IBSRunTimeException, TargetJavaMethodCallException                                        | Errors happening when calling the underlying Java classes. |
 
 ## Managing Contexts and Static Variables
 We have three modes of Integrity management :
@@ -33,4 +33,12 @@ This page describes the state of how Calls access static variables. The structur
 1. Environment Variable Setting
 2. Calling Java
 3. (optional) Environment Variable Setting
-4. Calling Java
+4. Calling Java
+
+## Log Handling
+The logs are by default deleted after a certain time. In production, we have opted for the following rules:
+* They are stored in the directory 'ibs_output'
+* When the size becomes larger than 50MB, we archive the contents to a zip file
+* The zip files are deleted on a regular basis based on the following rules:
+  * The files exceed 3GB 
+  * The files are more than 10 days old.
diff --git a/...rvice/src/main/java/com/adobe/campaign/tests/bridge/service/IntegroBridgeClassLoader.java b/...rvice/src/main/java/com/adobe/campaign/tests/bridge/service/IntegroBridgeClassLoader.java
@@ -46,10 +46,10 @@ private synchronized  Class getClass(String in_classPath) throws ClassNotFoundEx
         // is this class already loaded?
             Class cls = super.findLoadedClass(in_classPath);
         if (cls != null) {
-            log.debug("Class {} has already been loaded.",in_classPath);
+            log.trace("Class {} has already been loaded.",in_classPath);
             return cls;
         } else {
-            log.debug("Class {} has not been loaded. Loading now.", in_classPath);
+            log.trace("Class {} has not been loaded. Loading now.", in_classPath);
         }
 
 
@@ -89,7 +89,7 @@ private synchronized  Class getClass(String in_classPath) throws ClassNotFoundEx
      */
     @Override
     public Class loadClass(String in_classFullPath) throws ClassNotFoundException {
-        log.debug("Preparing class {}", in_classFullPath);
+        log.trace("Preparing class {}", in_classFullPath);
 
         if (ConfigValueHandlerIBS.INTEGRITY_PACKAGE_INJECTION_MODE.is("manual", "semi-manual")) {
             if (isClassAmongPackagePaths(in_classFullPath)) {

diff --git a/integroBridgeService/src/main/resources/log4j2.xml b/integroBridgeService/src/main/resources/log4j2.xml
@@ -12,17 +12,36 @@
 <Configuration xmlns="http://logging.apache.org/log4j/2.0/config">
 
     <Appenders>
-        <File name="FILE" fileName="ibs_output/logs/logfile.log">
-            <PatternLayout pattern="%-5p | %d{yyyy-MM-dd HH:mm:ss} | [%t] %C{2} (%F:%L) - %m%n"/>
-        </File>
+        <RollingFile name="RollingFile" fileName="ibs_output/logfile.log"
+                     filePattern="ibs_output/logfile-$${date:yyyy-MM-dd}.log.gz">
+            <PatternLayout>
+                <Pattern>%-5p | %d{yyyy-MM-dd HH:mm:ss} | [%t] %C{2} (%F:%L) - %m%n</Pattern>
+            </PatternLayout>
+            <Policies>
+                <SizeBasedTriggeringPolicy size="50MB"/>
+            </Policies>
+
+            <DefaultRolloverStrategy max="12">
+                <Delete basePath="ibs_output" maxDepth="2">
+                    <IfFileName glob="logfile*.log.gz" >
+                        <IfAny>
+                            <IfAccumulatedFileSize exceeds="3GB" />
+                            <IfLastModified age="P10D" />
+                        </IfAny>
+                    </IfFileName>
+                </Delete>
+            </DefaultRolloverStrategy>
+        </RollingFile>
         <Console name="STDOUT" target="SYSTEM_OUT">
             <PatternLayout pattern="%-5p | %d{yyyy-MM-dd HH:mm:ss} | [%t] %C{2} (%F:%L) - %m%n"/>
         </Console>
+
     </Appenders>
 
 
   <Loggers>
     <Root level="DEBUG">
+        <AppenderRef ref="RollingFile" level="DEBUG"/>
       <AppenderRef ref="FILE" level="DEBUG"/>
       <AppenderRef ref="STDOUT" level="INFO"/>
       <!--You can set ref="STDOUT" level ="DEBUG" to allow all the logs in the console locally-->

diff --git a/integroBridgeService/src/test/resources/log4j2.xml b/integroBridgeService/src/test/resources/log4j2.xml
@@ -12,20 +12,38 @@
 <Configuration xmlns="http://logging.apache.org/log4j/2.0/config">
 
     <Appenders>
-        <File name="FILE" fileName="ibstest_output/logs/logfile.log">
-            <PatternLayout pattern="%-5p | %d{yyyy-MM-dd HH:mm:ss} | [%t] %C{2} (%F:%L) - %m%n"/>
-        </File>
+        <RollingFile name="RollingFile" fileName="ibstest_output/logfile.log"
+                     filePattern="ibstest_output/logfile-$${date:yyyy-MM}.log.gz">
+            <PatternLayout>
+                <Pattern>%-5p | %d{yyyy-MM-dd HH:mm:ss} | [%t] %C{2} (%F:%L) - %m%n</Pattern>
+            </PatternLayout>
+            <Policies>
+                <SizeBasedTriggeringPolicy size="5MB"/>
+            </Policies>
+
+            <DefaultRolloverStrategy max="10">
+                <Delete basePath="ibstest_output" maxDepth="2">
+                    <IfFileName glob="logfile*.log.gz" >
+                        <IfAny>
+                            <IfAccumulatedFileSize exceeds="50MB" />
+                            <IfLastModified age="P90D" />
+                        </IfAny>
+                    </IfFileName>
+                </Delete>
+            </DefaultRolloverStrategy>
+
+        </RollingFile>
         <Console name="STDOUT" target="SYSTEM_OUT">
             <PatternLayout pattern="%-5p | %d{yyyy-MM-dd HH:mm:ss} | [%t] %C{2} (%F:%L) - %m%n"/>
         </Console>
     </Appenders>
 
+    <Loggers>
+        <Root level="DEBUG">
+            <AppenderRef ref="STDOUT" level="TRACE"/>
+            <AppenderRef ref="RollingFile" level="DEBUG"/>
 
-  <Loggers>
-    <Root level="DEBUG">
-      <AppenderRef ref="FILE" level="DEBUG"/>
-      <AppenderRef ref="STDOUT" level="TRACE"/>
-      <!--You can set ref="STDOUT" level ="DEBUG" to allow all the logs in the console locally-->
-    </Root>
-  </Loggers>
+            <!--You can set ref="STDOUT" level ="DEBUG" to allow all the logs in the console locally-->
+        </Root>
+    </Loggers>
 </Configuration>