Add another logging destination; switch logger (#1847)

- switch to logback - use only one logback.xml config file - introduce logging of ETL to a separate file - change log pattern to be more concise - exclude some logging libraries from dependencies - use adapter "log4j-to-slf4j" to log elasticsearch log4j2 logs - add mnising log messages
hbz · Oct 23, 2023 · f64239b · f64239b
1 parent 7809673
commit f64239b
Show file tree

Hide file tree

Showing 7 changed files with 129 additions and 97 deletions.
diff --git a/pom.xml b/pom.xml
@@ -9,7 +9,7 @@
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 		<target.jdk>1.8</target.jdk>
 		<junit.version>4.8.2</junit.version>
-		<logback.version>0.9.30</logback.version>
+		<logback.version>1.3.11</logback.version>
 	</properties>
 	<dependencies>
 		<dependency>
@@ -26,6 +26,12 @@
 			<groupId>org.metafacture</groupId>
 			<artifactId>metafacture-json</artifactId>
 			<version>5.6.0</version>
+			<exclusions>
+				<exclusion>
+					<groupId>commons-logging</groupId>
+					<artifactId>commons-logging</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.metafacture</groupId>
@@ -132,12 +138,26 @@
 					<groupId>log4j</groupId>
 					<artifactId>log4j-api</artifactId>
 				</exclusion>
+				<exclusion>
+					<groupId>commons-logging</groupId>
+					<artifactId>commons-logging</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.apache.logging.log4j</groupId>
+					<artifactId>log4j-api</artifactId>
+				</exclusion>
 			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>org.elasticsearch.client</groupId>
 			<artifactId>transport</artifactId>
 			<version>5.6.3</version>
+			<exclusions>
+				<exclusion>
+					<groupId>commons-logging</groupId>
+					<artifactId>commons-logging</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
 		<dependency>
 			<groupId>junit</groupId>
@@ -164,6 +184,26 @@
 			<artifactId>jaxb-api</artifactId>
 			<version>2.3.0</version>
 		</dependency>
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-classic</artifactId>
+			<version>${logback.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-core</artifactId>
+			<version>${logback.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<version>2.0.9</version>
+		</dependency>
+		<dependency>
+		<groupId>org.apache.logging.log4j</groupId>
+		<artifactId>log4j-to-slf4j</artifactId>
+		<version>2.9.0</version>
+		</dependency>
 	</dependencies>
 	<build>
 		<resources>

diff --git a/src/main/java/de/hbz/lobid/helper/Etikett.java b/src/main/java/de/hbz/lobid/helper/Etikett.java
@@ -233,4 +233,4 @@ public Map<String, String> getMultilangLabel() {
 		return multilangLabel;
 	}
 
-}
+}
diff --git a/src/main/resources/log4j.xml b/src/main/resources/log4j.xml
diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml
diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml
@@ -0,0 +1,72 @@
+<configuration>
+  <appender name="FILE_PLAY" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>./logs/play.log</file>
+    <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+        <!-- Daily rollover with compression -->
+        <fileNamePattern>play-log-%d{yyyy-MM-dd}.%i.gz</fileNamePattern>
+        <!-- keep 3 weeks worth of history -->
+        <maxHistory>21</maxHistory>
+        <!-- roll log file > 100MB -->
+        <maxFileSize>100MB</maxFileSize>
+    </rollingPolicy>
+    <encoder>
+      <pattern>%date{yyyy-MM-dd HH:mm:ss ZZZZ} [%level] from %logger{36} - %message%n%xException</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="FILE_ETL" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>./logs/etl.log</file>
+    <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+        <!-- Daily rollover with compression -->
+        <fileNamePattern>etl-log-%d{yyyy-MM-dd}.%i.gz</fileNamePattern>
+        <!-- keep 3 weeks worth of history -->
+        <maxHistory>21</maxHistory>
+        <!-- roll log file > 100MB -->
+        <maxFileSize>100MB</maxFileSize>
+    </rollingPolicy>
+    <encoder>
+        <pattern>%date{yyyy-MM-dd HH:mm:ss} [%level] %logger{52} in %thread - %message%n%xException</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>%logger{15} - %message%n%xException{10}</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="ASYNCSTDOUT" class="ch.qos.logback.classic.AsyncAppender">
+    <appender-ref ref="STDOUT" />
+  </appender>
+
+  <!-- Off these ones as they are annoying, and anyway we manage configuration ourself -->
+  <logger name="com.avaje.ebean.config.PropertyMapLoader" level="OFF" />
+  <logger name="com.avaje.ebeaninternal.server.core.XmlConfigLoader" level="OFF" />
+  <logger name="com.avaje.ebeaninternal.server.lib.BackgroundThread" level="OFF" />
+  <logger name="com.gargoylesoftware.htmlunit.javascript" level="OFF" />
+
+  <logger name="play" level="INFO" >
+    <appender-ref ref="FILE_PLAY" />
+  </logger>
+
+  <logger name="application" level="INFO" >
+    <appender-ref ref="FILE_ETL" />
+  </logger>
+  <logger name="org.metafacture" level="INFO" >
+    <appender-ref ref="FILE_ETL" />
+  </logger>
+  <logger name="org.lobid" level="INFO" >
+    <appender-ref ref="FILE_ETL" />
+  </logger>
+  <logger name="de.hbz.lobid" level="INFO" >
+    <appender-ref ref="FILE_ETL" />
+  </logger>
+  <logger name="org.elasticsearch" level="INFO" >
+    <appender-ref ref="FILE_ETL" />
+  </logger>
+
+  <root level="INFO">
+    <appender-ref ref="ASYNCSTDOUT" />
+  </root>
+
+</configuration>
diff --git a/web/app/controllers/resources/WebhookAlmaFix.java b/web/app/controllers/resources/WebhookAlmaFix.java
@@ -5,6 +5,7 @@
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import org.lobid.resources.run.AlmaMarcXmlFix2lobidJsonEs;
+
 import play.Logger;
 import play.mvc.Controller;
 import play.mvc.Http;
@@ -87,19 +88,22 @@ public static Result updateAlma(final String GIVEN_TOKEN) {
         Logger.error(msg);
         AlmaMarcXmlFix2lobidJsonEs.sendMail("Triggering of " + ETL_OF + KIND, false,
             msg);
+        Logger.warn(msg);
         return status(500, msg);
       }
     }
     catch (IOException e) {
-      msg = composeMessage(e.getMessage());
+      msg = composeMessage("Problems with data file\n" + e.getMessage());
       AlmaMarcXmlFix2lobidJsonEs.sendMail("Triggering of " + ETL_OF + KIND, false,
           msg);
-      return status(500, "Problems with data file\n" + msg);
+      Logger.warn(msg);
+      return status(500, msg);
     }
     if (AlmaMarcXmlFix2lobidJsonEs.threadAlreadyStarted) {
       msg = composeMessage(String.format(MSG_UPDATE_ALREADY_RUNNING, indexNameOfUpdate));
       AlmaMarcXmlFix2lobidJsonEs.sendMail(ETL_OF + KIND, false,
           msg);
+      Logger.warn(msg);
       return status(423, msg);
     }
     msg = composeMessage(String.format(msgStartEtl, KIND));
@@ -156,17 +160,21 @@ public static Result basedumpAlma(final String GIVEN_TOKEN) {
         Logger.error(msg);
         AlmaMarcXmlFix2lobidJsonEs.sendMail("Triggering of " + ETL_OF + KIND, false,
             msg);
+        Logger.warn(msg);
         return status(500, msg);
       }
     }
     catch (IOException e) {
-      return status(500, "Problems with data file\n" + e);
+      msg = composeMessage("IO Problems with data file\n" + e);
+      Logger.warn(msg);
+      return status(500, msg);
     }
     createIndexNameOfBasedump = indexNameOfBasedump + "-" + LocalDateTime.now()
         .format(DateTimeFormatter.ofPattern("yyyyMMdd-kkmm"));
     if (AlmaMarcXmlFix2lobidJsonEs.threadAlreadyStarted) {
       msg = composeMessage(String.format(MSG_CREATE_INDEX_ALREADY_RUNNING, createIndexNameOfBasedump));
       AlmaMarcXmlFix2lobidJsonEs.sendMail(ETL_OF + KIND, false, msg);
+      Logger.warn(msg);
       return status(423, msg);
     }
     msg = composeMessage(String.format(msgStartEtl, KIND));
@@ -200,29 +208,31 @@ public static Result switchEsAlias(final String GIVEN_TOKEN) {
     String msg = composeMessage(subject);
     Logger.info(msg);
     if (!GIVEN_TOKEN.equals(token)) {
-      Logger.info("Wrong token: " + GIVEN_TOKEN);
       return wrongToken(subject, GIVEN_TOKEN);
     }
     boolean success;
     if (AlmaMarcXmlFix2lobidJsonEs.threadAlreadyStarted) {
       msg = composeMessage(String.format(MSG_CREATE_INDEX_ALREADY_RUNNING, createIndexNameOfBasedump));
       AlmaMarcXmlFix2lobidJsonEs.sendMail("Failed: " + subject, false, msg);
+      Logger.warn(msg);
       return status(423, msg);
     }
     AlmaMarcXmlFix2lobidJsonEs.setSwitchVariables(alias1, alias2, clusterHost, basedumpSwitchMindocs, basedumpSwitchMinsize);
     success = AlmaMarcXmlFix2lobidJsonEs.switchAlias();
     if (success) {
       msg = composeMessage(AlmaMarcXmlFix2lobidJsonEs.MSG_SUCCESS + subject);
+      Logger.info(msg);
       return ok(msg);
     }
     msg = composeMessage(AlmaMarcXmlFix2lobidJsonEs.MSG_FAIL + subject);
+    Logger.warn(msg);
     return internalServerError(msg);
   }
 
   private static Result wrongToken(final String KIND,
       final String GIVEN_TOKEN) {
     String msg = composeMessage(String.format(msgWrongToken, GIVEN_TOKEN, KIND));
-    Logger.error(msg);
+    Logger.warn(msg);
     AlmaMarcXmlFix2lobidJsonEs.sendMail(KIND, false, msg);
     return forbidden(msg);
   }

diff --git a/web/conf/logback.xml b/web/conf/logback.xml
-Original file line number
+Diff line change
@@ Expand Up / @@ -233,4 +233,4 @@ public Map<String, String> getMultilangLabel() { @@
     		return multilangLabel;
     	}
-    }
+    }