Skip to content

Commit

Permalink
fix: added logger name in props (#59)
Browse files Browse the repository at this point in the history
- sometimes app uses spark logger instead of Checkita, and because of
this we got an error
- in addition moved default logging settings from code to log4j2.properties in resources folder.
  • Loading branch information
dmrmlvv authored Sep 30, 2024
1 parent 51050a8 commit 1a1d5a8
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 46 deletions.
32 changes: 32 additions & 0 deletions checkita-core/src/main/resources/log4j2.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Set everything to be logged to the console
# Log level for all external packages is "warn" unless defined explicitly.
# Log level for Checkita classes is defined explicitly further in this file.
rootLogger.level = warn
rootLogger.appenderRef.stdout.ref = console

appender.console.type = Console
appender.console.name = console
appender.console.target = SYSTEM_OUT
appender.console.layout.type = PatternLayout
appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n%ex

# Settings to quiet third party logs that are too verbose
logger.jetty2.name = org.sparkproject.jetty.util.component.AbstractLifeCycle
logger.jetty2.level = error

# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs
# in SparkSQL with Hive support:
logger.metastore.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
logger.metastore.level = fatal
logger.hive_functionregistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry
logger.hive_functionregistry.level = error

# Parquet related log level is set to error as well:
logger.parquet.name = org.apache.parquet.CorruptStatistics
logger.parquet.level = error
logger.parquet2.name = parquet.CorruptStatistics
logger.parquet2.level = error

# EXPLICIT LOG LEVEL FOR CHECKITA CLASSES:
logger.checkita.name = org.checkita
logger.checkita.level = debug
49 changes: 3 additions & 46 deletions checkita-core/src/main/scala/org/checkita/dqf/utils/Logging.scala
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,8 @@ trait Logging {
* Gets log4j2 properties from corresponding file in the resources directory.
* @return Input stream with log4j2 properties
*/
private def getPropsFromResources: Option[InputStream] = Try(
private def getPropsFromResources: InputStream =
getClass.getResourceAsStream("/log4j2.properties")
).toOption.flatMap(Option(_)) // take care of null input stream

/**
* Gets log4j2 properties from corresponding file in the working directory.
Expand All @@ -41,49 +40,6 @@ trait Logging {
new FileInputStream("log4j2.properties")
).toOption.flatMap(Option(_)) // take care of null input stream

/**
* Default logging configuration if user-defined one wasn't provided.
* @return Input stream with default log4j properties
*/
private def getDefaultProps: InputStream = {
val defaultProperties =
"""
|# Set everything to be logged to the console
|# Log level for all external packages is "warn" unless defined explicitly.
|# Log level for Checkita classes is defined explicitly further in this file.
|rootLogger.level = warn
|rootLogger.appenderRef.stdout.ref = console
|
|appender.console.type = Console
|appender.console.name = console
|appender.console.target = SYSTEM_OUT
|appender.console.layout.type = PatternLayout
|appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n%ex
|
|# Settings to quiet third party logs that are too verbose
|logger.jetty2.name = org.sparkproject.jetty.util.component.AbstractLifeCycle
|logger.jetty2.level = error
|
|# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs
|# in SparkSQL with Hive support:
|logger.metastore.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
|logger.metastore.level = fatal
|logger.hive_functionregistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry
|logger.hive_functionregistry.level = error
|
|# Parquet related log level is set to error as well:
|logger.parquet.name = org.apache.parquet.CorruptStatistics
|logger.parquet.level = error
|logger.parquet2.name = parquet.CorruptStatistics
|logger.parquet2.level = error
|
|# EXPLICIT LOG LEVEL FOR CHECKITA CLASSES:
|logger.checkita.name = org.checkita
|logger.checkita.level = debug
|""".stripMargin
toInputStream(defaultProperties, StandardCharsets.UTF_8)
}

/**
* Refactors log4j properties stream with actual Checkita logger level provided at application start.
* @param is Input stream with log4j2 properties
Expand All @@ -93,6 +49,7 @@ trait Logging {
private def refactorProperties(is: InputStream, lvl: log4j2.Level): InputStream = {
val props = new Properties()
props.load(is)
props.setProperty("logger.checkita.name", "org.checkita")
props.setProperty("logger.checkita.level", lvl.toString)
toInputStream(props.asScala.map{ case (k, v) => s"$k = $v"}.mkString("\n"), StandardCharsets.UTF_8)
}
Expand All @@ -119,7 +76,7 @@ trait Logging {
* @param lvl Root logger level defined at application start
*/
def initLogger(lvl: log4j2.Level): Unit = {
val configStream = getPropsFromResources orElse getPropsFromWorkDir getOrElse getDefaultProps
val configStream = getPropsFromWorkDir getOrElse getPropsFromResources
val refactoredStream = refactorProperties(configStream, lvl)
reconfigureLog4J2(refactoredStream)
}
Expand Down

0 comments on commit 1a1d5a8

Please sign in to comment.