Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove logaggregator #439

Merged
merged 13 commits into from
Sep 15, 2023
38 changes: 34 additions & 4 deletions logprep/run_logprep.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
from logprep._version import get_versions
from logprep.processor.base.rule import Rule
from logprep.runner import Runner
from logprep.util.aggregating_logger import AggregatingLogger
from logprep.util.auto_rule_tester.auto_rule_corpus_tester import RuleCorpusTester
from logprep.util.auto_rule_tester.auto_rule_tester import AutoRuleTester
from logprep.util.configuration import Configuration, InvalidConfigurationError
Expand All @@ -25,6 +24,31 @@
from logprep.util.schema_and_rule_checker import SchemaAndRuleChecker
from logprep.util.time_measurement import TimeMeasurement

from logging import (
getLogger,
CRITICAL,
FATAL,
ERROR,
WARNING,
INFO,
DEBUG,
NOTSET,
ekneg54 marked this conversation as resolved.
Show resolved Hide resolved
basicConfig,
Logger,
)
from logging.handlers import SysLogHandler

name_to_level = {
"CRITICAL": CRITICAL,
"FATAL": FATAL,
"ERROR": ERROR,
"WARN": WARNING,
"WARNING": WARNING,
"INFO": INFO,
"DEBUG": DEBUG,
"NOTSET": NOTSET,
}
ekneg54 marked this conversation as resolved.
Show resolved Hide resolved

warnings.simplefilter("always", DeprecationWarning)
logging.captureWarnings(True)

Expand Down Expand Up @@ -137,10 +161,16 @@ def get_versions_string(args) -> str:
return version_string


def _setup_logger(args, config):
def _setup_logger(args, config: Configuration):
try:
AggregatingLogger.setup(config, logger_disabled=args.disable_logging)
logger = AggregatingLogger.create("Logprep")
log_config = config.get("logger", {})
log_level_name = log_config.get("level", "INFO")
ekneg54 marked this conversation as resolved.
Show resolved Hide resolved
log_level = name_to_level.get(log_level_name)
ekneg54 marked this conversation as resolved.
Show resolved Hide resolved
basicConfig(
level=log_level, format="%(asctime)-15s %(name)-5s %(levelname)-8s: %(message)s"
)
logger = logging.getLogger("Logprep")
logger.info(f"Log level set to '{log_level_name}'")
ekneg54 marked this conversation as resolved.
Show resolved Hide resolved
for version in get_versions_string(args).split("\n"):
logger.info(version)
except BaseException as error: # pylint: disable=broad-except
Expand Down
2 changes: 1 addition & 1 deletion logprep/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ def start(self):
self._continue_iterating.value = True
self._schedule_config_refresh_job()
self._logger.info("Startup complete")
self._logger.debug("Runner iterating")
for _ in self._keep_iterating():
self._loop()
self.stop()
Expand All @@ -194,7 +195,6 @@ def start(self):

def _loop(self):
self.scheduler.run_pending()
self._logger.debug("Runner iterating")
self._manager.restart_failed_pipeline()
# Note: We are waiting half the timeout because when shutting down, we also have to
# wait for the logprep's timeout before the shutdown is actually initiated.
Expand Down
96 changes: 0 additions & 96 deletions logprep/util/aggregating_logger.py

This file was deleted.

97 changes: 0 additions & 97 deletions logprep/util/log_aggregator.py

This file was deleted.

2 changes: 1 addition & 1 deletion tests/unit/test_run_logprep.py
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ def test_main_calls_runner_stop_on_any_exception(self, mock_stop, mock_start):
mock_stop.assert_called()

def test_logprep_exits_if_logger_can_not_be_created(self):
with mock.patch("logprep.run_logprep.AggregatingLogger.create") as mock_create:
with mock.patch("logging.getLogger") as mock_create:
mock_create.side_effect = BaseException
config_path = "quickstart/exampledata/config/pipeline.yml"
with mock.patch("sys.argv", ["logprep", config_path]):
Expand Down
Loading