diff --git a/src/alogamous/__main__.py b/src/alogamous/__main__.py index 26250de..afa3b74 100644 --- a/src/alogamous/__main__.py +++ b/src/alogamous/__main__.py @@ -40,15 +40,14 @@ [ # echo_analyzer.EchoAnalyzer(), daily_count_analyzer.DailyCountAnalyzer(line_parser), - error_counter_analyzer.ErrorCounterAnalyzer(), - flag_duplicate_log_messages.FlagDuplicateLogMessages(), + error_counter_analyzer.ErrorCounterAnalyzer(line_parser), + flag_duplicate_log_messages.FlagDuplicateLogMessages(line_parser), line_count_analyzer.LineCountAnalyzer(), format_analyzer.FormatAnalyzer(line_parser), - warning_analyzer.WarningAnalyzer(), loginfo_analyzer.InfoAnalyzer(line_parser), stack_trace_analyzer.StackTraceAnalyzer(line_parser), # startup_header_analyzer.StartupHeaderAnalyzer(line_parser), - warning_analyzer.WarningAnalyzer(), + warning_analyzer.WarningAnalyzer(line_parser), ], reader.read(), output_file, diff --git a/src/alogamous/daily_warning_analyzer.py b/src/alogamous/daily_warning_analyzer.py index fd35802..d8f83f6 100644 --- a/src/alogamous/daily_warning_analyzer.py +++ b/src/alogamous/daily_warning_analyzer.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datetime from alogamous import analyzer, log_line_parser diff --git a/src/alogamous/error_counter_analyzer.py b/src/alogamous/error_counter_analyzer.py index 733d356..6983881 100644 --- a/src/alogamous/error_counter_analyzer.py +++ b/src/alogamous/error_counter_analyzer.py @@ -1,12 +1,14 @@ -from alogamous import analyzer +from alogamous import analyzer, log_line_parser class ErrorCounterAnalyzer(analyzer.Analyzer): - def __init__(self): + def __init__(self, line_parser): + self.parser = line_parser self.error_number = 0 def read_log_line(self, line): - if "ERROR" in line: + parsed_line = self.parser.parse(line) + if parsed_line["type"] == log_line_parser.LineType.LOG_LINE and parsed_line["level"].lower() == "error": self.error_number += 1 def report(self, out_stream): diff --git a/src/alogamous/flag_duplicate_log_messages.py b/src/alogamous/flag_duplicate_log_messages.py index 6e21188..f156796 100644 --- a/src/alogamous/flag_duplicate_log_messages.py +++ b/src/alogamous/flag_duplicate_log_messages.py @@ -1,21 +1,24 @@ -from alogamous import analyzer +from alogamous import analyzer, log_line_parser class FlagDuplicateLogMessages(analyzer.Analyzer): - def __init__(self): + def __init__(self, line_parser): + self.parser = line_parser self.logMessages = set() self.duplicateMessages = set() def read_log_line(self, line): - message = line.split("-")[-1].strip() - if message in self.logMessages: - self.duplicateMessages.add(message) - else: - self.logMessages.add(message) + parsed_line = self.parser.parse(line) + if parsed_line["type"] == log_line_parser.LineType.LOG_LINE: + message = parsed_line["message"] + if message in self.logMessages: + self.duplicateMessages.add(message) + else: + self.logMessages.add(message) def report(self, out_stream): if len(self.duplicateMessages) > 0: - out_stream.write("Duplicate Log Messages:\n") - out_stream.write("\n".join(self.duplicateMessages)) + out_stream.write("Duplicate Log Messages:\n- ") + out_stream.write("\n- ".join(self.duplicateMessages)) else: out_stream.write("No duplicate log messages") diff --git a/src/alogamous/loginfo_analyzer.py b/src/alogamous/loginfo_analyzer.py index 34e8cf0..ec0c868 100644 --- a/src/alogamous/loginfo_analyzer.py +++ b/src/alogamous/loginfo_analyzer.py @@ -10,7 +10,7 @@ def __init__(self, parser): def read_log_line(self, line): parsed_line = self.parser.parse(line) line_type = parsed_line["type"] - if line_type == LineType.LOG_LINE and parsed_line["log_level"].lower() == "info": + if line_type == LineType.LOG_LINE and parsed_line["level"].lower() == "info": self.infomessage_counter += 1 def report(self, out_stream): diff --git a/src/alogamous/warning_analyzer.py b/src/alogamous/warning_analyzer.py index 8f8d1c6..275f6fc 100644 --- a/src/alogamous/warning_analyzer.py +++ b/src/alogamous/warning_analyzer.py @@ -1,14 +1,14 @@ -from alogamous import analyzer +from alogamous import analyzer, log_line_parser class WarningAnalyzer(analyzer.Analyzer): - def __init__(self): + def __init__(self, line_parser): + self.parser = line_parser self.count = 0 def read_log_line(self, line): - line_list = line.split(" ") - # List should have format [date, timestamp, -, root, -, log message type, -, first word of message...] - if line_list[5].lower() == "warning": + parsed_line = self.parser.parse(line) + if parsed_line["type"] == log_line_parser.LineType.LOG_LINE and parsed_line["level"].lower().startswith("warn"): self.count += 1 def report(self, out_stream): diff --git a/tests/flag_duplicate_log_messages_test.py b/tests/flag_duplicate_log_messages_test.py index 95f1cfa..f8fbded 100644 --- a/tests/flag_duplicate_log_messages_test.py +++ b/tests/flag_duplicate_log_messages_test.py @@ -1,27 +1,54 @@ import io -from alogamous import flag_duplicate_log_messages +from alogamous import flag_duplicate_log_messages, log_line_parser def test_flag_duplicate_log_messages(): - flagger = flag_duplicate_log_messages.FlagDuplicateLogMessages() + line_parser = log_line_parser.LogLineParser( + ["datetime", "source", "level", "message"], " - ", "====================================================" + ) + flagger = flag_duplicate_log_messages.FlagDuplicateLogMessages(line_parser) in_stream = io.StringIO("""Date - root - INFO - log message 1 Date - root - WARNING - log message 2 Date - root - WARNING - log message 1""") out_stream = io.StringIO() for line in in_stream: - flagger.read_log_line(line) + flagger.read_log_line(line.rstrip()) flagger.report(out_stream) - assert out_stream.getvalue() == """Duplicate Log Messages:\nlog message 1""" + assert out_stream.getvalue() == """Duplicate Log Messages:\n- log message 1""" def test_flag_duplicate_log_messages_no_duplicates(): - flagger = flag_duplicate_log_messages.FlagDuplicateLogMessages() + line_parser = log_line_parser.LogLineParser( + ["datetime", "source", "level", "message"], " - ", "====================================================" + ) + flagger = flag_duplicate_log_messages.FlagDuplicateLogMessages(line_parser) in_stream = io.StringIO("""Date - root - INFO - log message 1 Date - root - WARNING - log message 2 Date - root - WARNING - log message 3""") out_stream = io.StringIO() for line in in_stream: - flagger.read_log_line(line) + flagger.read_log_line(line.rstrip()) + flagger.report(out_stream) + assert out_stream.getvalue() == """No duplicate log messages""" + + +def test_flag_duplicate_messages_with_header_and_dashes(): + line_parser = log_line_parser.LogLineParser( + ["datetime", "source", "level", "message"], " - ", "====================================================" + ) + flagger = flag_duplicate_log_messages.FlagDuplicateLogMessages(line_parser) + in_stream = io.StringIO("""==================================================== +STARTING Tracking service + Start time: 2024-06-20 09:00:00.001550+00:00 + Version: 2729a + Command line: ['.venv/bin/python3', '-m', 'app.tracking_service', '--market', 'US', '--version', '2729a'] +==================================================== +2024-06-20 11:00:17,983 - root - INFO - Adding subscription for pid None +2024-06-20 11:00:18,115 - root - INFO - Initialized Influx DB Client to host +2024-06-20 11:00:18,185 - root - INFO - Kafka reading from start of day 2024-06-20 05:00:00+00:00 on topic internal""") + out_stream = io.StringIO() + for line in in_stream: + flagger.read_log_line(line.rstrip()) flagger.report(out_stream) assert out_stream.getvalue() == """No duplicate log messages""" diff --git a/tests/info_counter_test.py b/tests/info_counter_test.py index 381ed5d..76d3d80 100644 --- a/tests/info_counter_test.py +++ b/tests/info_counter_test.py @@ -5,7 +5,7 @@ def test_report(): - infomessage_counter = InfoAnalyzer(LogLineParser(["log_level", "message"], ":", "HEADER")) + infomessage_counter = InfoAnalyzer(LogLineParser(["level", "message"], ":", "HEADER")) out_stream = io.StringIO() infomessage_counter.read_log_line("HEADER") @@ -20,7 +20,7 @@ def test_report(): def test_capital_info(): - infomessage_counter = InfoAnalyzer(LogLineParser(["log_level", "message"], ":", "HEADER")) + infomessage_counter = InfoAnalyzer(LogLineParser(["level", "message"], ":", "HEADER")) out_stream = io.StringIO() infomessage_counter.read_log_line("Info: line 1") @@ -33,7 +33,7 @@ def test_capital_info(): def test_report_mixedlines(): - infomessage_counter = InfoAnalyzer(LogLineParser(["log_level", "message"], ":", "HEADER")) + infomessage_counter = InfoAnalyzer(LogLineParser(["level", "message"], ":", "HEADER")) out_stream = io.StringIO() infomessage_counter.read_log_line("info: line 1") @@ -46,7 +46,7 @@ def test_report_mixedlines(): def test_report_without_info_messages(): - infomessage_counter = InfoAnalyzer(LogLineParser(["log_level", "message"], ":", "HEADER")) + infomessage_counter = InfoAnalyzer(LogLineParser(["level", "message"], ":", "HEADER")) out_stream = io.StringIO() infomessage_counter.read_log_line("Warning: line 1") @@ -59,7 +59,7 @@ def test_report_without_info_messages(): def test_no_imput_lines(): - infomessage_counter = InfoAnalyzer(LogLineParser(["log_level", "message"], ":", "HEADER")) + infomessage_counter = InfoAnalyzer(LogLineParser(["level", "message"], ":", "HEADER")) out_stream = io.StringIO() infomessage_counter.report(out_stream) diff --git a/tests/test_error_counter.py b/tests/test_error_counter.py index a15aa0d..6aa6c1e 100644 --- a/tests/test_error_counter.py +++ b/tests/test_error_counter.py @@ -1,10 +1,13 @@ import io -from alogamous import error_counter_analyzer +from alogamous import error_counter_analyzer, log_line_parser def test_error_counter(): - counter = error_counter_analyzer.ErrorCounterAnalyzer() + parser = log_line_parser.LogLineParser( + ["datetime", "source", "level", "message"], " - ", "====================================================" + ) + counter = error_counter_analyzer.ErrorCounterAnalyzer(parser) in_stream = io.StringIO( """2024-06-20 17:16:03,660 - root - ERROR - Caught exception N/A. 2024-06-20 17:16:03,660 - root - ERROR - Caught exception N/A. Message: Unclosed connector NoneType: None""" @@ -17,7 +20,10 @@ def test_error_counter(): def test_no_errors(): - counter = error_counter_analyzer.ErrorCounterAnalyzer() + parser = log_line_parser.LogLineParser( + ["datetime", "source", "level", "message"], " - ", "====================================================" + ) + counter = error_counter_analyzer.ErrorCounterAnalyzer(parser) in_stream = io.StringIO("""2024-06-20 17:17:04,278 - root - INFO - Updating prices 2024-06-20 17:24:34,091 - root - INFO - Closing client connection.""") out_stream = io.StringIO() @@ -28,7 +34,10 @@ def test_no_errors(): def test_no_input_lines(): - counter = error_counter_analyzer.ErrorCounterAnalyzer() + parser = log_line_parser.LogLineParser( + ["datetime", "source", "level", "message"], " - ", "====================================================" + ) + counter = error_counter_analyzer.ErrorCounterAnalyzer(parser) out_stream = io.StringIO() counter.report(out_stream) assert out_stream.getvalue().strip() == "Number of error lines: 0" diff --git a/tests/warning_analyzer_test.py b/tests/warning_analyzer_test.py index 4ef1ca7..c853e22 100644 --- a/tests/warning_analyzer_test.py +++ b/tests/warning_analyzer_test.py @@ -1,10 +1,13 @@ import io -from alogamous import warning_analyzer +from alogamous import log_line_parser, warning_analyzer def test_warning_count(): - counter = warning_analyzer.WarningAnalyzer() + parser = log_line_parser.LogLineParser( + ["datetime", "source", "level", "message"], " - ", "====================================================" + ) + counter = warning_analyzer.WarningAnalyzer(parser) in_stream = io.StringIO("""2024-06-20 11:00:18,185 - root - INFO - Kafka reading from start of day 2024-06-20 05:00:00+00:00 on topic internal from kafka.servers:9092 2024-06-20 11:00:19,328 - root - INFO - Kafka source starting for topic internal at current offset 7924032 end offset 7928950 on servers kafka.servers:9092 2024-06-20 11:00:22,329 - root - INFO - Kafka topic internal is caught up at offset 7928949