From 4ea02110308813ae68ce520ab8eaf07fc9c45d06 Mon Sep 17 00:00:00 2001 From: kitrady Date: Fri, 2 Aug 2024 20:29:46 -0500 Subject: [PATCH] improving info analyzer --- data/test_output_file.txt | 234 +++++++++++++++++------------- src/alogamous/loginfo_analyzer.py | 2 +- tests/info_counter_test.py | 10 +- 3 files changed, 143 insertions(+), 103 deletions(-) diff --git a/data/test_output_file.txt b/data/test_output_file.txt index 8bd4744..6585e58 100644 --- a/data/test_output_file.txt +++ b/data/test_output_file.txt @@ -1,124 +1,146 @@ -Number of error lines: 22 +Number of error lines: 0 Duplicate Log Messages: -STARTING Tracking service -20 05:00:00+00:00 on topic internal from kafka.servers:9092 -Could not find instrument for ric UTU.T, using provided identifier 3 -instrument not found for sedol OIN38378 -instrument not found for ric RUE.ST -prometheus client http server running -Closing client connection. -Could not find instrument for ric YT.ATS, using provided identifier 1 -instrument not found for sedol DHI7337 -Could not find instrument for sedol UY29387, trying ric RUE.ST -instrument not found for sedol BYP321337 -Updating subscribed topics to: frozenset({'internal'}) -20 09:00:00.001550+00:00 -market hours frequency -Caught exception N/A. Message: Unclosed client session NoneType: None -Version: 2729a -Could not find instrument for sedol BYP321337, trying ric YT.ATS -instrument not found for ric UYU.T -Caught exception N/A. Message: Unclosed connector NoneType: None -Updating prices -Could not find instrument for sedol RZZZZ2, trying ric UYU.T +Could not find symbol for sedol 3393DS8. Only publishing externally. +Could not find symbol for sedol 27ER995. Only publishing externally. +Version: 2757 +Reading topic java_topic from kafka servers brokers:9092 +Writing to topic bosun_internal to kafka servers brokers:9092 +Reading from kafka servers brokers:9092 +Could not find symbol for sedol 344232AS7. Only publishing externally. +Starting with config Config{hostName='host'services='[JAVA_PRICE_PROCESS]'} ==================================================== -Could not find instrument for sedol DHI7337, trying ric YT.ATS -instrument not found for ric JPQ.CC -20 -Scheduling Error Handler in 150.0 seconds -instrument not found for sedol UY29387 -Exception in message handler > TrackingService.on_order_change() missing 1 required positional argument: 'order_identifier' -instrument not found for ric YT.ATS -version', '2729a'] -Kafka source starting for topic internal at current offset 7924032 end offset 7928950 on servers kafka.servers:9092 -Could not find instrument for ric RUE.ST, using provided identifier 5 -Initialized prometheus server. -Adding subscription for pid None -Kafka topic internal is caught up at offset 7928949 -Could not find instrument for sedol OIN38378, trying ric 837.RIJ -instrument not found for sedol RZZZZ2 -Could not find instrument for ric 837.RIJ, using provided identifier 4 -Tracking service is caught up -tracking_service Closed -instrument not found for ric 837.RIJ -Could not find instrument for ric JPQ.CC, using provided identifier 2 -Initialized Influx DB Client to host +market', 'US'] +Starting KafkaReader at offset 1005022 +Starting KafkaReader at offset 28181084 +KafkaReader found no data on topic java_topic. This will only log once. +Writing to topic sip_last_trade_px to kafka servers brokers:9092 +STARTING PROCESS Java Price Process -Number of log lines: 151 +Number of log lines: 157 -Lines that do not conform to log format: -- Hello I am a bad log line +All lines conform to log line format -Lines that are part of the startup header: -- STARTING Tracking service -- Start time: 2024-06-20 09:00:00.001550+00:00 -- Version: 2729a -- Command line: ['.venv/bin/python3', '-m', 'app.tracking_service', '--market', 'US', '--version', '2729a'] -- STARTING Tracking service -- Start time: 2024-06-20 09:00:00.001550+00:00 -- Version: 2729a -- Command line: ['.venv/bin/python3', '-m', 'app.tracking_service', '--market', 'US', '--version', '2729a'] -- STARTING Tracking service -- Start time: 2024-06-21 09:00:00.001550+00:00 -- Version: 2729a -- Command line: ['.venv/bin/python3', '-m', 'app.tracking_service', '--market', 'US', '--version', '2729a'] +Number of info messages: 12 + +No non-fatal stack traces were found + +Lines that are part of the startup header(s): +- STARTING PROCESS Java Price Process +- Start time: 2024-06-28 12:00:00.060562+00:00 +- Version: 2757 +- Command line: ['./build/app/java_process', '--market', 'US'] +- STARTING PROCESS Java Price Process +- Start time: 2024-06-27 12:00:00.060562+00:00 +- Version: 2757 +- Command line: ['./build/app/java_process', '--market', 'US'] + +No Warnings were detected. ------ a report has been reported ------ -Number of error lines: 22 +Number of error lines: 25 Duplicate Log Messages: -instrument not found for ric UYU.T -Could not find instrument for sedol OIN38378, trying ric 837.RIJ -prometheus client http server running -instrument not found for sedol RZZZZ2 -Caught exception N/A. Message: Unclosed connector NoneType: None -20 05:00:00+00:00 on topic internal from kafka.servers:9092 -tracking_service Closed -instrument not found for ric RUE.ST -version', '2729a'] -Version: 2729a -Could not find instrument for ric YT.ATS, using provided identifier 1 -Adding subscription for pid None -Kafka source starting for topic internal at current offset 7924032 end offset 7928950 on servers kafka.servers:9092 -20 09:00:00.001550+00:00 -Could not find instrument for sedol UY29387, trying ric RUE.ST -Could not find instrument for ric RUE.ST, using provided identifier 5 -==================================================== +instrument not found for sedol DHI7337 +Could not find instrument for sedol RZZZZ2, trying ric UYU.T +Could not find instrument for sedol BYP321337, trying ric YT.ATS market hours frequency -Closing client connection. -STARTING Tracking service -instrument not found for ric 837.RIJ +Could not find instrument for sedol DHI7337, trying ric YT.ATS +Scheduling Error Handler in 150.0 seconds Updating subscribed topics to: frozenset({'internal'}) +==================================================== +Could not find instrument for ric JPQ.CC, using provided identifier 2 +Initialized Influx DB Client to host +instrument not found for ric JPQ.CC +instrument not found for sedol UY29387 +Caught exception N/A. Message: Unclosed client session NoneType: None +29 exchange=ASDN +20 +Kafka source starting for topic internal at current offset 7924032 end offset 7928950 on servers kafka.servers:9092 +29. +instrument not found for ric 837.RIJ +29 exchange=YS +Version: 2729a +instrument not found for ric YT.ATS +Caught up and static data complete; starting timer. +Could not find instrument for ric RUE.ST, using provided identifier 5 +20 09:00:00.001550+00:00 instrument not found for sedol OIN38378 +29 exchange=NMQ +29 exchange=ASDI +Updating subscribed topics to: frozenset({'internal_topic'}) +Could not find instrument for sedol OIN38378, trying ric 837.RIJ +Updating prices +tracking_service Closed +Tracking service is caught up +29 exchange=XSS +29 +prometheus client http server running on http://prometheus.co +Could not find instrument for ric YT.ATS, using provided identifier 1 +instrument not found for sedol RZZZZ2 Initialized prometheus server. +29 exchange=BSTT +20 05:00:00+00:00 on topic internal from kafka.servers:9092 Could not find instrument for ric 837.RIJ, using provided identifier 4 -Tracking service is caught up +Could not find instrument for sedol UY29387, trying ric RUE.ST +sql.co/qty +prometheus client http server running +29 exchange=XSET +Caught exception N/A. Message: Unclosed connector NoneType: None +market', 'US'] Exception in message handler > TrackingService.on_order_change() missing 1 required positional argument: 'order_identifier' -20 -Scheduling Error Handler in 150.0 seconds -Updating prices -instrument not found for ric JPQ.CC -Initialized Influx DB Client to host -instrument not found for sedol BYP321337 -Could not find instrument for sedol DHI7337, trying ric YT.ATS -Could not find instrument for ric JPQ.CC, using provided identifier 2 -instrument not found for sedol DHI7337 -Could not find instrument for sedol RZZZZ2, trying ric UYU.T -instrument not found for ric YT.ATS -Could not find instrument for sedol BYP321337, trying ric YT.ATS Could not find instrument for ric UTU.T, using provided identifier 3 -Caught exception N/A. Message: Unclosed client session NoneType: None +Updating subscribed topics to: frozenset({'topic_internal'}) +Adding subscription for pid None +29 exchange=SXDF +29 and symbol not in instrument_list +instrument not found for ric RUE.ST Kafka topic internal is caught up at offset 7928949 -instrument not found for sedol UY29387 +instrument not found for ric UYU.T +instrument not found for sedol BYP321337 +version', '2729a'] +29 exchange=UVD +STARTING Tracking service +Closing client connection. -Number of log lines: 151 +Number of log lines: 608 Lines that do not conform to log format: - Hello I am a bad log line +- NoneType: None + +Number of info messages: 444 -Lines that are part of the startup header: +1 non-fatal stack trace(s) found: +- Traceback (most recent call last): +- File "", line 1938, in _run_module_as_main +- File "", line 88, in _run_code +- File ".build/2811/execution/execution_service", line 973, in +- app.run(life_cycle_runner.run, life_cycle_runner.stop) +- File ".build/2811/app/application.py", line 219, in run +- run(self.start(my_date, main, stop)) +- File ".build/2811/.venv/lib/python3.11/runners.py", line 190, in run +- return runner.run(main) +- ^^^^^^^^^^^^^^^^ +- File ".build/2811/.venv/lib/python3.11/runners.py", line 118, in run +- return self._loop.run_until_complete(task) +- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- File ".build/2811/.venv/lib/python3.11/events.py", line 653, in run_until_done +- return future.result() +- ^^^^^^^^^^^^^^^ +- File "2811/app/application.py", line 421, in start +- await self.task +- File "2811/messages/app/runner.py", line 449, in run +- await asyncio.gather(*self.running_tasks) +- File "2811/messages/processor.py", line 340, in run +- await self.dispatch(message) +- File ".build/2811/execution/execution_service", line 1315, in market_test +- if symbol and obj.region != 'NORTHAMERICA' +- ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +- AttributeError: 'NoneType' object has no attribute 'region' + +Lines that are part of the startup header(s): - STARTING Tracking service - Start time: 2024-06-20 09:00:00.001550+00:00 - Version: 2729a @@ -131,5 +153,23 @@ Lines that are part of the startup header: - Start time: 2024-06-21 09:00:00.001550+00:00 - Version: 2729a - Command line: ['.venv/bin/python3', '-m', 'app.tracking_service', '--market', 'US', '--version', '2729a'] +- STARTING PROCESS Quantity Loader +- Start time: 2024-05-30 10:30:00.034654+00:00 +- Version: 2673 +- Command line: ['.venv/bin/python3', '-m', 'loaders.quantity_loader', '--market', 'US'] +- STARTING PROCESS Batch Writer +- Start time: 2024-06-10 10:30:00.153687+00:00 +- Version: 2692 +- Command line: ['.venv/bin/python3', '-m', 'db.batch_writer', '--market', 'US'] +- STARTING PROCESS Access Point +- Start time: 2025-10-01 12:30:00.022996+00:00 +- Version: 2808 +- Command line: ['.venv/bin/python3', '-m', 'process.access_point', '--market', 'US', '--ap', 'ARC'] +- STARTING PROCESS Execution Server +- Start time: 2024-07-23 21:00:00.011070+00:00 +- Version: 2811 +- Command line: ['.venv/bin/python3', '-m', 'execution/execution_service', '--version', '2811'] + +69 Warnings were detected. ------ a report has been reported ------ diff --git a/src/alogamous/loginfo_analyzer.py b/src/alogamous/loginfo_analyzer.py index 34e8cf0..ec0c868 100644 --- a/src/alogamous/loginfo_analyzer.py +++ b/src/alogamous/loginfo_analyzer.py @@ -10,7 +10,7 @@ def __init__(self, parser): def read_log_line(self, line): parsed_line = self.parser.parse(line) line_type = parsed_line["type"] - if line_type == LineType.LOG_LINE and parsed_line["log_level"].lower() == "info": + if line_type == LineType.LOG_LINE and parsed_line["level"].lower() == "info": self.infomessage_counter += 1 def report(self, out_stream): diff --git a/tests/info_counter_test.py b/tests/info_counter_test.py index 381ed5d..76d3d80 100644 --- a/tests/info_counter_test.py +++ b/tests/info_counter_test.py @@ -5,7 +5,7 @@ def test_report(): - infomessage_counter = InfoAnalyzer(LogLineParser(["log_level", "message"], ":", "HEADER")) + infomessage_counter = InfoAnalyzer(LogLineParser(["level", "message"], ":", "HEADER")) out_stream = io.StringIO() infomessage_counter.read_log_line("HEADER") @@ -20,7 +20,7 @@ def test_report(): def test_capital_info(): - infomessage_counter = InfoAnalyzer(LogLineParser(["log_level", "message"], ":", "HEADER")) + infomessage_counter = InfoAnalyzer(LogLineParser(["level", "message"], ":", "HEADER")) out_stream = io.StringIO() infomessage_counter.read_log_line("Info: line 1") @@ -33,7 +33,7 @@ def test_capital_info(): def test_report_mixedlines(): - infomessage_counter = InfoAnalyzer(LogLineParser(["log_level", "message"], ":", "HEADER")) + infomessage_counter = InfoAnalyzer(LogLineParser(["level", "message"], ":", "HEADER")) out_stream = io.StringIO() infomessage_counter.read_log_line("info: line 1") @@ -46,7 +46,7 @@ def test_report_mixedlines(): def test_report_without_info_messages(): - infomessage_counter = InfoAnalyzer(LogLineParser(["log_level", "message"], ":", "HEADER")) + infomessage_counter = InfoAnalyzer(LogLineParser(["level", "message"], ":", "HEADER")) out_stream = io.StringIO() infomessage_counter.read_log_line("Warning: line 1") @@ -59,7 +59,7 @@ def test_report_without_info_messages(): def test_no_imput_lines(): - infomessage_counter = InfoAnalyzer(LogLineParser(["log_level", "message"], ":", "HEADER")) + infomessage_counter = InfoAnalyzer(LogLineParser(["level", "message"], ":", "HEADER")) out_stream = io.StringIO() infomessage_counter.report(out_stream)