From 1dd3aa4384aa1f4d53a9e7e5f53fdfd4ec91f544 Mon Sep 17 00:00:00 2001 From: boazhaim <160493207+boazhaim@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:16:57 +0200 Subject: [PATCH] issue:4151975-Log analyzer telemetry file extract (#272) * supporting telemetry logs extraction * Minor fixes for user friendliness * some pylint * more pylint * more pylint * final pylint * Fixing directory extract * Comment fix + removing bad code * Adding docstring --- .../src/loganalyze/.pylintrc | 3 +++ .../src/loganalyze/log_analyzer.py | 9 ++++++++- .../loganalyze/log_analyzers/base_analyzer.py | 16 ++++++++++++++- .../logs_extraction/base_extractor.py | 18 ++++++++++++++++- .../logs_extraction/directory_extractor.py | 15 ++++++++------ .../logs_extraction/tar_extractor.py | 20 +++++++++++++++---- 6 files changed, 68 insertions(+), 13 deletions(-) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/.pylintrc b/plugins/ufm_log_analyzer_plugin/src/loganalyze/.pylintrc index 92e8f0be..79193d66 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/.pylintrc +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/.pylintrc @@ -4,3 +4,6 @@ disable=missing-function-docstring, missing-module-docstring, too-few-public-methods, logging-fstring-interpolation, + +[DESIGN] +max-locals=20 \ No newline at end of file diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py index 1cdece7b..add4a7be 100755 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py @@ -50,13 +50,20 @@ import loganalyze.logger as log +# This list holds all the log files that we extract directly. +# If a log file has a directory name, we will search for the given +# Log file in that log directory, if the log is found, we will copy +# it but also rename it to dir_name_log_name. +# Notice that we only support one level of directory. LOGS_TO_EXTRACT = [ "event.log", "ufmhealth.log", "ufm.log", "ibdiagnet2.log", "console.log", - "rest_api.log" + "rest_api.log", + "ufm_logs/ibdiagnet2_port_counters.log", + "secondary_telemetry/ibdiagnet2_port_counters.log" ] DIRECTORIES_TO_EXTRACT = [ diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py index ecffb517..1d6db24f 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py @@ -125,7 +125,21 @@ def full_analysis(self): Run all the analysis and returns a list of all the graphs created and their title """ for func in self._funcs_for_analysis: - func() + # Since we don't know who we are calling and how they + # Behave, this is a way to protect all the functions + # In case a function is raising an exception. + try: + func() + except: # pylint: disable=bare-except + function_name = func.__name__ + try: + class_name = "" + if "." in func.__qualname__: + class_name = func.__qualname__.split('.')[0] + log.LOGGER.debug(f"Error when calling {function_name} {class_name}, skipping") + except: # pylint: disable=bare-except + pass + return self._images_created if len(self._images_created) > 0 else [] diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py index d778d054..eb8c36d3 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py @@ -9,9 +9,10 @@ # This software product is governed by the End User License Agreement # provided with the software product. # +import os from pathlib import Path from abc import abstractmethod -from typing import List +from typing import List, Set class BaseExtractor: @@ -27,6 +28,21 @@ def is_exists_get_as_path(self, location) -> Path: return location return None + @staticmethod + def _split_based_on_dir(files:Set[str]): + single_name_logs = set() + logs_with_dirs = {} + for log_name in files: + dir_name = os.path.dirname(log_name) + base_name = os.path.basename(log_name) + if dir_name: + if dir_name not in logs_with_dirs: + logs_with_dirs[dir_name] = set() + logs_with_dirs[dir_name].add(base_name) + else: + single_name_logs.add(base_name) + return single_name_logs, logs_with_dirs + @abstractmethod def extract_files(self, files_to_extract: List[str], directories_to_extract: List[str], diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py index 08604515..8adcc790 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py @@ -30,20 +30,23 @@ def extract_files(self, files_to_extract: List[str], if not os.path.exists(destination): os.makedirs(destination) - # Convert the list to a set for faster lookup files_to_extract = set(files_to_extract) directories_to_extract = set(directories_to_extract) found_files = set() not_found_files = set(files_to_extract) + _, logs_with_dirs = self._split_based_on_dir(files_to_extract) - # Traverse the source directory and its subdirectories for root, _, files in os.walk(self.dir_path): for file_name in files: - full_dir_name = os.path.dirname(file_name) - last_dir_name = os.path.basename(full_dir_name) - if file_name in files_to_extract or last_dir_name in directories_to_extract: + last_dir_name = os.path.basename(root) + is_logs_with_dir_flag = last_dir_name in logs_with_dirs and \ + file_name in logs_with_dirs[last_dir_name] + if file_name in files_to_extract or last_dir_name in directories_to_extract or\ + is_logs_with_dir_flag: src_file_path = os.path.join(root, file_name) - dest_file_path = os.path.join(destination, file_name) + new_file_name = f"{last_dir_name}_{file_name}" if is_logs_with_dir_flag \ + else file_name + dest_file_path = os.path.join(destination, new_file_name) shutil.copy2(src_file_path, dest_file_path) found_files.add(dest_file_path) not_found_files.discard(file_name) diff --git a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py index a0730a8b..c001e11e 100644 --- a/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py +++ b/plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py @@ -25,7 +25,6 @@ LOGS_GZ_POSTFIX = ".gz" GZIP_MAGIC_NUMBER = b"\x1f\x8b" # Magic number to understand if a file is really a gzip - class DumpFilesExtractor(BaseExtractor): def __init__(self, dump_path: Path) -> None: dump_path = self.is_exists_get_as_path(dump_path) @@ -44,12 +43,20 @@ def _get_files_from_tar( files_went_over = set() failed_extract = set() folders_to_remove = set() + single_log_name, logs_with_dirs = self._split_based_on_dir(files_to_extract) for member in opened_file: base_name = os.path.basename(member.name) - dir_name = os.path.dirname(member.name) - if base_name in files_to_extract or \ - os.path.basename(dir_name) in directories_to_extract: + full_dir_path = os.path.dirname(member.name) + parent_dir_name = os.path.basename(full_dir_path) + original_base_name = base_name + is_logs_with_dir_flag = parent_dir_name in logs_with_dirs and \ + base_name in logs_with_dirs[parent_dir_name] + if base_name in single_log_name or \ + parent_dir_name in directories_to_extract or \ + is_logs_with_dir_flag: try: + if is_logs_with_dir_flag: + base_name = f"{parent_dir_name}_{base_name}" opened_file.extract(member, path=destination) extracted_file_path = os.path.join(destination, str(member.path)) log.LOGGER.debug(f"Extracted {base_name}") @@ -63,6 +70,11 @@ def _get_files_from_tar( files_went_over.add(base_name) if base_name in files_to_extract: files_to_extract.remove(base_name) + elif is_logs_with_dir_flag: + logs_with_dirs[parent_dir_name].discard(original_base_name) + if len(logs_with_dirs[parent_dir_name]) == 0: + del logs_with_dirs[parent_dir_name] + files_extracted = files_went_over.difference(failed_extract) # When extracting the files from the tar, they are also taken with their # directories from inside the tar, there is no way to only take the file