issue:4151975-Log analyzer telemetry file extract (#272)

* supporting telemetry logs extraction

* Minor fixes for user friendliness

* some pylint

* more pylint

* more pylint

* final pylint

* Fixing directory extract

* Comment fix + removing bad code

* Adding docstring

plugins/ufm_log_analyzer_plugin/src/loganalyze/.pylintrc

@@ -4,3 +4,6 @@ disable=missing-function-docstring,
         missing-module-docstring,
         too-few-public-methods,
         logging-fstring-interpolation,
+
+[DESIGN]
+max-locals=20

plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzer.py

@@ -50,13 +50,20 @@
 
 import loganalyze.logger as log
 
+# This list holds all the log files that we extract directly.
+# If a log file has a directory name, we will search for the given
+# Log file in that log directory, if the log is found, we will copy
+# it but also rename it to dir_name_log_name.
+# Notice that we only support one level of directory.
 LOGS_TO_EXTRACT = [
     "event.log",
     "ufmhealth.log",
     "ufm.log",
     "ibdiagnet2.log",
     "console.log",
-    "rest_api.log"
+    "rest_api.log",
+    "ufm_logs/ibdiagnet2_port_counters.log",
+    "secondary_telemetry/ibdiagnet2_port_counters.log"
 ]
 
 DIRECTORIES_TO_EXTRACT = [

plugins/ufm_log_analyzer_plugin/src/loganalyze/log_analyzers/base_analyzer.py

@@ -125,7 +125,21 @@ def full_analysis(self):
         Run all the analysis and returns a list of all the graphs created and their title
         """
         for func in self._funcs_for_analysis:
-            func()
+            # Since we don't know who we are calling and how they
+            # Behave, this is a way to protect all the functions
+            # In case a function is raising an exception.
+            try:
+                func()
+            except: # pylint: disable=bare-except
+                function_name = func.__name__
+                try:
+                    class_name = ""
+                    if "." in func.__qualname__:
+                        class_name = func.__qualname__.split('.')[0]
+                    log.LOGGER.debug(f"Error when calling {function_name} {class_name}, skipping")
+                except: # pylint: disable=bare-except
+                    pass
+
         return self._images_created if len(self._images_created) > 0 else []
 
 

plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/base_extractor.py

@@ -9,9 +9,10 @@
 # This software product is governed by the End User License Agreement
 # provided with the software product.
 #
+import os
 from pathlib import Path
 from abc import abstractmethod
-from typing import List
+from typing import List, Set
 
 
 class BaseExtractor:
@@ -27,6 +28,21 @@ def is_exists_get_as_path(self, location) -> Path:
             return location
         return None
 
+    @staticmethod
+    def _split_based_on_dir(files:Set[str]):
+        single_name_logs = set()
+        logs_with_dirs = {}
+        for log_name in files:
+            dir_name = os.path.dirname(log_name)
+            base_name = os.path.basename(log_name)
+            if dir_name:
+                if dir_name not in logs_with_dirs:
+                    logs_with_dirs[dir_name] = set()
+                logs_with_dirs[dir_name].add(base_name)
+            else:
+                single_name_logs.add(base_name)
+        return single_name_logs, logs_with_dirs
+
     @abstractmethod
     def extract_files(self, files_to_extract: List[str],
                       directories_to_extract: List[str],

plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/directory_extractor.py

@@ -30,20 +30,23 @@ def extract_files(self, files_to_extract: List[str],
         if not os.path.exists(destination):
             os.makedirs(destination)
 
-        # Convert the list to a set for faster lookup
         files_to_extract = set(files_to_extract)
         directories_to_extract = set(directories_to_extract)
         found_files = set()
         not_found_files = set(files_to_extract)
+        _, logs_with_dirs = self._split_based_on_dir(files_to_extract)
 
-        # Traverse the source directory and its subdirectories
         for root, _, files in os.walk(self.dir_path):
             for file_name in files:
-                full_dir_name = os.path.dirname(file_name)
-                last_dir_name = os.path.basename(full_dir_name)
-                if file_name in files_to_extract or last_dir_name in directories_to_extract:
+                last_dir_name = os.path.basename(root)
+                is_logs_with_dir_flag = last_dir_name in logs_with_dirs and \
+                    file_name in logs_with_dirs[last_dir_name]
+                if file_name in files_to_extract or last_dir_name in directories_to_extract or\
+                    is_logs_with_dir_flag:
                     src_file_path = os.path.join(root, file_name)
-                    dest_file_path = os.path.join(destination, file_name)
+                    new_file_name = f"{last_dir_name}_{file_name}" if is_logs_with_dir_flag \
+                        else file_name
+                    dest_file_path = os.path.join(destination, new_file_name)
                     shutil.copy2(src_file_path, dest_file_path)
                     found_files.add(dest_file_path)
                     not_found_files.discard(file_name)

plugins/ufm_log_analyzer_plugin/src/loganalyze/logs_extraction/tar_extractor.py

@@ -25,7 +25,6 @@
 LOGS_GZ_POSTFIX = ".gz"
 GZIP_MAGIC_NUMBER = b"\x1f\x8b"  # Magic number to understand if a file is really a gzip
 
-
 class DumpFilesExtractor(BaseExtractor):
     def __init__(self, dump_path: Path) -> None:
         dump_path = self.is_exists_get_as_path(dump_path)
@@ -44,12 +43,20 @@ def _get_files_from_tar(
         files_went_over = set()
         failed_extract = set()
         folders_to_remove = set()
+        single_log_name, logs_with_dirs = self._split_based_on_dir(files_to_extract)
         for member in opened_file:
             base_name = os.path.basename(member.name)
-            dir_name = os.path.dirname(member.name)
-            if base_name in files_to_extract or \
-                os.path.basename(dir_name) in directories_to_extract:
+            full_dir_path = os.path.dirname(member.name)
+            parent_dir_name = os.path.basename(full_dir_path)
+            original_base_name = base_name
+            is_logs_with_dir_flag = parent_dir_name in logs_with_dirs and \
+                base_name in logs_with_dirs[parent_dir_name]
+            if base_name in single_log_name or \
+                parent_dir_name in directories_to_extract or \
+                is_logs_with_dir_flag:
                 try:
+                    if is_logs_with_dir_flag:
+                        base_name = f"{parent_dir_name}_{base_name}"
                     opened_file.extract(member, path=destination)
                     extracted_file_path = os.path.join(destination, str(member.path))
                     log.LOGGER.debug(f"Extracted {base_name}")
@@ -63,6 +70,11 @@ def _get_files_from_tar(
                     files_went_over.add(base_name)
                     if base_name in files_to_extract:
                         files_to_extract.remove(base_name)
+                    elif is_logs_with_dir_flag:
+                        logs_with_dirs[parent_dir_name].discard(original_base_name)
+                        if len(logs_with_dirs[parent_dir_name]) == 0:
+                            del logs_with_dirs[parent_dir_name]
+
         files_extracted = files_went_over.difference(failed_extract)
         # When extracting the files from the tar, they are also taken with their
         # directories from inside the tar, there is no way to only take the file