Add python runner module S28

.github/workflows/check_project.yml

@@ -21,9 +21,10 @@ jobs:
       uses: actions/checkout@v2
     - name: Install dependencies for check script
       run: |
-        sudo apt-get install -y shellcheck python3-pip
+        sudo apt-get install -y shellcheck python3-pip flake8
         PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -U requests
         PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install semgrep
+        PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install prospector
         mkdir ./external
         git clone https://github.com/returntocorp/semgrep-rules.git external/semgrep-rules
     - name: Run check_project.sh

check_project.sh

@@ -13,7 +13,7 @@
 # SPDX-License-Identifier: GPL-3.0-only
 #
 # Author(s): Michael Messner, Pascal Eckmann
-# Contributor(s): Benedikt Kuehne
+# Contributor(s): Benedikt Kuehne, Thomas Gingele
 
 # Description:  Check all shell scripts inside ./helpers, ./modules, emba and itself with shellchecker
 
@@ -22,6 +22,7 @@ STRICT_MODE=1
 INSTALLER_DIR="./installer"
 HELP_DIR="./helpers"
 MOD_DIR="./modules"
+PYTHON_MOD_DIR="${MOD_DIR}/S28_python_run"
 MOD_DIR_LOCAL="./modules_local"
 CONF_DIR="./config"
 EXT_DIR="./external"
@@ -49,6 +50,7 @@ MODULES_TO_CHECK_ARR=()
 MODULES_TO_CHECK_ARR_TAB=()
 MODULES_TO_CHECK_ARR_SEMGREP=()
 MODULES_TO_CHECK_ARR_DOCKER=()
+MODULES_TO_CHECK_ARR_PYTHON=()
 MODULES_TO_CHECK_ARR_PERM=()
 MODULES_TO_CHECK_ARR_COMMENT=()
 MODULES_TO_CHECK_ARR_GREP=()
@@ -123,7 +125,6 @@ import_emba_main() {
   done
 }
 
-
 dockerchecker() {
   echo -e "\\n""${ORANGE}""${BOLD}""EMBA docker-files check""${NC}"
   echo -e "${BOLD}""=================================================================""${NC}"
@@ -140,6 +141,28 @@ dockerchecker() {
   done
 }
 
+pythoncheck() {
+  echo -e "\\n""${ORANGE}""${BOLD}""EMBA Python modules check""${NC}"
+  echo -e "${BOLD}""=================================================================""${NC}"
+  PYTHON_MODULES=()
+  PROSPECTOR_BIN=$(which prospector || echo -n "${EXT_DIR}""/emba_venv/bin/prospector")
+  mapfile -t PYTHON_MODULES < <(find "${PYTHON_MOD_DIR}" -iname "*.py" 2>/dev/null)
+  for PYTHON_MODULE in "${PYTHON_MODULES[@]}"; do
+    if (file "${PYTHON_MODULE}" | grep -q "Python script"); then
+      echo -e "\\n""${GREEN}""Run Python check on ${PYTHON_MODULE}:""${NC}""\\n"
+      PYTHON_ISSUE_FOUND=0
+      flake8 "${PYTHON_MODULE}" || ((PYTHON_ISSUE_FOUND=PYTHON_ISSUE_FOUND+$?))
+      "${PROSPECTOR_BIN}" --no-autodetect "${PYTHON_MODULE}" || ((PYTHON_ISSUE_FOUND=PYTHON_ISSUE_FOUND+$?))
+      if [[ "${PYTHON_ISSUE_FOUND}" -eq 0 ]]; then
+        echo -e "${GREEN}""${BOLD}""==> SUCCESS""${NC}""\\n"
+      else
+        echo -e "\\n""${ORANGE}${BOLD}==> FIX ERRORS""${NC}""\\n"
+        MODULES_TO_CHECK_ARR_PYTHON+=( "${PYTHON_MODULE}" )
+      fi
+    fi
+  done
+}
+
 check() {
   echo -e "\\n""${ORANGE}""${BOLD}""Embedded Linux Analyzer Shellcheck""${NC}"
   echo -e "${BOLD}""=================================================================""${NC}"
@@ -305,6 +328,14 @@ summary() {
     done
     echo -e "${ORANGE}""WARNING: Fix the errors before pushing to the EMBA repository!"
   fi
+  if [[ "${#MODULES_TO_CHECK_ARR_PYTHON[@]}" -gt 0 ]]; then
+    echo -e "\\n\\n""${GREEN}${BOLD}""SUMMARY:${NC}\\n"
+    echo -e "Modules to check (Python): ${#MODULES_TO_CHECK_ARR_PYTHON[@]}\\n"
+    for PYTHON_MODULE in "${MODULES_TO_CHECK_ARR_PYTHON[@]}"; do
+      echo -e "${ORANGE}${BOLD}==> FIX MODULE: ""${PYTHON_MODULE}""${NC}"
+    done
+    echo -e "${ORANGE}""WARNING: Fix the errors before pushing to the EMBA repository!"
+  fi
   if [[ "${CNT_VAR_CHECKER_ISSUES}" -gt 0 ]]; then
     echo -e "\\n\\n""${GREEN}${BOLD}""SUMMARY:${NC}\\n"
     echo -e "Found ${ORANGE}${CNT_VAR_CHECKER_ISSUES}${NC} variable scope issues in EMBA scripts${NC}\\n"
@@ -441,6 +472,7 @@ var_checker modules
 var_checker helpers
 function_entry_space_check
 dockerchecker
+pythoncheck
 copy_right_check "Siemens Energy AG" 2025 ./ ./external
 list_linter_exceptions shellcheck ./ ./external
 list_linter_exceptions semgrep ./ ./external
@@ -451,6 +483,6 @@ if [[ "${#MODULES_TO_CHECK_ARR_TAB[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR[
   [[ "${#MODULES_TO_CHECK_ARR_DOCKER[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR_PERM[@]}" -gt 0 ]] || \
   [[ "${#MODULES_TO_CHECK_ARR_COMMENT[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR_GREP[@]}" -gt 0 ]] || \
   [[ "${#MODULES_TO_CHECK_ARR_COPYRIGHT[@]}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR_FCT_SPACE[@]}" -gt 0 ]] || \
-  [[ "${CNT_VAR_CHECKER_ISSUES}" -gt 0 ]]; then
+  [[ "${CNT_VAR_CHECKER_ISSUES}" -gt 0 ]] || [[ "${#MODULES_TO_CHECK_ARR_PYTHON[@]}" -gt 0 ]]; then
   exit 1
 fi

installer/I01_default_apps_host.sh

@@ -23,6 +23,7 @@ I01_default_apps_host() {
   echo -e "\\n""${ORANGE}""${BOLD}""These applications will be installed/updated:""${NC}"
   print_tool_info "jq" 1
   print_tool_info "shellcheck" 1
+  print_tool_info "flake8" 1
   print_tool_info "unzip" 1
   print_tool_info "bc" 1
   print_tool_info "coreutils" 1
@@ -39,6 +40,7 @@ I01_default_apps_host() {
   # python3.10-request
   print_tool_info "python3-pip" 1
   print_pip_info "requests"
+  print_pip_info "prospector"
 
   if [[ "${LIST_DEP}" -eq 1 ]] ; then
     ANSWER=("n")
@@ -51,6 +53,7 @@ I01_default_apps_host() {
       echo
       apt-get install "${INSTALL_APP_LIST[@]}" -y
       pip_install "requests" "-U"
+      pip_install "prospector" "-U"
 
       if ! command -v "${DOCKER_COMPOSE[@]}" > /dev/null; then
         echo "Installing ${DOCKER_COMPOSE[*]} manually:"

modules/S28_python_run.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# EMBA - EMBEDDED LINUX ANALYZER
+#
+# Copyright 2024-2024 Thomas Gingele <[email protected]>
+#
+# EMBA comes with ABSOLUTELY NO WARRANTY. This is free software, and you are
+# welcome to redistribute it under the terms of the GNU General Public License.
+# See LICENSE file for usage of this software.
+#
+# EMBA is licensed under GPLv3
+#
+# Author(s): Thomas Gingele
+#
+# Description:  This is an experimental EMBA module. It is designed to run user-defined python
+#               scripts during the analysis.
+#
+
+S28_python_run() {
+    module_log_init "${FUNCNAME[0]}"
+    module_title "Python Runner"
+    pre_module_reporter "${FUNCNAME[0]}"
+
+    local lSCRIPT_DIR="${MOD_DIR}/${FUNCNAME[0]}"
+    local lPYTHON_SCRIPT_COUNT=${#PYTHON_SCRIPTS[@]}
+    local lCOUNT_SUBMODULE_FINDINGS=0
+    local lCOUNT_TOTAL_FINDINGS=0
+    local lSCRIPT=""
+
+    if [[ ${lPYTHON_SCRIPT_COUNT} -gt 0 ]]; then
+        print_output "[*] ${lPYTHON_SCRIPT_COUNT} Python script/s queued for execution."
+
+        for lSCRIPT in "${PYTHON_SCRIPTS[@]}"; do
+            sub_module_title "Execution of Python runner for ${ORANGE}${lSCRIPT}${NC}"
+            print_output "[*] Executing: ${ORANGE}${lSCRIPT_DIR}/${lSCRIPT}.py${NC}"
+
+            lCOUNT_SUBMODULE_FINDINGS=$(python3 "${lSCRIPT_DIR}/${lSCRIPT}.py" | grep "FINDINGS" | sed "s/FINDINGS://")
+            lCOUNT_TOTAL_FINDINGS=$((lCOUNT_TOTAL_FINDINGS + lCOUNT_SUBMODULE_FINDINGS))
+
+            cat "${LOG_PATH_MODULE}/${lSCRIPT}.txt" >> "${LOG_FILE}"
+            print_output "[*] Python module ${ORANGE}${lSCRIPT}${NC} reported a total of ${ORANGE}${lCOUNT_SUBMODULE_FINDINGS}${NC} findings."
+        done
+
+    else
+        print_output "[*] No Python scripts queued for execution."
+    fi
+
+    sub_module_title "Final results for ${FUNCNAME[0]}"
+    print_output "Total results count: ${lCOUNT_TOTAL_FINDINGS}"
+    module_end_log "${FUNCNAME[0]}" "${lCOUNT_TOTAL_FINDINGS}"
+}

modules/S28_python_run/embaformatting.py

@@ -0,0 +1,57 @@
+#!/usr/bin/python3
+# pylint: disable=too-few-public-methods
+#   to-few-public-methods: Format is treated similarly to an enum and hence
+#                          has no public methods.
+"""
+EMBA - EMBEDDED LINUX ANALYZER
+
+Copyright 2025-2025 Thomas Gingele <[email protected]>
+
+EMBA comes with ABSOLUTELY NO WARRANTY. This is free software, and you are
+welcome to redistribute it under the terms of the GNU General Public License.
+See LICENSE file for usage of this software.
+
+EMBA is licensed under GPLv3
+SPDX-License-Identifier: GPL-3.0-only
+
+Author(s): Thomas Gingele
+
+Description: This file converts EMBAs formatting variables into values
+             which can be used by Python.
+"""
+import re
+
+from os import environ
+
+
+class FormatMeta(type):
+    """
+    Metaclass for a Singlton of type Format.
+    """
+
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            instance = super().__call__(*args, **kwargs)
+            cls._instances[cls] = instance
+
+        return cls._instances[cls]
+
+
+class Format(metaclass=FormatMeta):
+    """
+    Class holding all formatting/color codes from the EMBA environment
+    variables as attributes.
+    """
+
+    def __init__(self):
+        """
+        This function grabs all available formatting codes
+        from the environment variables via a regex
+        and adds them to the Format instance as attributes.
+        """
+        pattern = r"^\\033\[[;0-9]+m$"
+        for key in environ:
+            if re.search(pattern, environ[key]):
+                setattr(self, key, environ[key].replace("\\033", "\x1b"))

modules/S28_python_run/embamodule.py

@@ -0,0 +1,158 @@
+#!/usr/bin/python3
+# pylint: disable=consider-using-with, no-member
+#   consider-using-with: The log file is opened without 'with' on purpose
+#                        to enable writing to it across methods.
+#   no-member: Attributes of the Format class are dynamically generated during
+#              runtime.
+"""
+EMBA - EMBEDDED LINUX ANALYZER
+
+Copyright 2024-2024 Thomas Gingele <[email protected]>
+
+EMBA comes with ABSOLUTELY NO WARRANTY. This is free software, and you are
+welcome to redistribute it under the terms of the GNU General Public License.
+See LICENSE file for usage of this software.
+
+EMBA is licensed under GPLv3
+SPDX-License-Identifier: GPL-3.0-only
+
+Author(s): Thomas Gingele
+
+Description: This file contains wrapper code for custom Python modules.
+"""
+from os import _Environ
+from embaformatting import Format
+
+
+class EmbaModule():
+    """
+    Module handling class for EMBA python scripts.
+
+    Functions:
+        __init__:
+            Create a new instance of the class and set up logging.
+
+        __del__:
+            Close module files and destroy the class instance.
+
+        __write_formatted_log:
+            Base method for logging. Should not be
+            called by Python modules directly.
+
+        log:
+            Log a new message into the module log files.
+
+        add_finding:
+            Add a new finding to the module. This will later be
+            used during report generation.
+
+        panic:
+            Ensures propper logging when throwing exceptions.
+    """
+
+    def __init__(self, argv: list, env: _Environ):
+        self.format = Format()
+        self.findings = []
+        self.filename = argv[0].split("/")[-1].split('.')[0]
+
+        try:
+            self.logfile_dir = env.get('LOG_PATH_MODULE')
+            self.logfile = open(
+                f"{self.logfile_dir}/{self.filename}.txt",
+                "w",
+                encoding="utf-8"
+            )
+
+        except KeyError as key_error:
+            err = f"Unable to determine log path for module '{self.filename}'."
+            self.panic(err)
+            raise key_error
+
+        except PermissionError as perm_error:
+            err = f"Access to '{self.filename}.py' denied."
+            self.panic(err)
+            raise perm_error
+
+        except FileNotFoundError as file_not_found_error:
+            err = f"Unable to access '{self.filename}'."
+            self.panic(err)
+            raise file_not_found_error
+
+    def __del__(self):
+        self.logfile.close()
+
+    def __write_formatted_log(self, operator: str, text: str):
+        lines = text.split('\n')
+        for line in lines:
+            self.logfile.write(f"[{operator}] {line}\n")
+
+    def log(self, text: str):
+        """
+        Creates a log entry.
+        Supports multiple lines.
+
+        Parameters:
+            text (str): The contents of the log entry.
+        """
+        self.__write_formatted_log(
+            f"{self.format.ORANGE}*{self.format.NC}",
+            text
+        )
+
+    def add_finding(self, description: str):
+        """
+        Creates a log entry.
+        Supports multiple lines.
+
+        Parameters:
+            description (str): A description of the finding.
+        """
+        self.findings.append(description)
+        self.__write_formatted_log(
+            f"{self.format.GREEN}F{len(self.findings)}{self.format.NC}",
+            description
+        )
+
+    def panic(self, description: str):
+        """
+        Formats and logs error messages.
+        Does NOT terminate the script.
+        Supports multiple lines.
+
+        Parameters:
+            description (str): A description of the error or an error message.
+        """
+        self.__write_formatted_log(
+            f"{self.format.RED}!{self.format.NC}",
+            description
+        )
+
+
+def setup_module(argv: list, env: _Environ):
+    """
+    Creates a new emba module wrapper.
+
+    Parameters:
+        argv (list): The list of arguments used to start the Python process.
+        env (_Environ): The environment variables of the Python process.
+
+    Returns:
+        A new EmbaModule class instance.
+    """
+    return EmbaModule(argv, env)
+
+
+def shutdown_module(module: EmbaModule):
+    """
+    Shut down an emba python module.
+    This will also print the amount of findings as an
+    interger so EMBA can parse the number.
+
+    Parameters:
+        module (EmbaModule): A class instance of EmbaModule.
+
+    Returns:
+        none
+    """
+    print(f"FINDINGS:{len(module.findings)}", end="")
+    del module