Merge branch 'main' into prs/enhance-analyze-ceph

Makefile

@@ -4,12 +4,8 @@ COLOUR_BLUE=\033[0;34m
 COLOUR_END=\033[0m
 
 .DEFAULT_GOAL:=help
-SHELL := /bin/bash
+SHELL:=/bin/bash
 
-# Get needed paths and information from locally installed librados
-export RADOSLIB_VERSION := 2.0.0
-export GENERAL_LIB_LOCATION := ${shell pip show rados | grep -oP "(?<=Location: ).*"}
-export RADOSLIB_INSTALLED_VERSION := ${shell pip show rados | grep Version | awk '{print $$2}'}
 
 ## checking if docker, or podman should be used. Podman is preferred.
 ifeq ($(shell command -v podman 2> /dev/null),)
@@ -34,9 +30,13 @@ export ROOKIFY_VERSION?=0.0.0.dev1
 
 .PHONY: help
 help: ## Display this help message
-	@echo -e '${COLOUR_RED}Usage: make <command>${COLOUR_END}'
+	@echo -e '\n${COLOUR_BLUE}ROOKIFY MAKEFILE${COLOUR_BLUE}'
+	@echo -e '\n${COLOUR_RED}Usage: make <command>${COLOUR_END}'
 	@cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]'  | \
 	    awk -F ':.*?## ' 'NF==2 {printf "  %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}'
+	@echo -e '${COLOUR_RED}OSISM helperscript usage: make <command>${COLOUR_END}'
+	@cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]'  | \
+	    awk -F ':.*?#osism# ' 'NF==2 {printf "  %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}'
 
 .PHONY: setup
 setup: check-radoslib setup-venv setup-pre-commit ## Setup the pre-commit environment and then the venv environment
@@ -60,19 +60,18 @@ update-requirements: ## Update the requirements.txt with newer versions of pip p
 	pip freeze -l > requirements.txt
 
 .PHONY: check-radoslib
+export RADOSLIB_VERSION:=2.0.0
 check-radoslib: ## Checks if radoslib is installed and if it contains the right version
-	@if [ -z "${GENERAL_LIB_LOCATION}" ]; then \
-		echo -e "${COLOUR_RED}ERROR: 'rados' library not found. Please make sure it's installed.${COLOUR_END}"; \
-		exit 1; \
-	else \
-		echo -e "GENERAL_LIB_LOCATION: $(GENERAL_LIB_LOCATION)"; \
-	fi
-	@if [ "${RADOSLIB_INSTALLED_VERSION}" != "${RADOSLIB_VERSION}" ]; then \
-		echo -e "${COLOUR_RED}ERROR: Incorrect version of 'rados' library found. Expected version $(RADOSLIB_VERSION), found $$RADOSLIB_INSTALLED_VERSION.${COLOUR_END}"; \
-		exit 1; \
-	else \
-		echo -e "RADOSLIB_INSTALLED_VERSION: $(RADOSLIB_INSTALLED_VERSION)"; \
-	fi
+	# Get needed paths and information from locally installed librados
+	./scripts/check_local_rados_lib_installation.sh ${RADOSLIB_VERSION}
+
+.PHONY: build-local-rookify
+build-local-rookify: ## This builds rookify into .venv/bin/rookify
+	source .venv/bin/activate && pip install -e .
+
+.PHONY: build-container
+build-container: ## Build container from Dockerfile, add e.g. ROOKIFY_VERSION=0.0.1 to specify the version. Default value is 0.0.0.dev1
+	${CONTAINERCMD} build --build-arg ROOKIFY_VERSION=$(ROOKIFY_VERSION) --target rookify -t rookify:latest -f Dockerfile .
 
 .PHONY: run-local-rookify
 run-local-rookify: ## Runs rookify in the local development environment (requires setup-venv)
@@ -82,9 +81,9 @@ run-local-rookify: ## Runs rookify in the local development environment (require
 run-rookify: ## Runs rookify in the container
 	docker exec -it rookify-dev /app/rookify/.venv/bin/rookify
 
-.PHONY: build-container
-build-container: ## Build container from Dockerfile, add e.g. ROOKIFY_VERSION=0.0.1 to specify the version. Default value is 0.0.0.dev1
-	${CONTAINERCMD} build --build-arg ROOKIFY_VERSION=$(ROOKIFY_VERSION) --target rookify -t rookify:latest -f Dockerfile .
+.PHONY: get-testbed-configs-for-rookify-testing
+get-testbed-configs-for-rookify-testing: ## Gets the needed config (like .kube, /etc/ceph and so on) from the testbed
+	bash ./scripts/get_configs_from_testbed.sh
 
 .PHONY: run-tests-locally
 run-tests-locally: ## Runs the tests in the tests directory. NB: check that your local setup is connected through vpn to the testbed!
@@ -109,3 +108,11 @@ down: ## Remove the containers as setup by docker-compose.yml
 .PHONY: up
 up: ## Sets up the container as specified in docker-compose.yml and opens a bash terminal
 	${CONTAINERCMD} compose up -d
+
+##
+# Add osism specific scripts below here (so they appear below helper header)
+##
+
+.PHONY: get-config
+get-config: #osism# Gets configuration files from the OSISM testbed
+	./scripts/osism/get_osism_configs_from_testbed.sh

requirements.txt

@@ -33,3 +33,4 @@ yamale==5.1.0
 websocket-client==1.7.0
 wrapt==1.16.0
 pytest==8.0.2
+pytest-structlog==1.1

scripts/check_local_rados_lib_installation.sh

@@ -0,0 +1,29 @@
+#!/bin/env bash
+
+# Check if a version argument is provided
+if [ $# -eq 1 ]; then
+    RADOSLIB_VERSION="$1"
+else
+    # Default version if no argument is provided
+    RADOSLIB_VERSION="2.0.0"
+fi
+
+# Get the location of the installed rados library
+GENERAL_LIB_LOCATION=$(pip show rados | grep -oP "(?<=Location: ).*")
+
+# Get the installed version of the rados library
+RADOSLIB_INSTALLED_VERSION=$(pip show rados | grep Version | awk '{print $2}')
+
+# Check if the rados library is installed
+if [ -z "$GENERAL_LIB_LOCATION" ]; then
+    echo -e "\033[0;31mERROR: 'rados' library not found. Please make sure it's installed.\033[0m"
+    exit 1
+fi
+
+# Check if the installed version matches the expected version
+if [ "$RADOSLIB_INSTALLED_VERSION" != "$RADOSLIB_VERSION" ]; then
+    echo -e "\033[0;31mERROR: 'rados' library version $RADOSLIB_INSTALLED_VERSION does not match the expected version $RADOSLIB_VERSION.\033[0m"
+    exit 1
+else
+    echo -e "\033[0;32m'rados' library version $RADOSLIB_INSTALLED_VERSION is correct.\033[0m"
+fi

src/rookify/__main__.py

@@ -1,33 +1,67 @@
 # -*- coding: utf-8 -*-
 
+import sys
+import argparse
 from argparse import ArgumentParser
+from typing import Any, Dict
 from .modules import load_modules
 from .modules.machine import Machine
+from .modules.module import ModuleHandler
 from .logger import configure_logging, get_logger
 from .yaml import load_config
 
 
-def main() -> None:
+def parse_args(args: list[str]) -> argparse.Namespace:
+    # Putting args-parser in seperate function to make this testable
     arg_parser = ArgumentParser("Rookify")
+
+    # --dry-run option
     arg_parser.add_argument("--dry-run", action="store_true", dest="dry_run_mode")
-    args = arg_parser.parse_args()
+
+    arg_parser.add_argument(
+        "--show-states",
+        action="store_true",
+        dest="show_states",
+        help="Show states of the modules.",
+    )
+
+    return arg_parser.parse_args(args)
+
+
+def main() -> None:
+    args = parse_args(sys.argv[1:])
 
     # Load configuration file
     try:
-        config = load_config("config.yaml")
+        config: Dict[str, Any] = load_config("config.yaml")
     except FileNotFoundError as err:
         raise SystemExit(f"Could not load config: {err}")
 
     # Configure logging
     try:
-        configure_logging(config["logging"])
+        if args.show_states is True:
+            configure_logging(
+                {"level": "ERROR", "format": {"renderer": "console", "time": "iso"}}
+            )
+        else:
+            configure_logging(config["logging"])
     except Exception as e:
         raise SystemExit(f"Error configuring logging: {e}")
 
+    # Get Logger
     log = get_logger()
-    log.debug("Executing Rookify")
+
+    log.info("Executing Rookify ...")
 
     machine = Machine(config["general"].get("machine_pickle_file"))
+
     load_modules(machine, config)
 
-    machine.execute(args.dry_run_mode)
+    if args.show_states is True:
+        ModuleHandler.show_states(machine, config)
+    else:
+        machine.execute(dry_run_mode=args.dry_run_mode)
+
+
+if __name__ == "__main__":
+    main()

src/rookify/modules/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import importlib
-from typing import Any, Dict
+from typing import Any, Dict, List
 from ..logger import get_logger
 from .machine import Machine
 
@@ -22,14 +22,28 @@ def __init__(self, module_name: str, message: str):
         self.message = message
 
 
-def _load_module(machine: Machine, config: Dict[str, Any], module_name: str) -> None:
+_modules_loaded: List[Any] = []
+
+
+def get_modules() -> List[Any]:
+    global _modules_loaded
+    return _modules_loaded.copy()
+
+
+def _load_module(
+    machine: Machine,
+    config: Dict[str, Any],
+    module_name: str,
+) -> None:
     """
     Dynamically loads a module from the 'rookify.modules' package.
 
     :param module_names: The module names to load
     :return: returns tuple of preflight_modules, modules
     """
 
+    global _modules_loaded
+
     if "." in module_name:
         absolute_module_name = module_name
     else:
@@ -40,7 +54,7 @@ def _load_module(machine: Machine, config: Dict[str, Any], module_name: str) ->
     except ModuleNotFoundError as e:
         raise ModuleLoadException(module_name, str(e))
 
-    additional_modules = []
+    additional_module_names = []
 
     if not hasattr(module, "ModuleHandler") or not callable(
         getattr(module.ModuleHandler, "register_states")
@@ -49,10 +63,13 @@ def _load_module(machine: Machine, config: Dict[str, Any], module_name: str) ->
 
     if hasattr(module.ModuleHandler, "REQUIRES"):
         assert isinstance(module.ModuleHandler.REQUIRES, list)
-        additional_modules = module.ModuleHandler.REQUIRES
+        additional_module_names = module.ModuleHandler.REQUIRES
+
+    for additional_module_name in additional_module_names:
+        _load_module(machine, config, additional_module_name)
 
-    for module_name in additional_modules:
-        _load_module(machine, config, module_name)
+    if module not in _modules_loaded:
+        _modules_loaded.append(module)
 
     module.ModuleHandler.register_states(machine, config)
 

src/rookify/modules/analyze_ceph/main.py

@@ -1,12 +1,39 @@
 # -*- coding: utf-8 -*-
 
-from typing import Any
+from collections import OrderedDict
+from typing import Any, Dict, Optional
 from ..machine import Machine
 from ..module import ModuleHandler
 
 
 class AnalyzeCephHandler(ModuleHandler):
-    def preflight(self) -> None:
+    def _process_command_result(
+        self, state_data: Any, command: str, value: Optional[Any] = None
+    ) -> bool:
+        """
+        Helper method to process commands by either setting or checking state data.
+        """
+
+        command_parts = command.split(" ")
+        state_structure_data = state_data  # the root of the data structure
+
+        # Traverse the dictionary structure based on command parts
+        for idx, key in enumerate(command_parts):
+            # Last part of the command
+            if len(command_parts) == idx + 1:
+                if value is None:
+                    return key in state_structure_data
+
+                state_structure_data[key] = value
+            else:
+                if key not in state_structure_data:
+                    state_structure_data[key] = {}
+
+                state_structure_data = state_structure_data[key]
+
+        return True
+
+    def preflight(self) -> Any:
         state = self.machine.get_preflight_state("AnalyzeCephHandler")
 
         if getattr(state, "data", None) is not None:
@@ -15,20 +42,41 @@ def preflight(self) -> None:
         commands = ["fs ls", "node ls", "report"]
         state.data = {}
 
+        # Execute each command and store the result
         for command in commands:
-            parts = command.split(" ")
-            leaf = state.data
-
-            for idx, part in enumerate(parts):
-                if len(parts) == idx + 1:
-                    leaf[part] = self.ceph.mon_command(command)
-                else:
-                    if part not in leaf:
-                        leaf[part] = {}
-                    leaf = leaf[part]
+            result = self.ceph.mon_command(command)
+            self._process_command_result(state.data, command, result)
 
         self.logger.info("AnalyzeCephHandler ran successfully.")
 
+    def get_readable_key_value_state(self) -> Dict[str, str]:
+        state = self.machine.get_preflight_state("AnalyzeCephHandler")
+
+        kv_state_data = OrderedDict()
+
+        if "mon" not in state.data or "dump" not in state.data["mon"]:
+            kv_state_data["ceph mon dump"] = "Not analyzed yet"
+        else:
+            kv_state_data["ceph mon dump"] = self._get_readable_json_dump(
+                state.data["mon"]["dump"]
+            )
+
+        if "osd" not in state.data or "dump" not in state.data["osd"]:
+            kv_state_data["ceph osd dump"] = "Not analyzed yet"
+        else:
+            kv_state_data["ceph osd dump"] = self._get_readable_json_dump(
+                state.data["osd"]["dump"]
+            )
+
+        if "device" not in state.data or "ls" not in state.data["device"]:
+            kv_state_data["OSD devices"] = "Not analyzed yet"
+        else:
+            kv_state_data["OSD devices"] = self._get_readable_json_dump(
+                state.data["device"]["ls"]
+            )
+
+        return kv_state_data
+
     @staticmethod
     def register_preflight_state(
         machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any

src/rookify/modules/cephx_auth_config/main.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-from typing import Any
+from typing import Any, Dict
 from ..exception import ModuleException
 from ..machine import Machine
 from ..module import ModuleHandler
@@ -30,6 +30,12 @@ def preflight(self) -> None:
     def is_cephx_set(self, values: str) -> Any:
         return "cephx" in [value.strip() for value in values.split(",")]
 
+    def get_readable_key_value_state(self) -> Dict[str, str]:
+        is_verified = self.machine.get_preflight_state_data(
+            "CephXAuthHandler", "verified", default_value=False
+        )
+        return {"cephx auth is verified": str(is_verified)}
+
     @staticmethod
     def register_preflight_state(
         machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any