From a30c9caee4baa008e8d1688fe6892ad426131d5d Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Tue, 10 Sep 2024 12:56:48 +0000 Subject: [PATCH 01/27] feat: add picklefile read option with tests Signed-off-by: Boekhorst --- src/rookify/__main__.py | 102 ++++++++++++++++-- src/rookify/modules/machine.py | 1 + tests/test_main.py | 192 +++++++++++++++++++++++++++++++++ 3 files changed, 286 insertions(+), 9 deletions(-) create mode 100644 tests/test_main.py diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 4dd6383..5b61e27 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -1,20 +1,67 @@ # -*- coding: utf-8 -*- -from argparse import ArgumentParser +import json +from pickle import Unpickler +import sys +import argparse +from argparse import ArgumentParser, Namespace +from typing import Any, Dict, Optional from .modules import load_modules from .modules.machine import Machine from .logger import configure_logging, get_logger from .yaml import load_config -def main() -> None: +def parse_args(args: list[str]) -> Namespace: + # Putting args-parser in seperate function to make this testable arg_parser = ArgumentParser("Rookify") arg_parser.add_argument("--dry-run", action="store_true", dest="dry_run_mode") - args = arg_parser.parse_args() + + # Custom ShowAction to set 'all' if nothing is specified for --show + class ShowAction(argparse.Action): + def __call__( + self, + parser: ArgumentParser, + namespace: Namespace, + values: Optional[Any], + option_string: Optional[str] = None, + ) -> None: + setattr(namespace, self.dest, values if values is not None else "all") + + arg_parser.add_argument( + "--show", + nargs="?", + action=ShowAction, + dest="show_progress", + metavar="
", + help="Show the state of progress, as read from the pickle file. Default argument is 'all', you can also specify a section you want to look at.", + required=False, + ) + return arg_parser.parse_args(args) + + +def load_pickler(pickle_file_name: str) -> Any: + with open(pickle_file_name, "ab+") as pickle_file: + pickle_file.seek(0) + states_data = Unpickler(pickle_file).load() + return states_data + + +def sort_pickle_file(unsorted_states_data: Dict[str, Any]) -> Dict[str, Any]: + # sort the pickle-file alfabetically + iterable_dict = iter(unsorted_states_data) + first_key = next(iterable_dict) + data_values = unsorted_states_data[first_key]["data"] + sorted_data_by_keys = {k: data_values[k] for k in sorted(data_values)} + return sorted_data_by_keys + + +def main() -> None: + args = parse_args(sys.argv[1:]) # Load configuration file try: - config = load_config("config.yaml") + config: Dict[str, Any] = load_config("config.yaml") except FileNotFoundError as err: raise SystemExit(f"Could not load config: {err}") @@ -23,11 +70,48 @@ def main() -> None: configure_logging(config["logging"]) except Exception as e: raise SystemExit(f"Error configuring logging: {e}") - + # Get Logger log = get_logger() - log.debug("Executing Rookify") - machine = Machine(config["general"].get("machine_pickle_file")) - load_modules(machine, config) + # Get Pickle File if configured in config.yaml + pickle_file_name = config["general"].get("machine_pickle_file") + if pickle_file_name is None: + log.info("No pickle file was set in the configuration.") + else: + log.info(f"Pickle file set: {pickle_file_name}") + + # Get Pickle File if configured in config.yaml + pickle_file_name = config["general"].get("machine_pickle_file") + if pickle_file_name is None: + log.info("No pickle file was set in the configuration.") + else: + log.info(f"Pickle file set: {pickle_file_name}") + + # If show_progress is not None and pickle_file_name is not None, show the current progress of the migration + if args.show_progress is not None: + if pickle_file_name is None: + return + states_data = load_pickler(pickle_file_name) + sorted_states_data = sort_pickle_file(states_data) + + # Check if a specific section should be listed + if args.show_progress != "all": + if args.show_progress not in sorted_states_data.keys(): + get_logger().error(f"The section {args.show_progress} does not exist") + return + else: + sorted_states_data = sorted_states_data[args.show_progress] + + get_logger().info( + 'Current state as retrieved from pickle-file: \n "{0}": {1}'.format( + args.show_progress, json.dumps(sorted_states_data, indent=4) + ) + ) + # Else run the rook migration + else: + log.debug("Executing Rookify") + + machine = Machine(config["general"].get("machine_pickle_file")) + load_modules(machine, config) - machine.execute(args.dry_run_mode) + machine.execute(dry_run_mode=args.dry_run_mode) diff --git a/src/rookify/modules/machine.py b/src/rookify/modules/machine.py index cb8c9d5..3a6bcf8 100644 --- a/src/rookify/modules/machine.py +++ b/src/rookify/modules/machine.py @@ -52,6 +52,7 @@ def execute(self, dry_run_mode: bool = False) -> None: def _execute(self, pickle_file: Optional[IO[Any]] = None) -> None: states_data = {} + # Read pickle file if it exists, to continue from the stored state if pickle_file is not None and pickle_file.tell() > 0: pickle_file.seek(0) diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..9ef0bfb --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,192 @@ +import sys +from typing import Any, Callable, Optional +import pytest +from _pytest.monkeypatch import MonkeyPatch +import yaml +from unittest.mock import MagicMock +import pytest_structlog + +from argparse import Namespace + +from rookify.__main__ import parse_args, main, sort_pickle_file +import rookify.yaml + + +# Test the arugment parser +def test_parse_args_dry_run() -> None: + args = parse_args(["--dry-run"]) + expected = Namespace(dry_run_mode=True, show_progress=None) + assert args == expected + + +def test_parse_args_show_progress() -> None: + args = parse_args(["--show"]) + expected = Namespace(dry_run_mode=False, show_progress="all") + assert args == expected + + +def test_parse_args_both_flags() -> None: + args = parse_args(["--dry-run", "--show"]) + expected = Namespace(dry_run_mode=True, show_progress="all") + assert args == expected + + +def test_parse_args_no_flags() -> None: + args = parse_args([]) + expected = Namespace(dry_run_mode=False, show_progress=None) + assert args == expected + + +# Test the --show option + + +@pytest.fixture # type: ignore +def mock_load_config(monkeypatch: MonkeyPatch) -> Callable[[Optional[Any]], MagicMock]: + def _mock_load_config(pickle_file: Optional[Any] = None) -> MagicMock: + # Mock the configuration data + # Load config.example.yaml + with open("config.example.yaml", "r") as file: + config_data = yaml.safe_load(file) + + config_data["general"]["machine_pickle_file"] = pickle_file + + # Mock load_config function + mock = MagicMock(return_value=config_data) + monkeypatch.setattr(rookify.__main__, "load_config", mock) + + return mock + + return _mock_load_config + + +@pytest.fixture # type: ignore +def mock_load_pickler(monkeypatch: MonkeyPatch) -> MagicMock: + # Mock states_data from pickle file + mock_states_data = {"teststuff": {"data": {"mock_key": "mock_value"}}} + + # Mock load_pickler function + mock = MagicMock(return_value=mock_states_data) + monkeypatch.setattr(rookify.__main__, "load_pickler", mock) + + return mock + + +def test_main_show_progress( + mock_load_config: Callable[[Optional[Any]], MagicMock], + mock_load_pickler: MonkeyPatch, + monkeypatch: MonkeyPatch, + log: pytest_structlog.StructuredLogCapture, +) -> None: + # Load example config with mock.pickle as pickle file + mock_load_config("mock.pickle") + + # Mock sys.argv to simulate command-line arguments + monkeypatch.setattr(sys, "argv", ["main_script.py", "--show", "--dry-run"]) + + # Run main() + main() + + # Verify logging messages + expected_log_message = ( + 'Current state as retrieved from pickle-file: \n "all": ' + "{\n" + ' "mock_key": "mock_value"\n' + "}" + ) + assert log.has("Pickle file set: mock.pickle", level="info") + assert log.has(expected_log_message, level="info") + + +def test_main_no_pickle_file( + mock_load_config: Callable[[Optional[str]], MagicMock], + mock_load_pickler: MonkeyPatch, + monkeypatch: MonkeyPatch, + log: pytest_structlog.StructuredLogCapture, +) -> None: + # Load a configuration without pickle: This should load the default data.pickle file if it is available + mock_load_config(None) + + # Mock sys.argv to simulate command-line arguments + monkeypatch.setattr(sys, "argv", ["main_script.py", "--show", "--dry-run"]) + + # Run main() + main() + + # Assertions + assert log.has("No pickle file was set in the configuration.") + + +def test_sort_pickle_file() -> None: + # Prepare unsorted data + unsorted_states_data = getUnsortedData() + + # Expected keys order + expected_order = ["device", "fs", "mon", "node", "osd", "ssh"] + + # Run sort_pickle_file + sorted_states_data = sort_pickle_file(unsorted_states_data) + + # Assert that order is correct + sorted_states_data_keys = list(sorted_states_data.keys()) + + assert expected_order == sorted_states_data_keys + + +def getUnsortedData() -> Any: + unsorted_states_data = { + "preflighttestdata": { + "data": { + "mon": { + "dump": {"epoch": 1}, + "mons": [{"rank": 0}], + "quorum": [0, 1, 2], + }, + "fs": {"dump": {"epoch": 1}}, + "device": {"ls": ["devid", 1]}, + "osd": { + "dump": [{"epoch": 138}], + "osds": [{"osd": 0}], + "osd_xinfo": [{"osd": 0}], + }, + "node": { + "ls": { + "mon": { + "test-node-0": ["test-node-0"], + "test-node-1": ["test-node-1"], + "test-node-2": ["test-node-2"], + }, + "osd": { + "test-node-0": [2, 3], + "test-node-1": [0, 5], + "test-node-2": [1, 4], + }, + "mds": { + "test-node-0": ["test-node-0"], + "test-node-1": ["test-node-1"], + "test-node-2": ["test-node-2"], + }, + "mgr": { + "test-node-0": ["test-node-0"], + "test-node-1": ["test-node-1"], + "test-node-2": ["test-node-2"], + }, + } + }, + "ssh": { + "osd": { + "test-node-0": { + "devices": ["/dev/ceph-bla-1", "/dev/ceph-bla-2"] + }, + "test-node-1": { + "devices": ["/dev/ceph-bla-3", "/dev/ceph-bla-4"] + }, + "test-node-2": { + "devices": ["/dev/ceph-bla-5", "/dev/ceph-bla-6"] + }, + } + }, + } + } + } + + return unsorted_states_data From d27d5ad8aa608e1802015f3d900ee7a195a21614 Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Tue, 10 Sep 2024 13:09:41 +0000 Subject: [PATCH 02/27] updating Makefile as in #73 Signed-off-by: Boekhorst --- Makefile | 78 ++++++++++++------- scripts/check_local_rados_lib_installation.sh | 29 +++++++ 2 files changed, 77 insertions(+), 30 deletions(-) create mode 100755 scripts/check_local_rados_lib_installation.sh diff --git a/Makefile b/Makefile index ec30b5c..3f9399a 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,8 @@ COLOUR_BLUE=\033[0;34m COLOUR_END=\033[0m .DEFAULT_GOAL:=help -SHELL := /bin/bash +SHELL:=/bin/bash -# Get needed paths and information from locally installed librados -export RADOSLIB_VERSION := 2.0.0 -export GENERAL_LIB_LOCATION := ${shell pip show rados | grep -oP "(?<=Location: ).*"} -export RADOSLIB_INSTALLED_VERSION := ${shell pip show rados | grep Version | awk '{print $$2}'} ## checking if docker, or podman should be used. Podman is preferred. ifeq ($(shell command -v podman 2> /dev/null),) @@ -18,27 +14,41 @@ else CONTAINERCMD=podman endif +# Checking if python exists +ifneq (, $(shell command -v python)) + $(info Python is installed as 'python') + PYTHON := $(shell command -v python) +else ifneq (, $(shell command -v python3)) + $(info Python3 is installed as 'python3') + PYTHON := $(shell command -v python3) +else + $(error Neither python nor python3 is installed) +endif + ## Export default rookify version -export ROOKIFY_VERSION ?= "0.0.0.dev1" +export ROOKIFY_VERSION?=0.0.0.dev1 .PHONY: help help: ## Display this help message - @echo -e '${COLOUR_RED}Usage: make ${COLOUR_END}' + @echo -e '\n${COLOUR_BLUE}ROOKIFY MAKEFILE${COLOUR_BLUE}' + @echo -e '\n${COLOUR_RED}Usage: make ${COLOUR_END}' @cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]' | \ awk -F ':.*?## ' 'NF==2 {printf " %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}' + @echo -e '${COLOUR_RED}OSISM helperscript usage: make ${COLOUR_END}' + @cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]' | \ + awk -F ':.*?#osism# ' 'NF==2 {printf " %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}' .PHONY: setup -setup: setup-pre-commit check-radoslib setup-venv ## Setup the pre-commit environment and then the venv environment +setup: check-radoslib setup-venv setup-pre-commit ## Setup the pre-commit environment and then the venv environment .PHONY: setup-pre-commit setup-pre-commit: - pip install --user pre-commit && pre-commit install + ./.venv/bin/pip install --user pre-commit && ./.venv/bin/python -m pre_commit install .PHONY: setup-venv setup-venv: - python -m venv --system-site-packages ./.venv && \ - source ./.venv/bin/activate && \ - pip install -r requirements.txt + ${PYTHON} -m venv --system-site-packages ./.venv && \ + ./.venv/bin/pip install -r requirements.txt .PHONY: run-precommit run-precommit: ## Run pre-commit to check if all files running through @@ -50,33 +60,33 @@ update-requirements: ## Update the requirements.txt with newer versions of pip p pip freeze -l > requirements.txt .PHONY: check-radoslib +export RADOSLIB_VERSION:=2.0.0 check-radoslib: ## Checks if radoslib is installed and if it contains the right version - @if [ -z "${GENERAL_LIB_LOCATION}" ]; then \ - echo -e "${COLOUR_RED}ERROR: 'rados' library not found. Please make sure it's installed.${COLOUR_END}"; \ - exit 1; \ - else \ - echo -e "GENERAL_LIB_LOCATION: $(GENERAL_LIB_LOCATION)"; \ - fi - @if [ "${RADOSLIB_INSTALLED_VERSION}" != "${RADOSLIB_VERSION}" ]; then \ - echo -e "${COLOUR_RED}ERROR: Incorrect version of 'rados' library found. Expected version $(RADOSLIB_VERSION), found $$RADOSLIB_INSTALLED_VERSION.${COLOUR_END}"; \ - exit 1; \ - else \ - echo -e "RADOSLIB_INSTALLED_VERSION: $(RADOSLIB_INSTALLED_VERSION)"; \ - fi + # Get needed paths and information from locally installed librados + ./scripts/check_local_rados_lib_installation.sh ${RADOSLIB_VERSION} + +.PHONY: build-local-rookify +build-local-rookify: ## This builds rookify into .venv/bin/rookify + source .venv/bin/activate && pip install -e . + +.PHONY: build-container +build-container: ## Build container from Dockerfile, add e.g. ROOKIFY_VERSION=0.0.1 to specify the version. Default value is 0.0.0.dev1 + ${CONTAINERCMD} build --build-arg ROOKIFY_VERSION=$(ROOKIFY_VERSION) --target rookify -t rookify:latest -f Dockerfile . .PHONY: run-local-rookify run-local-rookify: ## Runs rookify in the local development environment (requires setup-venv) - $(eval PYTHONPATH="${PYTHONPATH}:$(pwd)/src") - source ./.venv/bin/activate && \ - cd src && python -m rookify + if [ ! -f ./.venv/bin/rookify ]; then \ + ${MAKE} build-local-rookify; \ + fi; \ + .venv/bin/rookify .PHONY: run-rookify run-rookify: ## Runs rookify in the container docker exec -it rookify-dev /app/rookify/.venv/bin/rookify -.PHONY: build-container -build-container: ## Build container from Dockerfile, add e.g. ROOKIFY_VERSION=0.0.1 to specify the version. Default value is 0.0.0.dev1 - ${CONTAINERCMD} build --build-arg ROOKIFY_VERSION=$(ROOKIFY_VERSION) --target rookify -t rookify:latest -f Dockerfile . +.PHONY: get-testbed-configs-for-rookify-testing +get-testbed-configs-for-rookify-testing: ## Gets the needed config (like .kube, /etc/ceph and so on) from the testbed + bash ./scripts/get_configs_from_testbed.sh .PHONY: run-tests-locally run-tests-locally: ## Runs the tests in the tests directory. NB: check that your local setup is connected through vpn to the testbed! @@ -101,3 +111,11 @@ down: ## Remove the containers as setup by docker-compose.yml .PHONY: up up: ## Sets up the container as specified in docker-compose.yml and opens a bash terminal ${CONTAINERCMD} compose up -d + +## +# Add osism specific scripts below here (so they appear below helper header) +## + +.PHONY: get-config +get-config: #osism# Gets configuration files from the OSISM testbed + ./scripts/osism/get_osism_configs_from_testbed.sh diff --git a/scripts/check_local_rados_lib_installation.sh b/scripts/check_local_rados_lib_installation.sh new file mode 100755 index 0000000..6d88303 --- /dev/null +++ b/scripts/check_local_rados_lib_installation.sh @@ -0,0 +1,29 @@ +#!/bin/env bash + +# Check if a version argument is provided +if [ $# -eq 1 ]; then + RADOSLIB_VERSION="$1" +else + # Default version if no argument is provided + RADOSLIB_VERSION="2.0.0" +fi + +# Get the location of the installed rados library +GENERAL_LIB_LOCATION=$(pip show rados | grep -oP "(?<=Location: ).*") + +# Get the installed version of the rados library +RADOSLIB_INSTALLED_VERSION=$(pip show rados | grep Version | awk '{print $2}') + +# Check if the rados library is installed +if [ -z "$GENERAL_LIB_LOCATION" ]; then + echo -e "\033[0;31mERROR: 'rados' library not found. Please make sure it's installed.\033[0m" + exit 1 +fi + +# Check if the installed version matches the expected version +if [ "$RADOSLIB_INSTALLED_VERSION" != "$RADOSLIB_VERSION" ]; then + echo -e "\033[0;31mERROR: 'rados' library version $RADOSLIB_INSTALLED_VERSION does not match the expected version $RADOSLIB_VERSION.\033[0m" + exit 1 +else + echo -e "\033[0;32m'rados' library version $RADOSLIB_INSTALLED_VERSION is correct.\033[0m" +fi From 02fc7c10190cff84367a87e0e618133459defbaf Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Tue, 10 Sep 2024 13:51:26 +0000 Subject: [PATCH 03/27] fix: addin pytest_structlog Signed-off-by: Boekhorst --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 1c8606a..330c82c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,3 +32,5 @@ urllib3==2.2.1 yamale==5.1.0 websocket-client==1.7.0 wrapt==1.16.0 +pytest==8.0.2 +pytest-structlog==1.1 From 6fe4b39d5a86b2600cb75ba1b64222d7041e1f43 Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Tue, 10 Sep 2024 13:58:18 +0000 Subject: [PATCH 04/27] fix: rename show progress to read pickle Signed-off-by: Boekhorst --- src/rookify/__main__.py | 26 +++++++++++++------------- tests/test_main.py | 27 ++++++++++++++++----------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 5b61e27..a7de3f0 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -17,8 +17,8 @@ def parse_args(args: list[str]) -> Namespace: arg_parser = ArgumentParser("Rookify") arg_parser.add_argument("--dry-run", action="store_true", dest="dry_run_mode") - # Custom ShowAction to set 'all' if nothing is specified for --show - class ShowAction(argparse.Action): + # Custom ReadAction to set 'all' if nothing is specified for --read-pickle + class ReadAction(argparse.Action): def __call__( self, parser: ArgumentParser, @@ -29,12 +29,12 @@ def __call__( setattr(namespace, self.dest, values if values is not None else "all") arg_parser.add_argument( - "--show", + "--read-pickle", nargs="?", - action=ShowAction, - dest="show_progress", + action=ReadAction, + dest="read_pickle", metavar="
", - help="Show the state of progress, as read from the pickle file. Default argument is 'all', you can also specify a section you want to look at.", + help="Show the content of the pickle file. Default argument is 'all', you can also specify a section you want to look at.", required=False, ) return arg_parser.parse_args(args) @@ -87,24 +87,24 @@ def main() -> None: else: log.info(f"Pickle file set: {pickle_file_name}") - # If show_progress is not None and pickle_file_name is not None, show the current progress of the migration - if args.show_progress is not None: + # If read_pickle is not None and pickle_file_name is not None, show the current progress of the migration + if args.read_pickle is not None: if pickle_file_name is None: return states_data = load_pickler(pickle_file_name) sorted_states_data = sort_pickle_file(states_data) # Check if a specific section should be listed - if args.show_progress != "all": - if args.show_progress not in sorted_states_data.keys(): - get_logger().error(f"The section {args.show_progress} does not exist") + if args.read_pickle != "all": + if args.read_pickle not in sorted_states_data.keys(): + get_logger().error(f"The section {args.read_pickle} does not exist") return else: - sorted_states_data = sorted_states_data[args.show_progress] + sorted_states_data = sorted_states_data[args.read_pickle] get_logger().info( 'Current state as retrieved from pickle-file: \n "{0}": {1}'.format( - args.show_progress, json.dumps(sorted_states_data, indent=4) + args.read_pickle, json.dumps(sorted_states_data, indent=4) ) ) # Else run the rook migration diff --git a/tests/test_main.py b/tests/test_main.py index 9ef0bfb..f4c7856 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -15,29 +15,34 @@ # Test the arugment parser def test_parse_args_dry_run() -> None: args = parse_args(["--dry-run"]) - expected = Namespace(dry_run_mode=True, show_progress=None) + expected = Namespace(dry_run_mode=True, read_pickle=None) assert args == expected -def test_parse_args_show_progress() -> None: - args = parse_args(["--show"]) - expected = Namespace(dry_run_mode=False, show_progress="all") +def test_parse_args_read_pickle() -> None: + args = parse_args(["--read-pickle"]) + expected = Namespace(dry_run_mode=False, read_pickle="all") assert args == expected def test_parse_args_both_flags() -> None: - args = parse_args(["--dry-run", "--show"]) - expected = Namespace(dry_run_mode=True, show_progress="all") + args = parse_args(["--dry-run", "--read-pickle"]) + expected = Namespace(dry_run_mode=True, read_pickle="all") assert args == expected def test_parse_args_no_flags() -> None: args = parse_args([]) - expected = Namespace(dry_run_mode=False, show_progress=None) + expected = Namespace(dry_run_mode=False, read_pickle=None) assert args == expected -# Test the --show option +# def test_parse_args_show_progress() -> None: +# args = parse_args([]) +# expected = Namespace(dry_run_mode=True, show_progress=None) +# assert args == expected + +# Test the --read-pickle option @pytest.fixture # type: ignore @@ -71,7 +76,7 @@ def mock_load_pickler(monkeypatch: MonkeyPatch) -> MagicMock: return mock -def test_main_show_progress( +def test_main_read_pickle( mock_load_config: Callable[[Optional[Any]], MagicMock], mock_load_pickler: MonkeyPatch, monkeypatch: MonkeyPatch, @@ -81,7 +86,7 @@ def test_main_show_progress( mock_load_config("mock.pickle") # Mock sys.argv to simulate command-line arguments - monkeypatch.setattr(sys, "argv", ["main_script.py", "--show", "--dry-run"]) + monkeypatch.setattr(sys, "argv", ["main_script.py", "--read-pickle", "--dry-run"]) # Run main() main() @@ -107,7 +112,7 @@ def test_main_no_pickle_file( mock_load_config(None) # Mock sys.argv to simulate command-line arguments - monkeypatch.setattr(sys, "argv", ["main_script.py", "--show", "--dry-run"]) + monkeypatch.setattr(sys, "argv", ["main_script.py", "--read-pickle", "--dry-run"]) # Run main() main() From 08d73561800a2a3cc1b530bbe41b9b004b0e62e8 Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Tue, 10 Sep 2024 14:19:46 +0000 Subject: [PATCH 05/27] feat: adding cli option to show progress, adding tests as well Signed-off-by: Boekhorst --- src/rookify/__main__.py | 21 +++++++++++++++++++++ tests/test_main.py | 34 ++++++++++++++++++++++++---------- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index a7de3f0..c07c4d6 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -28,6 +28,17 @@ def __call__( ) -> None: setattr(namespace, self.dest, values if values is not None else "all") + # Custom ShowProgressAction to set 'all' if nothing is specified for --show-progress + class ShowProgressAction(argparse.Action): + def __call__( + self, + parser: ArgumentParser, + namespace: Namespace, + values: Optional[Any], + option_string: Optional[str] = None, + ) -> None: + setattr(namespace, self.dest, values if values is not None else "all") + arg_parser.add_argument( "--read-pickle", nargs="?", @@ -37,6 +48,16 @@ def __call__( help="Show the content of the pickle file. Default argument is 'all', you can also specify a section you want to look at.", required=False, ) + + arg_parser.add_argument( + "--show-progress", + nargs="?", + action=ShowProgressAction, + dest="show_progress", + metavar="", + help="Show progress of the modules. Default argument is 'all', you can also specify a module you want to get the progress status from.", + required=False, + ) return arg_parser.parse_args(args) diff --git a/tests/test_main.py b/tests/test_main.py index f4c7856..f61edf3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -15,34 +15,48 @@ # Test the arugment parser def test_parse_args_dry_run() -> None: args = parse_args(["--dry-run"]) - expected = Namespace(dry_run_mode=True, read_pickle=None) + expected = Namespace(dry_run_mode=True, read_pickle=None, show_progress=None) assert args == expected def test_parse_args_read_pickle() -> None: args = parse_args(["--read-pickle"]) - expected = Namespace(dry_run_mode=False, read_pickle="all") + expected = Namespace(dry_run_mode=False, read_pickle="all", show_progress=None) assert args == expected def test_parse_args_both_flags() -> None: args = parse_args(["--dry-run", "--read-pickle"]) - expected = Namespace(dry_run_mode=True, read_pickle="all") + expected = Namespace(dry_run_mode=True, read_pickle="all", show_progress=None) + assert args == expected + + +def test_parse_args_show_progress() -> None: + args = parse_args(["--show-progress"]) + expected = Namespace(dry_run_mode=False, read_pickle=None, show_progress="all") + assert args == expected + + +# check: should it be possible to add all arguments? +def test_parse_args_both_dry_run_show_progress() -> None: + args = parse_args(["--dry-run", "--read-pickle", "--show-progress"]) + expected = Namespace(dry_run_mode=True, read_pickle="all", show_progress="all") + assert args == expected + + +def test_parse_args_all_dry_run_show_progress_read_pickle() -> None: + args = parse_args(["--dry-run", "--show-progress"]) + expected = Namespace(dry_run_mode=True, read_pickle=None, show_progress="all") assert args == expected def test_parse_args_no_flags() -> None: args = parse_args([]) - expected = Namespace(dry_run_mode=False, read_pickle=None) + expected = Namespace(dry_run_mode=False, read_pickle=None, show_progress=None) assert args == expected -# def test_parse_args_show_progress() -> None: -# args = parse_args([]) -# expected = Namespace(dry_run_mode=True, show_progress=None) -# assert args == expected - -# Test the --read-pickle option +# Test the --read-pickle and --show-progress options @pytest.fixture # type: ignore From 3e9d54decb01750197ff5d146a9b9387a52abf24 Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Wed, 11 Sep 2024 08:44:12 +0000 Subject: [PATCH 06/27] fix: change decision tree for arguments to clearify usage of --show-progress Signed-off-by: Boekhorst --- mock.pickle | Bin 0 -> 66 bytes src/rookify/__main__.py | 76 ++++++++++++++++------- src/rookify/modules/analyze_ceph/main.py | 4 ++ tests/test_main.py | 33 ++++++++++ 4 files changed, 90 insertions(+), 23 deletions(-) create mode 100644 mock.pickle diff --git a/mock.pickle b/mock.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a4d3c945bef508c85556f9882f454b589ae4b0d5 GIT binary patch literal 66 zcmZo*nQG1e0ku Namespace: +def parse_args(args: list[str]) -> argparse.Namespace: # Putting args-parser in seperate function to make this testable arg_parser = ArgumentParser("Rookify") arg_parser.add_argument("--dry-run", action="store_true", dest="dry_run_mode") @@ -22,7 +23,7 @@ class ReadAction(argparse.Action): def __call__( self, parser: ArgumentParser, - namespace: Namespace, + namespace: argparse.Namespace, values: Optional[Any], option_string: Optional[str] = None, ) -> None: @@ -33,7 +34,7 @@ class ShowProgressAction(argparse.Action): def __call__( self, parser: ArgumentParser, - namespace: Namespace, + namespace: argparse.Namespace, values: Optional[Any], option_string: Optional[str] = None, ) -> None: @@ -77,6 +78,26 @@ def sort_pickle_file(unsorted_states_data: Dict[str, Any]) -> Dict[str, Any]: return sorted_data_by_keys +def read_pickle_file( + args: argparse.Namespace, pickle_file_name: str, log: BindableLogger +) -> None: + states_data = load_pickler(pickle_file_name) + sorted_states_data = sort_pickle_file(states_data) + + # Check if a specific section should be listed + if args.read_pickle != "all": + if args.read_pickle not in sorted_states_data.keys(): + log.error(f"The section {args.read_pickle} does not exist") + else: + sorted_states_data = sorted_states_data[args.read_pickle] + + log.info( + 'Current state as retrieved from pickle-file: \n "{0}": {1}'.format( + args.read_pickle, json.dumps(sorted_states_data, indent=4) + ) + ) + + def main() -> None: args = parse_args(sys.argv[1:]) @@ -108,26 +129,35 @@ def main() -> None: else: log.info(f"Pickle file set: {pickle_file_name}") - # If read_pickle is not None and pickle_file_name is not None, show the current progress of the migration - if args.read_pickle is not None: - if pickle_file_name is None: - return - states_data = load_pickler(pickle_file_name) - sorted_states_data = sort_pickle_file(states_data) - - # Check if a specific section should be listed - if args.read_pickle != "all": - if args.read_pickle not in sorted_states_data.keys(): - get_logger().error(f"The section {args.read_pickle} does not exist") - return - else: - sorted_states_data = sorted_states_data[args.read_pickle] - - get_logger().info( - 'Current state as retrieved from pickle-file: \n "{0}": {1}'.format( - args.read_pickle, json.dumps(sorted_states_data, indent=4) - ) + # If read_pickle is run and there is a picklefile, show the picklefiles contents. + # NOTE: preflight mode (--dry-run) has no effect here, because no module actions are required. + if args.read_pickle is not None and pickle_file_name is not None: + read_pickle_file(args, pickle_file_name, log) + return + elif args.read_pickle is not None and pickle_file_name is None: + log.info( + "No pickle file configured to read from. Check if the pickle file exists and is configured in config.yaml" ) + return + + # If show_progress is run and there is a picklefile, show progress status based on picklefile contents + # NOTE: this is always run in preflight-mode (migration shoudl not be executed) + if args.show_progress is not None and pickle_file_name is not None: + # Check if a specific mode should be targeted + if args.show_progress != "all": + print("TEST") + print(args.show_progress) + + # TODO: implement module and method loading to let module check state itself + return + + # TODO: should progress be checkable if no pickle file is present? Should rookify check the status by analyzing the source and traget machines state? + elif args.show_progress is not None and pickle_file_name is None: + log.info( + "Currently rookify can only check the state of progress by analyzing the pickle file states." + ) + return + # Else run the rook migration else: log.debug("Executing Rookify") diff --git a/src/rookify/modules/analyze_ceph/main.py b/src/rookify/modules/analyze_ceph/main.py index 1698196..6bc0d71 100644 --- a/src/rookify/modules/analyze_ceph/main.py +++ b/src/rookify/modules/analyze_ceph/main.py @@ -6,6 +6,10 @@ class AnalyzeCephHandler(ModuleHandler): + def status(self) -> Any: + # state = self.machine.get_preflight_state("AnalyzeCephHandler") + self.logger.info("AnalyzeCephHander has allready been run") + def preflight(self) -> Any: commands = ["mon dump", "osd dump", "device ls", "fs ls", "node ls"] diff --git a/tests/test_main.py b/tests/test_main.py index f61edf3..3a0c5b7 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -11,6 +11,13 @@ from rookify.__main__ import parse_args, main, sort_pickle_file import rookify.yaml +# +# These tests test the src/rookify/__main__.py +# +# Run all tests: .venv/bin/python -m pytest tests/ +# Run one test with output: .venv/bin/python -m pytest tests/ -k name_of_test -s +# + # Test the arugment parser def test_parse_args_dry_run() -> None: @@ -37,6 +44,14 @@ def test_parse_args_show_progress() -> None: assert args == expected +def test_parse_args_show_progress_with_module() -> None: + args = parse_args(["--show-progress", "ceph-analyze"]) + expected = Namespace( + dry_run_mode=False, read_pickle=None, show_progress="ceph-analyze" + ) + assert args == expected + + # check: should it be possible to add all arguments? def test_parse_args_both_dry_run_show_progress() -> None: args = parse_args(["--dry-run", "--read-pickle", "--show-progress"]) @@ -209,3 +224,21 @@ def getUnsortedData() -> Any: } return unsorted_states_data + + +def test_ceph_analyze_progress( + mock_load_config: Callable[[Optional[Any]], MagicMock], + mock_load_pickler: MonkeyPatch, + monkeypatch: MonkeyPatch, + log: pytest_structlog.StructuredLogCapture, +) -> None: + # Load example config with mock.pickle as pickle file + mock_load_config("mock.pickle") + + # Mock sys.argv to simulate command-line arguments + monkeypatch.setattr( + sys, "argv", ["main_script.py", "--dry-run", "--show-progress", "analyze_ceph"] + ) + + # Run main() + main() From e15ce2694fd5e21146dcbf7d41ae979c78fb1f0b Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Wed, 11 Sep 2024 09:40:04 +0000 Subject: [PATCH 07/27] feat: add cli option to list modules. Also add initial code to deal with progressper module Signed-off-by: Boekhorst --- src/rookify/__main__.py | 50 ++++++++++++++++++++++++++++++++++++----- tests/test_main.py | 33 ++++++++++++++++++++------- 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 981d802..9ddcfbb 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -11,13 +11,21 @@ from .logger import configure_logging, get_logger from .yaml import load_config from structlog.typing import BindableLogger +from pathlib import Path def parse_args(args: list[str]) -> argparse.Namespace: # Putting args-parser in seperate function to make this testable arg_parser = ArgumentParser("Rookify") + + # --dry-run option arg_parser.add_argument("--dry-run", action="store_true", dest="dry_run_mode") + # --list-modules option + arg_parser.add_argument( + "--list-modules", action="store_true", help="List all modules" + ) + # Custom ReadAction to set 'all' if nothing is specified for --read-pickle class ReadAction(argparse.Action): def __call__( @@ -69,6 +77,16 @@ def load_pickler(pickle_file_name: str) -> Any: return states_data +def get_all_modules() -> list[str]: + base_path = Path(__file__).resolve().parent + module_path = Path(base_path) / "modules" + module_names = [] + for item in module_path.iterdir(): + if item.is_dir() and item.name != "__pycache__": + module_names.append(item.name) + return module_names + + def sort_pickle_file(unsorted_states_data: Dict[str, Any]) -> Dict[str, Any]: # sort the pickle-file alfabetically iterable_dict = iter(unsorted_states_data) @@ -98,9 +116,34 @@ def read_pickle_file( ) +def show_progress_from_pickle_file( + args: argparse.Namespace, pickle_file_name: str, log: BindableLogger +) -> None: + # states_data = load_pickler(pickle_file_name) + modules = get_all_modules() + + # Check if a specific module should be targeted + # TODO: allow for multiple modules + if args.show_progress != "all": + module = args.show_progress + if args.show_progress not in modules: + log.error(f"The module {module} does not exist") + log.info( + 'Progress of : \n "{0}": {1}'.format(args.show_progress, "some....progess....") + ) + + def main() -> None: args = parse_args(sys.argv[1:]) + # Handle --list-modules + if args.list_modules: + modules = get_all_modules() + print("Available modules:\n") + for module in modules: + print(f"- {module}") + return + # Load configuration file try: config: Dict[str, Any] = load_config("config.yaml") @@ -143,12 +186,7 @@ def main() -> None: # If show_progress is run and there is a picklefile, show progress status based on picklefile contents # NOTE: this is always run in preflight-mode (migration shoudl not be executed) if args.show_progress is not None and pickle_file_name is not None: - # Check if a specific mode should be targeted - if args.show_progress != "all": - print("TEST") - print(args.show_progress) - - # TODO: implement module and method loading to let module check state itself + show_progress_from_pickle_file(args, pickle_file_name, log) return # TODO: should progress be checkable if no pickle file is present? Should rookify check the status by analyzing the source and traget machines state? diff --git a/tests/test_main.py b/tests/test_main.py index 3a0c5b7..1836325 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -22,32 +22,43 @@ # Test the arugment parser def test_parse_args_dry_run() -> None: args = parse_args(["--dry-run"]) - expected = Namespace(dry_run_mode=True, read_pickle=None, show_progress=None) + expected = Namespace( + dry_run_mode=True, list_modules=False, read_pickle=None, show_progress=None + ) assert args == expected def test_parse_args_read_pickle() -> None: args = parse_args(["--read-pickle"]) - expected = Namespace(dry_run_mode=False, read_pickle="all", show_progress=None) + expected = Namespace( + dry_run_mode=False, list_modules=False, read_pickle="all", show_progress=None + ) assert args == expected def test_parse_args_both_flags() -> None: args = parse_args(["--dry-run", "--read-pickle"]) - expected = Namespace(dry_run_mode=True, read_pickle="all", show_progress=None) + expected = Namespace( + dry_run_mode=True, list_modules=False, read_pickle="all", show_progress=None + ) assert args == expected def test_parse_args_show_progress() -> None: args = parse_args(["--show-progress"]) - expected = Namespace(dry_run_mode=False, read_pickle=None, show_progress="all") + expected = Namespace( + dry_run_mode=False, list_modules=False, read_pickle=None, show_progress="all" + ) assert args == expected def test_parse_args_show_progress_with_module() -> None: args = parse_args(["--show-progress", "ceph-analyze"]) expected = Namespace( - dry_run_mode=False, read_pickle=None, show_progress="ceph-analyze" + dry_run_mode=False, + list_modules=False, + read_pickle=None, + show_progress="ceph-analyze", ) assert args == expected @@ -55,19 +66,25 @@ def test_parse_args_show_progress_with_module() -> None: # check: should it be possible to add all arguments? def test_parse_args_both_dry_run_show_progress() -> None: args = parse_args(["--dry-run", "--read-pickle", "--show-progress"]) - expected = Namespace(dry_run_mode=True, read_pickle="all", show_progress="all") + expected = Namespace( + dry_run_mode=True, list_modules=False, read_pickle="all", show_progress="all" + ) assert args == expected def test_parse_args_all_dry_run_show_progress_read_pickle() -> None: args = parse_args(["--dry-run", "--show-progress"]) - expected = Namespace(dry_run_mode=True, read_pickle=None, show_progress="all") + expected = Namespace( + dry_run_mode=True, list_modules=False, read_pickle=None, show_progress="all" + ) assert args == expected def test_parse_args_no_flags() -> None: args = parse_args([]) - expected = Namespace(dry_run_mode=False, read_pickle=None, show_progress=None) + expected = Namespace( + dry_run_mode=False, list_modules=False, read_pickle=None, show_progress=None + ) assert args == expected From db931ccd1323571d2493f37549952a8a80428af4 Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Wed, 11 Sep 2024 10:02:19 +0000 Subject: [PATCH 08/27] fix: simplify argument tests by add fixture Signed-off-by: Boekhorst --- tests/test_main.py | 97 +++++++++++++--------------------------------- 1 file changed, 27 insertions(+), 70 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 1836325..0fb4ed9 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,12 +1,12 @@ import sys -from typing import Any, Callable, Optional +from typing import Any, Callable, Optional, List, Tuple import pytest from _pytest.monkeypatch import MonkeyPatch import yaml from unittest.mock import MagicMock import pytest_structlog -from argparse import Namespace +import argparse from rookify.__main__ import parse_args, main, sort_pickle_file import rookify.yaml @@ -20,75 +20,32 @@ # Test the arugment parser -def test_parse_args_dry_run() -> None: - args = parse_args(["--dry-run"]) - expected = Namespace( - dry_run_mode=True, list_modules=False, read_pickle=None, show_progress=None - ) - assert args == expected - - -def test_parse_args_read_pickle() -> None: - args = parse_args(["--read-pickle"]) - expected = Namespace( - dry_run_mode=False, list_modules=False, read_pickle="all", show_progress=None - ) - assert args == expected - - -def test_parse_args_both_flags() -> None: - args = parse_args(["--dry-run", "--read-pickle"]) - expected = Namespace( - dry_run_mode=True, list_modules=False, read_pickle="all", show_progress=None - ) - assert args == expected - - -def test_parse_args_show_progress() -> None: - args = parse_args(["--show-progress"]) - expected = Namespace( - dry_run_mode=False, list_modules=False, read_pickle=None, show_progress="all" - ) - assert args == expected - - -def test_parse_args_show_progress_with_module() -> None: - args = parse_args(["--show-progress", "ceph-analyze"]) - expected = Namespace( - dry_run_mode=False, - list_modules=False, - read_pickle=None, - show_progress="ceph-analyze", - ) - assert args == expected - - -# check: should it be possible to add all arguments? -def test_parse_args_both_dry_run_show_progress() -> None: - args = parse_args(["--dry-run", "--read-pickle", "--show-progress"]) - expected = Namespace( - dry_run_mode=True, list_modules=False, read_pickle="all", show_progress="all" - ) - assert args == expected - - -def test_parse_args_all_dry_run_show_progress_read_pickle() -> None: - args = parse_args(["--dry-run", "--show-progress"]) - expected = Namespace( - dry_run_mode=True, list_modules=False, read_pickle=None, show_progress="all" - ) - assert args == expected - - -def test_parse_args_no_flags() -> None: - args = parse_args([]) - expected = Namespace( - dry_run_mode=False, list_modules=False, read_pickle=None, show_progress=None - ) - assert args == expected - -# Test the --read-pickle and --show-progress options +TestCase = Tuple[List[str], argparse.Namespace] + +# fmt: off +test_cases: List[TestCase] = [ + (["--dry-run"], argparse.Namespace(dry_run_mode=True, list_modules=False, read_pickle=None, show_progress=None)), + (["--read-pickle"], argparse.Namespace(dry_run_mode=False, list_modules=False, read_pickle="all", show_progress=None)), + (["--show-progress"], argparse.Namespace(dry_run_mode=False, list_modules=False, read_pickle=None, show_progress="all")), + (["--show-progress", "ceph-analyze"], argparse.Namespace(dry_run_mode=False, list_modules=False, read_pickle=None, show_progress="ceph-analyze")), + (["--dry-run", "--read-pickle"], argparse.Namespace(dry_run_mode=True, list_modules=False, read_pickle="all", show_progress=None)), + (["--dry-run", "--show-progress"], argparse.Namespace(dry_run_mode=True, list_modules=False, read_pickle=None, show_progress="all")), + (["--dry-run", "--show-progress", "--read-pickle"], argparse.Namespace(dry_run_mode=True, list_modules=False, read_pickle="all", show_progress="all")), + ([], argparse.Namespace(dry_run_mode=False, list_modules=False, read_pickle=None, show_progress=None)), +] +# fmt: on + + +@pytest.mark.parametrize( + "args_list, expected_namespace", + test_cases, +) # type: ignore +def test_parse_args( + args_list: List[str], expected_namespace: argparse.Namespace +) -> None: + args = parse_args(args_list) + assert args == expected_namespace @pytest.fixture # type: ignore From 55dd5856250858e3ac02b776aefe5523f8a699e1 Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Thu, 12 Sep 2024 06:22:13 +0000 Subject: [PATCH 09/27] fix: using more parametrized fixtures to add more tests and simplify code Signed-off-by: Boekhorst --- src/rookify/__main__.py | 7 +++---- tests/test_main.py | 32 +++++++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 9ddcfbb..4f2024f 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -123,14 +123,13 @@ def show_progress_from_pickle_file( modules = get_all_modules() # Check if a specific module should be targeted - # TODO: allow for multiple modules if args.show_progress != "all": module = args.show_progress if args.show_progress not in modules: log.error(f"The module {module} does not exist") - log.info( - 'Progress of : \n "{0}": {1}'.format(args.show_progress, "some....progess....") - ) + log.info("Show progress of the {0} module".format(args.show_progress)) + else: + log.info("Show progress of {0} modules".format(args.show_progress)) def main() -> None: diff --git a/tests/test_main.py b/tests/test_main.py index 0fb4ed9..6ddcec1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -21,6 +21,7 @@ # Test the arugment parser +# Custom type for argument parser tests TestCase = Tuple[List[str], argparse.Namespace] # fmt: off @@ -200,19 +201,40 @@ def getUnsortedData() -> Any: return unsorted_states_data -def test_ceph_analyze_progress( +# Custom Type for Logger of CLI +TestCaseLogger = Tuple[List[str], str, str] + +# fmt: off +logger_test_cases: List[TestCaseLogger] = [ + (["--show-progress", "analyze_ceph"], "Show progress of the analyze_ceph module", "info"), + (["--dry-run", "--show-progress", "analyze_ceph"], "Show progress of the analyze_ceph module", "info"), + (["--show-progress"], "Show progress of all modules", "info"), + (["--dry-run", "--show-progress"], "Show progress of all modules", "info") +] +# fmt: on + + +@pytest.mark.parametrize( + "args_list, expected_log_message, expected_level", + logger_test_cases, +) # type: ignore +def test_show_progress( mock_load_config: Callable[[Optional[Any]], MagicMock], - mock_load_pickler: MonkeyPatch, monkeypatch: MonkeyPatch, log: pytest_structlog.StructuredLogCapture, + args_list: List[str], + expected_log_message: str, + expected_level: str, ) -> None: # Load example config with mock.pickle as pickle file mock_load_config("mock.pickle") # Mock sys.argv to simulate command-line arguments - monkeypatch.setattr( - sys, "argv", ["main_script.py", "--dry-run", "--show-progress", "analyze_ceph"] - ) + args_list.insert(0, "main_script.py") + monkeypatch.setattr(sys, "argv", args_list) # Run main() main() + + # Assertions + assert log.has(expected_log_message, level=expected_level) From b33cd08c3b29ae7597632c35d7537059bb7546a0 Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Thu, 12 Sep 2024 08:34:54 +0000 Subject: [PATCH 10/27] feat: add status command to ceph-analyze module Signed-off-by: Boekhorst --- src/rookify/modules/analyze_ceph/main.py | 57 +++++++++++++++++------- tests/test_main.py | 5 ++- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/rookify/modules/analyze_ceph/main.py b/src/rookify/modules/analyze_ceph/main.py index 6bc0d71..607c7cf 100644 --- a/src/rookify/modules/analyze_ceph/main.py +++ b/src/rookify/modules/analyze_ceph/main.py @@ -1,35 +1,62 @@ # -*- coding: utf-8 -*- -from typing import Any +from typing import Any, Optional from ..machine import Machine from ..module import ModuleHandler class AnalyzeCephHandler(ModuleHandler): - def status(self) -> Any: - # state = self.machine.get_preflight_state("AnalyzeCephHandler") - self.logger.info("AnalyzeCephHander has allready been run") + def _process_command( + self, state_data: Any, command: str, value: Optional[Any] = None + ) -> bool: + """Helper method to process commands by either setting or checking state data.""" + parts = command.split(" ") + current_level = state_data # the root of the data structure + + # Traverse the dictionary structure based on command parts + for idx, part in enumerate(parts): + if len(parts) == idx + 1: # Last part of the command + if value is not None: + current_level[part] = value + else: + return part in current_level + else: + if part not in current_level: + current_level[part] = {} + current_level = current_level[part] + + return True def preflight(self) -> Any: commands = ["mon dump", "osd dump", "device ls", "fs ls", "node ls"] - state = self.machine.get_preflight_state("AnalyzeCephHandler") state.data = {} + # Execute each command and store the result for command in commands: - parts = command.split(" ") - leaf = state.data - - for idx, part in enumerate(parts): - if len(parts) == idx + 1: - leaf[part] = self.ceph.mon_command(command) - else: - if part not in leaf: - leaf[part] = {} - leaf = leaf[part] + result = self.ceph.mon_command(command) + self._process_command(state.data, command, result) self.logger.info("AnalyzeCephHandler ran successfully.") + def status(self) -> Any: + commands = ["mon dump", "osd dump", "device ls", "fs ls", "node ls"] + state = self.machine.get_preflight_state("AnalyzeCephHandler") + + # Check if all expected commands have been run + all_commands_found = True + for command in commands: + if not self._process_command(state.data, command): + all_commands_found = False + break + + # Log the status + if all_commands_found: + self.logger.info("AnalyzeCephHandler has already been run.") + self.logger.info("Current state data: %s", state.data) + else: + self.logger.info("Progress: Not all commands have been run yet.") + @staticmethod def register_preflight_state( machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any diff --git a/tests/test_main.py b/tests/test_main.py index 6ddcec1..18350e3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -209,7 +209,8 @@ def getUnsortedData() -> Any: (["--show-progress", "analyze_ceph"], "Show progress of the analyze_ceph module", "info"), (["--dry-run", "--show-progress", "analyze_ceph"], "Show progress of the analyze_ceph module", "info"), (["--show-progress"], "Show progress of all modules", "info"), - (["--dry-run", "--show-progress"], "Show progress of all modules", "info") + (["--dry-run", "--show-progress"], "Show progress of all modules", "info"), + (["--show-progress"], "Analyze ceph has been run", "info") ] # fmt: on @@ -218,7 +219,7 @@ def getUnsortedData() -> Any: "args_list, expected_log_message, expected_level", logger_test_cases, ) # type: ignore -def test_show_progress( +def test_show_progress_logs( mock_load_config: Callable[[Optional[Any]], MagicMock], monkeypatch: MonkeyPatch, log: pytest_structlog.StructuredLogCapture, From bff688cbc2f2d0b70de3de5b7b9ced55e399fbce Mon Sep 17 00:00:00 2001 From: Boekhorst Date: Thu, 12 Sep 2024 14:10:35 +0000 Subject: [PATCH 11/27] fix: add status command through load modules, probably should add an own status-mode? Signed-off-by: Boekhorst --- src/rookify/__main__.py | 37 +++++++++++------------- src/rookify/modules/__init__.py | 19 ++++++++---- src/rookify/modules/analyze_ceph/main.py | 4 ++- src/rookify/modules/machine.py | 4 ++- src/rookify/modules/module.py | 29 +++++++++++++++++-- tests/test_main.py | 2 +- 6 files changed, 64 insertions(+), 31 deletions(-) diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 4f2024f..7f079d5 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -116,9 +116,9 @@ def read_pickle_file( ) -def show_progress_from_pickle_file( +def show_progress_from_state( args: argparse.Namespace, pickle_file_name: str, log: BindableLogger -) -> None: +) -> bool: # states_data = load_pickler(pickle_file_name) modules = get_all_modules() @@ -127,9 +127,12 @@ def show_progress_from_pickle_file( module = args.show_progress if args.show_progress not in modules: log.error(f"The module {module} does not exist") + return False log.info("Show progress of the {0} module".format(args.show_progress)) + return True else: log.info("Show progress of {0} modules".format(args.show_progress)) + return True def main() -> None: @@ -182,24 +185,18 @@ def main() -> None: ) return - # If show_progress is run and there is a picklefile, show progress status based on picklefile contents - # NOTE: this is always run in preflight-mode (migration shoudl not be executed) - if args.show_progress is not None and pickle_file_name is not None: - show_progress_from_pickle_file(args, pickle_file_name, log) - return - - # TODO: should progress be checkable if no pickle file is present? Should rookify check the status by analyzing the source and traget machines state? - elif args.show_progress is not None and pickle_file_name is None: - log.info( - "Currently rookify can only check the state of progress by analyzing the pickle file states." - ) - return - - # Else run the rook migration else: - log.debug("Executing Rookify") - machine = Machine(config["general"].get("machine_pickle_file")) - load_modules(machine, config) - machine.execute(dry_run_mode=args.dry_run_mode) + if args.show_progress is not None: + if show_progress_from_state(args, pickle_file_name, log) is True: + load_modules(machine, config, show_progress=True) + # NOTE: this is always run in preflight-mode (migration should not be executed) + machine.execute(dry_run_mode=True) + return + else: + return + else: + load_modules(machine, config) + log.debug("Executing Rookify") + machine.execute(dry_run_mode=args.dry_run_mode) diff --git a/src/rookify/modules/__init__.py b/src/rookify/modules/__init__.py index a5c4947..d959cd3 100644 --- a/src/rookify/modules/__init__.py +++ b/src/rookify/modules/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import importlib -from typing import Any, Dict +from typing import Any, Dict, Optional from ..logger import get_logger from .machine import Machine @@ -22,7 +22,12 @@ def __init__(self, module_name: str, message: str): self.message = message -def _load_module(machine: Machine, config: Dict[str, Any], module_name: str) -> None: +def _load_module( + machine: Machine, + config: Dict[str, Any], + module_name: str, + show_progress: Optional[bool] = False, +) -> None: """ Dynamically loads a module from the 'rookify.modules' package. @@ -43,12 +48,14 @@ def _load_module(machine: Machine, config: Dict[str, Any], module_name: str) -> additional_modules = module.ModuleHandler.REQUIRES for module_name in additional_modules: - _load_module(machine, config, module_name) + _load_module(machine, config, module_name, show_progress) - module.ModuleHandler.register_states(machine, config) + module.ModuleHandler.register_states(machine, config, show_progress) -def load_modules(machine: Machine, config: Dict[str, Any]) -> None: +def load_modules( + machine: Machine, config: Dict[str, Any], show_progress: Optional[bool] = False +) -> None: """ Dynamically loads modules from the 'modules' package. @@ -61,7 +68,7 @@ def load_modules(machine: Machine, config: Dict[str, Any]) -> None: for entry in importlib.resources.files("rookify.modules").iterdir(): if entry.is_dir() and entry.name in config["migration_modules"]: migration_modules.remove(entry.name) - _load_module(machine, config, entry.name) + _load_module(machine, config, entry.name, show_progress) if len(migration_modules) > 0 or len(config["migration_modules"]) < 1: logger = get_logger() diff --git a/src/rookify/modules/analyze_ceph/main.py b/src/rookify/modules/analyze_ceph/main.py index 607c7cf..293c9e3 100644 --- a/src/rookify/modules/analyze_ceph/main.py +++ b/src/rookify/modules/analyze_ceph/main.py @@ -55,7 +55,9 @@ def status(self) -> Any: self.logger.info("AnalyzeCephHandler has already been run.") self.logger.info("Current state data: %s", state.data) else: - self.logger.info("Progress: Not all commands have been run yet.") + self.logger.info( + "AnalyzeCephHandler Progress: Not all commands have been run yet." + ) @staticmethod def register_preflight_state( diff --git a/src/rookify/modules/machine.py b/src/rookify/modules/machine.py index 3a6bcf8..fbc53fd 100644 --- a/src/rookify/modules/machine.py +++ b/src/rookify/modules/machine.py @@ -26,7 +26,9 @@ def add_execution_state(self, name: str, **kwargs: Dict[str, Any]) -> None: def add_preflight_state(self, name: str, **kwargs: Dict[str, Any]) -> None: self._preflight_states.append(self.__class__.state_cls(name, **kwargs)) - def execute(self, dry_run_mode: bool = False) -> None: + def execute( + self, dry_run_mode: bool = False, show_progress: Optional[bool] = False + ) -> None: states = self._preflight_states if not dry_run_mode: states = states + self._execution_states diff --git a/src/rookify/modules/module.py b/src/rookify/modules/module.py index ec6ae8b..0818ce5 100644 --- a/src/rookify/modules/module.py +++ b/src/rookify/modules/module.py @@ -59,6 +59,13 @@ def ssh(self) -> SSH: self.__ssh = SSH(self._config["ssh"]) return self.__ssh + @abc.abstractmethod + def status(self) -> None: + """ + Run the modules status check + """ + pass + @abc.abstractmethod def preflight(self) -> None: """ @@ -87,7 +94,12 @@ def load_template(self, filename: str, **variables: Any) -> Template: return template @classmethod - def register_states(cls, machine: Machine, config: Dict[str, Any]) -> None: + def register_states( + cls, + machine: Machine, + config: Dict[str, Any], + show_progress: Optional[bool] = False, + ) -> None: """ Register states for transitions """ @@ -116,7 +128,10 @@ def register_states(cls, machine: Machine, config: Dict[str, Any]) -> None: ) else: if preflight_state_name is not None: - cls.register_preflight_state(machine, preflight_state_name, handler) + if show_progress is True: + cls.register_status_state(machine, preflight_state_name, handler) + else: + cls.register_preflight_state(machine, preflight_state_name, handler) if execution_state_name is not None: cls.register_execution_state(machine, execution_state_name, handler) @@ -131,6 +146,16 @@ def register_preflight_state( machine.add_preflight_state(state_name, on_enter=handler.preflight, **kwargs) + @staticmethod + def register_status_state( + machine: Machine, state_name: str, handler: Any, **kwargs: Any + ) -> None: + """ + Register state for transitions + """ + + machine.add_preflight_state(state_name, on_enter=handler.status, **kwargs) + @staticmethod def register_execution_state( machine: Machine, state_name: str, handler: Any, **kwargs: Any diff --git a/tests/test_main.py b/tests/test_main.py index 18350e3..b163ae4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -210,7 +210,7 @@ def getUnsortedData() -> Any: (["--dry-run", "--show-progress", "analyze_ceph"], "Show progress of the analyze_ceph module", "info"), (["--show-progress"], "Show progress of all modules", "info"), (["--dry-run", "--show-progress"], "Show progress of all modules", "info"), - (["--show-progress"], "Analyze ceph has been run", "info") + (["--show-progress"], "AnalyzeCephHandler Progress: Not all commands have been run yet.", "info") ] # fmt: on From 3a7f91de8e6a331cf2ca6667dde53e9a5c2719bc Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Sat, 14 Sep 2024 11:54:33 +0200 Subject: [PATCH 12/27] Integrate presentation of the current state into `ModuleHandler`. This partly revers commit 3e9d54decb01750197ff5d146a9b9387a52abf24. This reverts commit e15ce2694fd5e21146dcbf7d41ae979c78fb1f0b. This reverts commit bff688cbc2f2d0b70de3de5b7b9ced55e399fbce. Signed-off-by: Tobias Wolf --- src/rookify/__main__.py | 189 +++++---------------------- src/rookify/modules/__init__.py | 31 +++-- src/rookify/modules/machine.py | 56 +++++--- src/rookify/modules/module.py | 44 +++++-- tests/test_main.py | 221 ++------------------------------ 5 files changed, 128 insertions(+), 413 deletions(-) diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index 7f079d5..b79f284 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -1,73 +1,15 @@ # -*- coding: utf-8 -*- -import json from pickle import Unpickler import sys import argparse from argparse import ArgumentParser -from typing import Any, Dict, Optional +from typing import Any, Dict from .modules import load_modules from .modules.machine import Machine +from .modules.module import ModuleHandler from .logger import configure_logging, get_logger from .yaml import load_config -from structlog.typing import BindableLogger -from pathlib import Path - - -def parse_args(args: list[str]) -> argparse.Namespace: - # Putting args-parser in seperate function to make this testable - arg_parser = ArgumentParser("Rookify") - - # --dry-run option - arg_parser.add_argument("--dry-run", action="store_true", dest="dry_run_mode") - - # --list-modules option - arg_parser.add_argument( - "--list-modules", action="store_true", help="List all modules" - ) - - # Custom ReadAction to set 'all' if nothing is specified for --read-pickle - class ReadAction(argparse.Action): - def __call__( - self, - parser: ArgumentParser, - namespace: argparse.Namespace, - values: Optional[Any], - option_string: Optional[str] = None, - ) -> None: - setattr(namespace, self.dest, values if values is not None else "all") - - # Custom ShowProgressAction to set 'all' if nothing is specified for --show-progress - class ShowProgressAction(argparse.Action): - def __call__( - self, - parser: ArgumentParser, - namespace: argparse.Namespace, - values: Optional[Any], - option_string: Optional[str] = None, - ) -> None: - setattr(namespace, self.dest, values if values is not None else "all") - - arg_parser.add_argument( - "--read-pickle", - nargs="?", - action=ReadAction, - dest="read_pickle", - metavar="
", - help="Show the content of the pickle file. Default argument is 'all', you can also specify a section you want to look at.", - required=False, - ) - - arg_parser.add_argument( - "--show-progress", - nargs="?", - action=ShowProgressAction, - dest="show_progress", - metavar="", - help="Show progress of the modules. Default argument is 'all', you can also specify a module you want to get the progress status from.", - required=False, - ) - return arg_parser.parse_args(args) def load_pickler(pickle_file_name: str) -> Any: @@ -77,75 +19,26 @@ def load_pickler(pickle_file_name: str) -> Any: return states_data -def get_all_modules() -> list[str]: - base_path = Path(__file__).resolve().parent - module_path = Path(base_path) / "modules" - module_names = [] - for item in module_path.iterdir(): - if item.is_dir() and item.name != "__pycache__": - module_names.append(item.name) - return module_names - - -def sort_pickle_file(unsorted_states_data: Dict[str, Any]) -> Dict[str, Any]: - # sort the pickle-file alfabetically - iterable_dict = iter(unsorted_states_data) - first_key = next(iterable_dict) - data_values = unsorted_states_data[first_key]["data"] - sorted_data_by_keys = {k: data_values[k] for k in sorted(data_values)} - return sorted_data_by_keys - - -def read_pickle_file( - args: argparse.Namespace, pickle_file_name: str, log: BindableLogger -) -> None: - states_data = load_pickler(pickle_file_name) - sorted_states_data = sort_pickle_file(states_data) +def parse_args(args: list[str]) -> argparse.Namespace: + # Putting args-parser in seperate function to make this testable + arg_parser = ArgumentParser("Rookify") - # Check if a specific section should be listed - if args.read_pickle != "all": - if args.read_pickle not in sorted_states_data.keys(): - log.error(f"The section {args.read_pickle} does not exist") - else: - sorted_states_data = sorted_states_data[args.read_pickle] + # --dry-run option + arg_parser.add_argument("--dry-run", action="store_true", dest="dry_run_mode") - log.info( - 'Current state as retrieved from pickle-file: \n "{0}": {1}'.format( - args.read_pickle, json.dumps(sorted_states_data, indent=4) - ) + arg_parser.add_argument( + "--show-states", + action="store_true", + dest="show_states", + help="Show states of the modules.", ) - -def show_progress_from_state( - args: argparse.Namespace, pickle_file_name: str, log: BindableLogger -) -> bool: - # states_data = load_pickler(pickle_file_name) - modules = get_all_modules() - - # Check if a specific module should be targeted - if args.show_progress != "all": - module = args.show_progress - if args.show_progress not in modules: - log.error(f"The module {module} does not exist") - return False - log.info("Show progress of the {0} module".format(args.show_progress)) - return True - else: - log.info("Show progress of {0} modules".format(args.show_progress)) - return True + return arg_parser.parse_args(args) def main() -> None: args = parse_args(sys.argv[1:]) - # Handle --list-modules - if args.list_modules: - modules = get_all_modules() - print("Available modules:\n") - for module in modules: - print(f"- {module}") - return - # Load configuration file try: config: Dict[str, Any] = load_config("config.yaml") @@ -154,49 +47,29 @@ def main() -> None: # Configure logging try: - configure_logging(config["logging"]) + if args.show_states is True: + configure_logging( + {"level": "ERROR", "format": {"renderer": "console", "time": "iso"}} + ) + else: + configure_logging(config["logging"]) except Exception as e: raise SystemExit(f"Error configuring logging: {e}") + # Get Logger log = get_logger() - # Get Pickle File if configured in config.yaml - pickle_file_name = config["general"].get("machine_pickle_file") - if pickle_file_name is None: - log.info("No pickle file was set in the configuration.") - else: - log.info(f"Pickle file set: {pickle_file_name}") + log.info("Executing Rookify ...") - # Get Pickle File if configured in config.yaml - pickle_file_name = config["general"].get("machine_pickle_file") - if pickle_file_name is None: - log.info("No pickle file was set in the configuration.") - else: - log.info(f"Pickle file set: {pickle_file_name}") - - # If read_pickle is run and there is a picklefile, show the picklefiles contents. - # NOTE: preflight mode (--dry-run) has no effect here, because no module actions are required. - if args.read_pickle is not None and pickle_file_name is not None: - read_pickle_file(args, pickle_file_name, log) - return - elif args.read_pickle is not None and pickle_file_name is None: - log.info( - "No pickle file configured to read from. Check if the pickle file exists and is configured in config.yaml" - ) - return + machine = Machine(config["general"].get("machine_pickle_file")) + + load_modules(machine, config) + if args.show_states is True: + ModuleHandler.show_states(machine, config) else: - machine = Machine(config["general"].get("machine_pickle_file")) - - if args.show_progress is not None: - if show_progress_from_state(args, pickle_file_name, log) is True: - load_modules(machine, config, show_progress=True) - # NOTE: this is always run in preflight-mode (migration should not be executed) - machine.execute(dry_run_mode=True) - return - else: - return - else: - load_modules(machine, config) - log.debug("Executing Rookify") - machine.execute(dry_run_mode=args.dry_run_mode) + machine.execute(dry_run_mode=args.dry_run_mode) + + +if __name__ == "__main__": + main() diff --git a/src/rookify/modules/__init__.py b/src/rookify/modules/__init__.py index d959cd3..ebd1c17 100644 --- a/src/rookify/modules/__init__.py +++ b/src/rookify/modules/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import importlib -from typing import Any, Dict, Optional +from typing import Any, Dict, List from ..logger import get_logger from .machine import Machine @@ -22,11 +22,18 @@ def __init__(self, module_name: str, message: str): self.message = message +_modules_loaded: List[Any] = [] + + +def get_modules() -> List[Any]: + global _modules_loaded + return _modules_loaded.copy() + + def _load_module( machine: Machine, config: Dict[str, Any], module_name: str, - show_progress: Optional[bool] = False, ) -> None: """ Dynamically loads a module from the 'rookify.modules' package. @@ -35,8 +42,10 @@ def _load_module( :return: returns tuple of preflight_modules, modules """ + global _modules_loaded + module = importlib.import_module("rookify.modules.{0}".format(module_name)) - additional_modules = [] + additional_module_names = [] if not hasattr(module, "ModuleHandler") or not callable( getattr(module.ModuleHandler, "register_states") @@ -45,17 +54,17 @@ def _load_module( if hasattr(module.ModuleHandler, "REQUIRES"): assert isinstance(module.ModuleHandler.REQUIRES, list) - additional_modules = module.ModuleHandler.REQUIRES + additional_module_names = module.ModuleHandler.REQUIRES - for module_name in additional_modules: - _load_module(machine, config, module_name, show_progress) + for additional_module_name in additional_module_names: + _load_module(machine, config, additional_module_name) - module.ModuleHandler.register_states(machine, config, show_progress) + if module not in _modules_loaded: + _modules_loaded.append(module) + module.ModuleHandler.register_states(machine, config) -def load_modules( - machine: Machine, config: Dict[str, Any], show_progress: Optional[bool] = False -) -> None: +def load_modules(machine: Machine, config: Dict[str, Any]) -> None: """ Dynamically loads modules from the 'modules' package. @@ -68,7 +77,7 @@ def load_modules( for entry in importlib.resources.files("rookify.modules").iterdir(): if entry.is_dir() and entry.name in config["migration_modules"]: migration_modules.remove(entry.name) - _load_module(machine, config, entry.name, show_progress) + _load_module(machine, config, entry.name) if len(migration_modules) > 0 or len(config["migration_modules"]) < 1: logger = get_logger() diff --git a/src/rookify/modules/machine.py b/src/rookify/modules/machine.py index c3877d7..2f42a32 100644 --- a/src/rookify/modules/machine.py +++ b/src/rookify/modules/machine.py @@ -26,22 +26,14 @@ def add_execution_state(self, name: str, **kwargs: Any) -> None: def add_preflight_state(self, name: str, **kwargs: Any) -> None: self._preflight_states.append(self.__class__.state_cls(name, **kwargs)) - def execute( - self, dry_run_mode: bool = False, show_progress: Optional[bool] = False - ) -> None: + def execute(self, dry_run_mode: bool = False) -> None: states = self._preflight_states if not dry_run_mode: states = states + self._execution_states - logger = get_logger() + self._register_states(states) - for state in states: - if state.name not in self.states: - logger.debug("Registering state '{0}'".format(state.name)) - self.add_state(state) - - self.add_state("migrated") - self.add_ordered_transitions(loop=False) + logger = get_logger() if self._machine_pickle_file is None: logger.info("Execution started without machine pickle file") @@ -54,12 +46,8 @@ def execute( def _execute(self, pickle_file: Optional[IO[Any]] = None) -> None: states_data = {} - # Read pickle file if it exists, to continue from the stored state - if pickle_file is not None and pickle_file.tell() > 0: - pickle_file.seek(0) - - states_data = Unpickler(pickle_file).load() - self._restore_state_data(states_data) + if pickle_file is not None: + self._restore_state_data(pickle_file) try: while True: @@ -110,7 +98,39 @@ def get_preflight_state_data( ) -> Any: return getattr(self.get_preflight_state(name), tag, default_value) - def _restore_state_data(self, data: Dict[str, Any]) -> None: + def register_states(self) -> None: + self._register_states(self._preflight_states + self._execution_states) + + if self._machine_pickle_file is not None: + with open(self._machine_pickle_file, "rb") as file: + file.seek(1) + self._restore_state_data(file) + + def _register_states(self, states: List[State]) -> None: + logger = get_logger() + + for state in states: + if state.name not in self.states: + logger.debug("Registering state '{0}'".format(state.name)) + self.add_state(state) + + self.add_state("migrated") + self.add_ordered_transitions(loop=False) + + """ +Read pickle file if it exists, to continue from the stored state. It is +required that the position of the pointer of the pickle file given is not +at the start. + """ + + def _restore_state_data(self, pickle_file: IO[Any]) -> None: + if pickle_file.tell() == 0: + return None + + pickle_file.seek(0) + + data = Unpickler(pickle_file).load() + for state_name in data: try: state = self.get_state(state_name) diff --git a/src/rookify/modules/module.py b/src/rookify/modules/module.py index ead1e4c..967c0bc 100644 --- a/src/rookify/modules/module.py +++ b/src/rookify/modules/module.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- +import abc +import json import os import structlog from typing import Any, Dict, Optional from ..logger import get_logger +from . import get_modules from .ceph import Ceph from .k8s import K8s from .machine import Machine @@ -58,6 +61,17 @@ def ssh(self) -> SSH: self._ssh = SSH(self._config["ssh"]) return self._ssh + def _get_readable_json_dump(self, data: Any) -> Any: + return json.dumps(data, sort_keys=True, indent="\t") + + def get_readable_key_value_state(self) -> Optional[Dict[str, str]]: + """ + Run the modules status check + """ + + return None + + @abc.abstractmethod def preflight(self) -> None: """ Run the modules preflight check @@ -118,10 +132,7 @@ def register_states( ) else: if preflight_state_name is not None: - if show_progress is True: - cls.register_status_state(machine, preflight_state_name, handler) - else: - cls.register_preflight_state(machine, preflight_state_name, handler) + cls.register_preflight_state(machine, preflight_state_name, handler) if execution_state_name is not None: cls.register_execution_state(machine, execution_state_name, handler) @@ -137,21 +148,28 @@ def register_preflight_state( machine.add_preflight_state(state_name, on_enter=handler.preflight, **kwargs) @staticmethod - def register_status_state( + def register_execution_state( machine: Machine, state_name: str, handler: Any, **kwargs: Any ) -> None: """ Register state for transitions """ - machine.add_preflight_state(state_name, on_enter=handler.status, **kwargs) + machine.add_execution_state(state_name, on_enter=handler.execute, **kwargs) @staticmethod - def register_execution_state( - machine: Machine, state_name: str, handler: Any, **kwargs: Any - ) -> None: - """ - Register state for transitions - """ + def show_states(machine: Machine, config: Dict[str, Any]) -> None: + machine.register_states() + modules = get_modules() - machine.add_execution_state(state_name, on_enter=handler.execute, **kwargs) + for module in modules: + module_handler = module.ModuleHandler(machine, config) + + if hasattr(module_handler, "get_readable_key_value_state"): + state_data = module_handler.get_readable_key_value_state() + + if state_data is None: + continue + + for state_key, state_value in state_data.items(): + print("{0}: {1}".format(state_key, state_value)) diff --git a/tests/test_main.py b/tests/test_main.py index b163ae4..6504e33 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,15 +1,7 @@ -import sys -from typing import Any, Callable, Optional, List, Tuple -import pytest -from _pytest.monkeypatch import MonkeyPatch -import yaml -from unittest.mock import MagicMock -import pytest_structlog - import argparse - -from rookify.__main__ import parse_args, main, sort_pickle_file -import rookify.yaml +import pytest +from rookify.__main__ import parse_args +from typing import List, Tuple # # These tests test the src/rookify/__main__.py @@ -18,22 +10,17 @@ # Run one test with output: .venv/bin/python -m pytest tests/ -k name_of_test -s # - -# Test the arugment parser +# Test the argument parser # Custom type for argument parser tests TestCase = Tuple[List[str], argparse.Namespace] # fmt: off test_cases: List[TestCase] = [ - (["--dry-run"], argparse.Namespace(dry_run_mode=True, list_modules=False, read_pickle=None, show_progress=None)), - (["--read-pickle"], argparse.Namespace(dry_run_mode=False, list_modules=False, read_pickle="all", show_progress=None)), - (["--show-progress"], argparse.Namespace(dry_run_mode=False, list_modules=False, read_pickle=None, show_progress="all")), - (["--show-progress", "ceph-analyze"], argparse.Namespace(dry_run_mode=False, list_modules=False, read_pickle=None, show_progress="ceph-analyze")), - (["--dry-run", "--read-pickle"], argparse.Namespace(dry_run_mode=True, list_modules=False, read_pickle="all", show_progress=None)), - (["--dry-run", "--show-progress"], argparse.Namespace(dry_run_mode=True, list_modules=False, read_pickle=None, show_progress="all")), - (["--dry-run", "--show-progress", "--read-pickle"], argparse.Namespace(dry_run_mode=True, list_modules=False, read_pickle="all", show_progress="all")), - ([], argparse.Namespace(dry_run_mode=False, list_modules=False, read_pickle=None, show_progress=None)), + (["--dry-run"], argparse.Namespace(dry_run_mode=True, show_states=False)), + (["--show-states"], argparse.Namespace(dry_run_mode=False, show_states=True)), + (["--dry-run", "--show-states"], argparse.Namespace(dry_run_mode=True, show_states=True)), + ([], argparse.Namespace(dry_run_mode=False, show_states=False)), ] # fmt: on @@ -47,195 +34,3 @@ def test_parse_args( ) -> None: args = parse_args(args_list) assert args == expected_namespace - - -@pytest.fixture # type: ignore -def mock_load_config(monkeypatch: MonkeyPatch) -> Callable[[Optional[Any]], MagicMock]: - def _mock_load_config(pickle_file: Optional[Any] = None) -> MagicMock: - # Mock the configuration data - # Load config.example.yaml - with open("config.example.yaml", "r") as file: - config_data = yaml.safe_load(file) - - config_data["general"]["machine_pickle_file"] = pickle_file - - # Mock load_config function - mock = MagicMock(return_value=config_data) - monkeypatch.setattr(rookify.__main__, "load_config", mock) - - return mock - - return _mock_load_config - - -@pytest.fixture # type: ignore -def mock_load_pickler(monkeypatch: MonkeyPatch) -> MagicMock: - # Mock states_data from pickle file - mock_states_data = {"teststuff": {"data": {"mock_key": "mock_value"}}} - - # Mock load_pickler function - mock = MagicMock(return_value=mock_states_data) - monkeypatch.setattr(rookify.__main__, "load_pickler", mock) - - return mock - - -def test_main_read_pickle( - mock_load_config: Callable[[Optional[Any]], MagicMock], - mock_load_pickler: MonkeyPatch, - monkeypatch: MonkeyPatch, - log: pytest_structlog.StructuredLogCapture, -) -> None: - # Load example config with mock.pickle as pickle file - mock_load_config("mock.pickle") - - # Mock sys.argv to simulate command-line arguments - monkeypatch.setattr(sys, "argv", ["main_script.py", "--read-pickle", "--dry-run"]) - - # Run main() - main() - - # Verify logging messages - expected_log_message = ( - 'Current state as retrieved from pickle-file: \n "all": ' - "{\n" - ' "mock_key": "mock_value"\n' - "}" - ) - assert log.has("Pickle file set: mock.pickle", level="info") - assert log.has(expected_log_message, level="info") - - -def test_main_no_pickle_file( - mock_load_config: Callable[[Optional[str]], MagicMock], - mock_load_pickler: MonkeyPatch, - monkeypatch: MonkeyPatch, - log: pytest_structlog.StructuredLogCapture, -) -> None: - # Load a configuration without pickle: This should load the default data.pickle file if it is available - mock_load_config(None) - - # Mock sys.argv to simulate command-line arguments - monkeypatch.setattr(sys, "argv", ["main_script.py", "--read-pickle", "--dry-run"]) - - # Run main() - main() - - # Assertions - assert log.has("No pickle file was set in the configuration.") - - -def test_sort_pickle_file() -> None: - # Prepare unsorted data - unsorted_states_data = getUnsortedData() - - # Expected keys order - expected_order = ["device", "fs", "mon", "node", "osd", "ssh"] - - # Run sort_pickle_file - sorted_states_data = sort_pickle_file(unsorted_states_data) - - # Assert that order is correct - sorted_states_data_keys = list(sorted_states_data.keys()) - - assert expected_order == sorted_states_data_keys - - -def getUnsortedData() -> Any: - unsorted_states_data = { - "preflighttestdata": { - "data": { - "mon": { - "dump": {"epoch": 1}, - "mons": [{"rank": 0}], - "quorum": [0, 1, 2], - }, - "fs": {"dump": {"epoch": 1}}, - "device": {"ls": ["devid", 1]}, - "osd": { - "dump": [{"epoch": 138}], - "osds": [{"osd": 0}], - "osd_xinfo": [{"osd": 0}], - }, - "node": { - "ls": { - "mon": { - "test-node-0": ["test-node-0"], - "test-node-1": ["test-node-1"], - "test-node-2": ["test-node-2"], - }, - "osd": { - "test-node-0": [2, 3], - "test-node-1": [0, 5], - "test-node-2": [1, 4], - }, - "mds": { - "test-node-0": ["test-node-0"], - "test-node-1": ["test-node-1"], - "test-node-2": ["test-node-2"], - }, - "mgr": { - "test-node-0": ["test-node-0"], - "test-node-1": ["test-node-1"], - "test-node-2": ["test-node-2"], - }, - } - }, - "ssh": { - "osd": { - "test-node-0": { - "devices": ["/dev/ceph-bla-1", "/dev/ceph-bla-2"] - }, - "test-node-1": { - "devices": ["/dev/ceph-bla-3", "/dev/ceph-bla-4"] - }, - "test-node-2": { - "devices": ["/dev/ceph-bla-5", "/dev/ceph-bla-6"] - }, - } - }, - } - } - } - - return unsorted_states_data - - -# Custom Type for Logger of CLI -TestCaseLogger = Tuple[List[str], str, str] - -# fmt: off -logger_test_cases: List[TestCaseLogger] = [ - (["--show-progress", "analyze_ceph"], "Show progress of the analyze_ceph module", "info"), - (["--dry-run", "--show-progress", "analyze_ceph"], "Show progress of the analyze_ceph module", "info"), - (["--show-progress"], "Show progress of all modules", "info"), - (["--dry-run", "--show-progress"], "Show progress of all modules", "info"), - (["--show-progress"], "AnalyzeCephHandler Progress: Not all commands have been run yet.", "info") -] -# fmt: on - - -@pytest.mark.parametrize( - "args_list, expected_log_message, expected_level", - logger_test_cases, -) # type: ignore -def test_show_progress_logs( - mock_load_config: Callable[[Optional[Any]], MagicMock], - monkeypatch: MonkeyPatch, - log: pytest_structlog.StructuredLogCapture, - args_list: List[str], - expected_log_message: str, - expected_level: str, -) -> None: - # Load example config with mock.pickle as pickle file - mock_load_config("mock.pickle") - - # Mock sys.argv to simulate command-line arguments - args_list.insert(0, "main_script.py") - monkeypatch.setattr(sys, "argv", args_list) - - # Run main() - main() - - # Assertions - assert log.has(expected_log_message, level=expected_level) From d89d12cfd526c0f71ed1f852e03e2b73563b042d Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Sat, 14 Sep 2024 11:59:25 +0200 Subject: [PATCH 13/27] Return representive state data for module `analyze_ceph` This reverts commit b33cd08c3b29ae7597632c35d7537059bb7546a0. Signed-off-by: Tobias Wolf --- src/rookify/modules/analyze_ceph/main.py | 39 +++++++++++++++--------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/rookify/modules/analyze_ceph/main.py b/src/rookify/modules/analyze_ceph/main.py index 293c9e3..25fc206 100644 --- a/src/rookify/modules/analyze_ceph/main.py +++ b/src/rookify/modules/analyze_ceph/main.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -from typing import Any, Optional +from collections import OrderedDict +from typing import Any, Dict, Optional from ..machine import Machine from ..module import ModuleHandler @@ -39,26 +40,34 @@ def preflight(self) -> Any: self.logger.info("AnalyzeCephHandler ran successfully.") - def status(self) -> Any: - commands = ["mon dump", "osd dump", "device ls", "fs ls", "node ls"] + def get_readable_key_value_state(self) -> Dict[str, str]: state = self.machine.get_preflight_state("AnalyzeCephHandler") - # Check if all expected commands have been run - all_commands_found = True - for command in commands: - if not self._process_command(state.data, command): - all_commands_found = False - break + kv_state_data = OrderedDict() + + if "mon" not in state.data or "dump" not in state.data["mon"]: + kv_state_data["ceph mon dump"] = "Not analyzed yet" + else: + kv_state_data["ceph mon dump"] = self._get_readable_json_dump( + state.data["mon"]["dump"] + ) - # Log the status - if all_commands_found: - self.logger.info("AnalyzeCephHandler has already been run.") - self.logger.info("Current state data: %s", state.data) + if "osd" not in state.data or "dump" not in state.data["osd"]: + kv_state_data["ceph osd dump"] = "Not analyzed yet" else: - self.logger.info( - "AnalyzeCephHandler Progress: Not all commands have been run yet." + kv_state_data["ceph osd dump"] = self._get_readable_json_dump( + state.data["osd"]["dump"] ) + if "device" not in state.data or "ls" not in state.data["device"]: + kv_state_data["OSD devices"] = "Not analyzed yet" + else: + kv_state_data["OSD devices"] = self._get_readable_json_dump( + state.data["device"]["ls"] + ) + + return kv_state_data + @staticmethod def register_preflight_state( machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any From 355e023a90d5cecc5b15c1d25c6be2cd2b6546e4 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 17 Sep 2024 20:25:15 +0200 Subject: [PATCH 14/27] Return representive state data for module `cephx_auth_config` Signed-off-by: Tobias Wolf --- src/rookify/modules/cephx_auth_config/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/rookify/modules/cephx_auth_config/main.py b/src/rookify/modules/cephx_auth_config/main.py index 98a50a4..fc7f4fe 100644 --- a/src/rookify/modules/cephx_auth_config/main.py +++ b/src/rookify/modules/cephx_auth_config/main.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from typing import Any +from typing import Any, Dict from ..exception import ModuleException from ..machine import Machine from ..module import ModuleHandler @@ -30,6 +30,12 @@ def preflight(self) -> Any: def is_cephx_set(self, values: str) -> Any: return "cephx" in [value.strip() for value in values.split(",")] + def get_readable_key_value_state(self) -> Dict[str, str]: + is_verified = self.machine.get_preflight_state_data( + "CephXAuthHandler", "verified", default_value=False + ) + return {"cephx auth is verified": str(is_verified)} + @staticmethod def register_preflight_state( machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any From f05e8f05bec391a5434a10ddb296534f11f39cdf Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 17 Sep 2024 20:26:13 +0200 Subject: [PATCH 15/27] Return representive state data for module `create_rook_resources` Signed-off-by: Tobias Wolf --- .../modules/create_rook_resources/main.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/rookify/modules/create_rook_resources/main.py b/src/rookify/modules/create_rook_resources/main.py index 48168b8..5ab018b 100644 --- a/src/rookify/modules/create_rook_resources/main.py +++ b/src/rookify/modules/create_rook_resources/main.py @@ -2,6 +2,7 @@ import kubernetes import json +from collections import OrderedDict from typing import Any, Dict from ..exception import ModuleException from ..machine import Machine @@ -135,6 +136,33 @@ def execute(self) -> None: "CreateRookResourcesHandler" ).secret = secret.to_dict() + def get_readable_key_value_state(self) -> Dict[str, str]: + kv_state_data = OrderedDict() + + configmap = self.machine.get_preflight_state_data( + "CreateRookResourcesHandler", "configmap" + ) + + if configmap is None: + kv_state_data["rook-ceph-mon-endpoints"] = "Not created yet" + else: + kv_state_data["rook-ceph-mon-endpoints"] = self._get_readable_json_dump( + configmap + ) + + secret = self.machine.get_execution_state_data( + "CreateRookResourcesHandler", "secret" + ) + + if secret is None: + kv_state_data["rook-ceph-mon-endpoints has been created"] = "False" + kv_state_data["rook-ceph-mon"] = "Not created yet" + else: + kv_state_data["rook-ceph-mon-endpoints has been created"] = "True" + kv_state_data["rook-ceph-mon"] = self._get_readable_json_dump(secret) + + return kv_state_data + @staticmethod def register_execution_state( machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any From cc35e40af9865e2a06d4f08d063d0ee55110e5e8 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 17 Sep 2024 20:26:35 +0200 Subject: [PATCH 16/27] Return representive state data for module `create_rook_cluster` Signed-off-by: Tobias Wolf --- .../modules/create_rook_cluster/main.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/rookify/modules/create_rook_cluster/main.py b/src/rookify/modules/create_rook_cluster/main.py index 28ce87a..8d932cd 100644 --- a/src/rookify/modules/create_rook_cluster/main.py +++ b/src/rookify/modules/create_rook_cluster/main.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -from typing import Any +from collections import OrderedDict +from typing import Any, Dict from ..exception import ModuleException from ..machine import Machine from ..module import ModuleHandler @@ -157,6 +158,28 @@ def _watch_cluster_phase_callback(self, event_object: Any) -> Any: return None + def get_readable_key_value_state(self) -> Dict[str, str]: + kv_state_data = OrderedDict() + + cluster_definition = self.machine.get_preflight_state_data( + "CreateRookClusterHandler", "cluster_definition" + ) + cluster_name = self._config["rook"]["cluster"]["name"] + + if cluster_definition is None: + kv_state_data[cluster_name] = "Not created yet" + else: + kv_state_data[cluster_name] = self._get_readable_json_dump( + cluster_definition + ) + + is_generated = self.machine.get_execution_state_data( + "CreateRookClusterHandler", "generated", default_value=False + ) + kv_state_data["{0} is generated".format(cluster_name)] = str(is_generated) + + return kv_state_data + @staticmethod def register_execution_state( machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any From ac40e55d2fcec0f0feda7879990ab72112ef4952 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 17 Sep 2024 20:26:58 +0200 Subject: [PATCH 17/27] Return representive state data for module `migrate_mons` Signed-off-by: Tobias Wolf --- src/rookify/modules/migrate_mons/main.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/rookify/modules/migrate_mons/main.py b/src/rookify/modules/migrate_mons/main.py index 596ddb8..8d8df4e 100644 --- a/src/rookify/modules/migrate_mons/main.py +++ b/src/rookify/modules/migrate_mons/main.py @@ -32,6 +32,15 @@ def execute(self) -> None: for mon in state_data["mon"]["dump"]["mons"]: self._migrate_mon(mon) + def get_readable_key_value_state(self) -> Dict[str, str]: + state_data = self.machine.get_preflight_state("AnalyzeCephHandler").data + + return { + "ceph mon daemons": self._get_readable_json_dump( + state_data["mon"]["dump"]["mons"] + ) + } + def _migrate_mon(self, mon: Dict[str, Any]) -> None: migrated_mons = self.machine.get_execution_state_data( "MigrateMonsHandler", "migrated_mons", default_value=[] From ae5399c6efdb760a25d59bc6bda6859d71b6efa5 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 17 Sep 2024 20:27:16 +0200 Subject: [PATCH 18/27] Return representive state data for module `migrate_mgrs` Signed-off-by: Tobias Wolf --- src/rookify/modules/migrate_mgrs/main.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/rookify/modules/migrate_mgrs/main.py b/src/rookify/modules/migrate_mgrs/main.py index 29be0f7..381b77b 100644 --- a/src/rookify/modules/migrate_mgrs/main.py +++ b/src/rookify/modules/migrate_mgrs/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from time import sleep -from typing import Any +from typing import Any, Dict from ..exception import ModuleException from ..machine import Machine from ..module import ModuleHandler @@ -22,9 +22,18 @@ def preflight(self) -> None: def execute(self) -> None: state_data = self.machine.get_preflight_state("AnalyzeCephHandler").data - for node, _ in state_data["node"]["ls"]["mgr"].items(): + for node in state_data["node"]["ls"]["mgr"].keys(): self._migrate_mgr(node) + def get_readable_key_value_state(self) -> Dict[str, str]: + state_data = self.machine.get_preflight_state("AnalyzeCephHandler").data + + return { + "ceph mgr daemons": self._get_readable_json_dump( + list(state_data["node"]["ls"]["mgr"].keys()) + ) + } + def _migrate_mgr(self, mgr_host: str) -> None: migrated_mgrs = self.machine.get_execution_state_data( "MigrateMgrsHandler", "migrated_mgrs", default_value=[] From 074ac03f190f3b911353cf14944b53e30d1d09d9 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 17 Sep 2024 20:27:31 +0200 Subject: [PATCH 19/27] Return representive state data for module `migrate_osds` Signed-off-by: Tobias Wolf --- src/rookify/modules/migrate_osds/main.py | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/rookify/modules/migrate_osds/main.py b/src/rookify/modules/migrate_osds/main.py index a14792a..acebdf5 100644 --- a/src/rookify/modules/migrate_osds/main.py +++ b/src/rookify/modules/migrate_osds/main.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from collections import OrderedDict from time import sleep from typing import Any, Dict, List from ..exception import ModuleException @@ -84,7 +85,7 @@ def preflight(self) -> None: def execute(self) -> None: osd_host_devices = self.machine.get_preflight_state_data( - "MigrateOSDsHandler", "osd_host_devices", default_value=[] + "MigrateOSDsHandler", "osd_host_devices", default_value={} ) state_data = self.machine.get_preflight_state("AnalyzeCephHandler").data @@ -92,6 +93,34 @@ def execute(self) -> None: for host in osd_host_devices.keys(): self.migrate_osds(host, state_data["node"]["ls"]["osd"][host]) + def get_readable_key_value_state(self) -> Dict[str, str]: + state_data = self.machine.get_preflight_state("AnalyzeCephHandler").data + + osd_host_devices = self.machine.get_preflight_state_data( + "MigrateOSDsHandler", "osd_host_devices", default_value={} + ) + + kv_state_data = OrderedDict() + + for host in state_data["node"]["ls"]["osd"].keys(): + if host in osd_host_devices: + osd_device_list = [] + + for osd_id, device_path in osd_host_devices[host].items(): + osd_device_list.append( + {"OSD ID": osd_id, "Device path": device_path} + ) + + kv_state_data["ceph OSD node {0} devices".format(host)] = ( + self._get_readable_json_dump(osd_device_list) + ) + else: + kv_state_data["ceph OSD node {0} devices".format(host)] = ( + "Not analyzed yet" + ) + + return kv_state_data + def migrate_osds(self, host: str, osd_ids: List[int]) -> None: migrated_osd_ids = self.machine.get_execution_state_data( "MigrateOSDsHandler", "migrated_osd_ids", default_value=[] From a8177f1ea56e5c3d6c54990fff8358fd7ca99dd3 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 17 Sep 2024 20:35:25 +0200 Subject: [PATCH 20/27] Revert "feat: add picklefile read option with tests" This reverts commit a30c9caee4baa008e8d1688fe6892ad426131d5d. Signed-off-by: Tobias Wolf --- mock.pickle | Bin 66 -> 0 bytes src/rookify/__main__.py | 8 -------- 2 files changed, 8 deletions(-) delete mode 100644 mock.pickle diff --git a/mock.pickle b/mock.pickle deleted file mode 100644 index a4d3c945bef508c85556f9882f454b589ae4b0d5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 66 zcmZo*nQG1e0ku Any: - with open(pickle_file_name, "ab+") as pickle_file: - pickle_file.seek(0) - states_data = Unpickler(pickle_file).load() - return states_data - - def parse_args(args: list[str]) -> argparse.Namespace: # Putting args-parser in seperate function to make this testable arg_parser = ArgumentParser("Rookify") From 6604ec2e06d047f567bcc3eb3058c56cd2d7c409 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Tue, 17 Sep 2024 20:39:37 +0200 Subject: [PATCH 21/27] Fix import error in `rookify.yaml` for Python < 3.11 Signed-off-by: Tobias Wolf --- src/rookify/yaml.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/rookify/yaml.py b/src/rookify/yaml.py index b0c178b..24dab07 100644 --- a/src/rookify/yaml.py +++ b/src/rookify/yaml.py @@ -1,15 +1,12 @@ # -*- coding: utf-8 -*- import importlib.resources -import importlib.resources.abc import yamale from pathlib import Path from typing import Any, Dict - -_config_schema_file: Path | importlib.resources.abc.Traversable = Path( - "rookify", "config.schema.yaml" -) +# Use Any instead importlib.resources.abc. Traversable for Python < 3.11 +_config_schema_file: Any = Path("rookify", "config.schema.yaml") for entry in importlib.resources.files("rookify").iterdir(): if entry.name == "config.schema.yaml": _config_schema_file = entry From 9971137bb44efff319eb2be8072baf7faff94a84 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 19 Sep 2024 21:22:00 +0200 Subject: [PATCH 22/27] Return representive state data for module `migrate_mds` Signed-off-by: Tobias Wolf --- src/rookify/modules/migrate_mds/main.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/rookify/modules/migrate_mds/main.py b/src/rookify/modules/migrate_mds/main.py index bf0200a..163bc56 100644 --- a/src/rookify/modules/migrate_mds/main.py +++ b/src/rookify/modules/migrate_mds/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from time import sleep -from typing import Any +from typing import Any, Dict from ..exception import ModuleException from ..machine import Machine from ..module import ModuleHandler @@ -139,6 +139,15 @@ def _enable_rook_based_mds(self, mds_host: str) -> None: "Rook based ceph-mds daemon node '{0}' available".format(mds_host) ) + def get_readable_key_value_state(self) -> Dict[str, str]: + state_data = self.machine.get_preflight_state("AnalyzeCephHandler").data + + return { + "ceph MDS daemons": self._get_readable_json_dump( + list(state_data["node"]["ls"]["mds"].keys()) + ) + } + @staticmethod def register_execution_state( machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any From bc9fa4a3e9e79c2d729cde591bc6d417ad0367c4 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 19 Sep 2024 21:22:26 +0200 Subject: [PATCH 23/27] Return representive state data for module `migrate_mds_pools` Signed-off-by: Tobias Wolf --- src/rookify/modules/migrate_mds_pools/main.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/rookify/modules/migrate_mds_pools/main.py b/src/rookify/modules/migrate_mds_pools/main.py index 8a796dd..df67695 100644 --- a/src/rookify/modules/migrate_mds_pools/main.py +++ b/src/rookify/modules/migrate_mds_pools/main.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from collections import OrderedDict from typing import Any, Dict from ..machine import Machine from ..module import ModuleHandler @@ -71,6 +72,24 @@ def execute(self) -> None: for pool in pools.values(): self._migrate_pool(pool) + def get_readable_key_value_state(self) -> Dict[str, str]: + migrated_mds_pools = self.machine.get_execution_state_data( + "MigrateMdsPoolsHandler", "migrated_mds_pools", default_value=[] + ) + + pools = self.machine.get_preflight_state("MigrateMdsPoolsHandler").pools + + kv_state_data = OrderedDict() + + for pool in pools: + key_name = "ceph MDS pool {0}".format(pool["name"]) + kv_state_data[key_name] = self._get_readable_json_dump(pool) + + key_name = "ceph MDS pool {0} is created".format(pool["name"]) + kv_state_data[key_name] = pool["name"] in migrated_mds_pools + + return kv_state_data + def _migrate_pool(self, pool: Dict[str, Any]) -> None: migrated_mds_pools = self.machine.get_execution_state_data( "MigrateMdsPoolsHandler", "migrated_mds_pools", default_value=[] From 4845e24eee7af7e3a0b5203b4f319a8624895756 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 19 Sep 2024 21:22:41 +0200 Subject: [PATCH 24/27] Return representive state data for module `migrate_osd_pools` Signed-off-by: Tobias Wolf --- src/rookify/modules/migrate_osd_pools/main.py | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/rookify/modules/migrate_osd_pools/main.py b/src/rookify/modules/migrate_osd_pools/main.py index eafe828..9da3253 100644 --- a/src/rookify/modules/migrate_osd_pools/main.py +++ b/src/rookify/modules/migrate_osd_pools/main.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -from typing import Any, Dict +from collections import OrderedDict +from typing import Any, Dict, List from ..machine import Machine from ..module import ModuleHandler @@ -14,8 +15,12 @@ class MigrateOSDPoolsHandler(ModuleHandler): ] def execute(self) -> None: - state_data = self.machine.get_preflight_state("AnalyzeCephHandler").data + pools = self._get_filtered_osd_pools_list() + + for pool in pools: + self._migrate_pool(pool) + def _get_filtered_osd_pools_list(self) -> List[Dict[str, Any]]: migrated_mds_pools = self.machine.get_execution_state_data( name="MigrateMdsPoolsHandler", tag="migrated_pools", default_value=[] ) @@ -25,6 +30,8 @@ def execute(self) -> None: migrated_pools = migrated_mds_pools + migrated_rgw_pools + state_data = self.machine.get_preflight_state("AnalyzeCephHandler").data + osd_pool_configurations = self.ceph.get_osd_pool_configurations_from_osd_dump( state_data["osd"]["dump"] ) @@ -38,8 +45,25 @@ def execute(self) -> None: ): pools.append(pool) + return pools + + def get_readable_key_value_state(self) -> Dict[str, str]: + migrated_pools = self.machine.get_execution_state_data( + "MigrateOSDPoolsHandler", "migrated_pools", default_value=[] + ) + + pools = self._get_filtered_osd_pools_list() + + kv_state_data = OrderedDict() + for pool in pools: - self._migrate_pool(pool) + key_name = "ceph OSD pool {0}".format(pool["pool_name"]) + kv_state_data[key_name] = self._get_readable_json_dump(pool) + + key_name = "ceph OSD pool {0} is created".format(pool["pool_name"]) + kv_state_data[key_name] = pool["pool_name"] in migrated_pools + + return kv_state_data def _migrate_pool(self, pool: Dict[str, Any]) -> None: migrated_pools = self.machine.get_execution_state_data( From 88cbc3472c3d1cd192fd7ed0ffacb4e9e33e7a98 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 19 Sep 2024 21:22:58 +0200 Subject: [PATCH 25/27] Return representive state data for module `migrate_rgw_pools` Signed-off-by: Tobias Wolf --- src/rookify/modules/migrate_rgw_pools/main.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/rookify/modules/migrate_rgw_pools/main.py b/src/rookify/modules/migrate_rgw_pools/main.py index a44adc2..dddd3c5 100644 --- a/src/rookify/modules/migrate_rgw_pools/main.py +++ b/src/rookify/modules/migrate_rgw_pools/main.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +from collections import OrderedDict from typing import Any, Dict from ..exception import ModuleException from ..machine import Machine @@ -70,6 +71,25 @@ def execute(self) -> None: for zone_name, zone_data in zones.items(): self._migrate_zone(zone_name, zone_data) + def get_readable_key_value_state(self) -> Dict[str, str]: + migrated_pools = self.machine.get_execution_state_data( + "MigrateRgwPoolsHandler", "migrated_pools", default_value=[] + ) + + zones = self.machine.get_preflight_state("MigrateRgwPoolsHandler").zones + + kv_state_data = OrderedDict() + + for zone_data in zones.values(): + for osd_pool in zone_data["osd_pools"].values(): + key_name = "ceph RGW pool {0}".format(osd_pool["pool_name"]) + kv_state_data[key_name] = self._get_readable_json_dump(osd_pool) + + key_name = "ceph RGW pool {0} is created".format(osd_pool["pool_name"]) + kv_state_data[key_name] = osd_pool["pool_name"] in migrated_pools + + return kv_state_data + def _migrate_zone(self, zone_name: str, zone_data: Dict[str, Any]) -> None: migrated_zones = self.machine.get_execution_state_data( "MigrateRgwPoolsHandler", "migrated_zones", default_value=[] From 94a029c481640416e7d31bafdc4ddadf0c771008 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Thu, 19 Sep 2024 21:23:49 +0200 Subject: [PATCH 26/27] Return representive state data for module `migrate_rgws` Signed-off-by: Tobias Wolf --- src/rookify/modules/migrate_rgws/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/rookify/modules/migrate_rgws/main.py b/src/rookify/modules/migrate_rgws/main.py index bf84dbf..9dea10c 100644 --- a/src/rookify/modules/migrate_rgws/main.py +++ b/src/rookify/modules/migrate_rgws/main.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from time import sleep -from typing import Any, List +from typing import Any, Dict, List from ..exception import ModuleException from ..machine import Machine from ..module import ModuleHandler @@ -133,6 +133,11 @@ def _migrate_rgw(self, rgw_host: str) -> None: "Rook based RGW daemon for node '{0}' available".format(rgw_host) ) + def get_readable_key_value_state(self) -> Dict[str, str]: + return { + "ceph RGW hosts": self._get_readable_json_dump(self._get_rgw_daemon_hosts()) + } + @staticmethod def register_execution_state( machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any From ad6f0dae05e17d170a958d45184797a2f1a669c2 Mon Sep 17 00:00:00 2001 From: Tobias Wolf Date: Sat, 21 Sep 2024 09:28:58 +0200 Subject: [PATCH 27/27] Fix serialization issue for JSON dumps in "show states" mode This implements a generic fix to use `repr()` for data to be encoded to JSON values that would normally cause errors like: `TypeError: Object of type datetime is not JSON serializable` Signed-off-by: Tobias Wolf --- src/rookify/modules/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rookify/modules/module.py b/src/rookify/modules/module.py index 967c0bc..76c23f1 100644 --- a/src/rookify/modules/module.py +++ b/src/rookify/modules/module.py @@ -62,7 +62,7 @@ def ssh(self) -> SSH: return self._ssh def _get_readable_json_dump(self, data: Any) -> Any: - return json.dumps(data, sort_keys=True, indent="\t") + return json.dumps(data, default=repr, sort_keys=True, indent="\t") def get_readable_key_value_state(self) -> Optional[Dict[str, str]]: """