diff --git a/Makefile b/Makefile index a99da0d..7086995 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,11 @@ setup-venv: source ./.venv/bin/activate && \ pip install --ignore-installed -r requirements.txt +.PHONY: run-precommit +run-precommit: ## Run pre-commit to check if all files running through + pre-commit run --all-files + + .PHONY: update-requirements update-requirements: ## Update the requirements.txt with newer versions of pip packages source ./.venv/bin/activate && \ @@ -50,3 +55,8 @@ run-local-rookify: ## Runs rookify in the local development environment (require $(eval PYTHONPATH="${PYTHONPATH}:$(pwd)/src") source ./.venv/bin/activate && \ cd src && python3 -m rookify + +download-ceph-folder-from-testbed: + ssh testbed-node-0 'sudo cp -r /etc/ceph ~/ceph_configs && sudo chown -R $$USER:$$USER ~/ceph_configs' + scp -r testbed-node-0:ceph_configs ./.ceph + ssh testbed-node-0 'rm -rf ~/ceph_configs' diff --git a/requirements.txt b/requirements.txt index 0d819fa..219a759 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,7 @@ requests==2.31.0 requests-oauthlib==1.3.1 rsa==4.9 six==1.16.0 +structlog==24.1.0 urllib3==2.2.1 websocket-client==1.7.0 wrapt==1.16.0 diff --git a/src/config.example.yaml b/src/config.example.yaml index 4ed0fdd..d382e8e 100644 --- a/src/config.example.yaml +++ b/src/config.example.yaml @@ -1,8 +1,14 @@ general: module_data_file: data.yaml +logging: + level: INFO # level at which logging should start + format: + time: "%Y-%m-%d %H:%M.%S" # other example: "iso" + renderer: console # or: json + ceph: - conf_file: ../.ceph/ceph.conf + config: ../.ceph/ceph.conf keyring: ../.ceph/ceph.client.admin.keyring ssh: diff --git a/src/rookify/__main__.py b/src/rookify/__main__.py index cac2ed7..990d768 100644 --- a/src/rookify/__main__.py +++ b/src/rookify/__main__.py @@ -5,18 +5,31 @@ from types import MappingProxyType from .yaml import load_yaml, save_yaml +from rookify.logger import configure_logging, get_logger def main() -> None: + # Load configuration file try: config = load_yaml("config.yaml") except FileNotFoundError as err: raise SystemExit(f"Could not load config: {err}") + + # Configure logging + try: + configure_logging(config["logging"]) + except Exception as e: + raise SystemExit(f"Error configuring logging: {e}") + + log = get_logger() + log.debug("Executing Rookify") + preflight_modules, migration_modules = rookify.modules.load_modules( config["migration_modules"] ) module_data = dict() + try: module_data.update(load_yaml(config["general"]["module_data_file"])) except FileNotFoundError: @@ -59,6 +72,8 @@ def main() -> None: save_yaml(config["general"]["module_data_file"], module_data) + log.info("Data was updated to module_data_file.") + if __name__ == "__main__": main() diff --git a/src/rookify/logger.py b/src/rookify/logger.py new file mode 100644 index 0000000..4eb77db --- /dev/null +++ b/src/rookify/logger.py @@ -0,0 +1,37 @@ +import structlog +import logging + +from typing import Any, Dict + + +def configure_logging(config: Dict[str, Any]) -> None: + LOG_LEVEL = getattr(logging, config["level"], logging.INFO) + LOG_TIME_FORMAT = config["format"]["time"] + LOG_RENDERER = config["format"]["renderer"] + + structlog.configure( + wrapper_class=structlog.make_filtering_bound_logger(LOG_LEVEL), + processors=[ + structlog.processors.TimeStamper(fmt=LOG_TIME_FORMAT), + structlog.processors.add_log_level, + ], + ) + + if LOG_RENDERER == "console": + structlog.configure( + processors=[ + *structlog.get_config()["processors"], + structlog.dev.ConsoleRenderer(), + ] + ) + else: + structlog.configure( + processors=[ + *structlog.get_config()["processors"], + structlog.processors.JSONRenderer(), + ] + ) + + +def get_logger() -> structlog.getLogger: + return structlog.get_logger() diff --git a/src/rookify/modules/analyze_ceph/main.py b/src/rookify/modules/analyze_ceph/main.py index 6cb023f..594b5f8 100644 --- a/src/rookify/modules/analyze_ceph/main.py +++ b/src/rookify/modules/analyze_ceph/main.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from ..module import ModuleHandler - from typing import Any, Dict @@ -20,10 +19,11 @@ def run(self) -> Any: leaf[part] = self.ceph.mon_command(command) leaf = leaf[part] + self.logger.info("Dictionary created") results["ssh"] = dict() results["ssh"]["osd"] = dict() for node, values in results["node"]["ls"]["osd"].items(): devices = self.ssh.command(node, "find /dev/ceph-*/*").stdout.splitlines() results["ssh"]["osd"][node] = {"devices": devices} - + self.logger.info("AnalyzeCephHandler ran successfully.") return results diff --git a/src/rookify/modules/migrate_monitors/main.py b/src/rookify/modules/migrate_monitors/main.py index 367a7ca..f057695 100644 --- a/src/rookify/modules/migrate_monitors/main.py +++ b/src/rookify/modules/migrate_monitors/main.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- from ..module import ModuleHandler +from typing import Dict, Any class MigrateMonitorsHandler(ModuleHandler): - pass + def run(self) -> Dict[str, Any]: + self.logger.info("MigrateMonitorsHandler ran successfully.") + return {} diff --git a/src/rookify/modules/migrate_osds/main.py b/src/rookify/modules/migrate_osds/main.py index 5377671..8f89c42 100644 --- a/src/rookify/modules/migrate_osds/main.py +++ b/src/rookify/modules/migrate_osds/main.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from ..module import ModuleHandler - from typing import Any, Dict @@ -34,5 +33,5 @@ def run(self) -> Any: osd["device"] = device break - print(osd_config) + self.logger.info(osd_config) return {} diff --git a/src/rookify/modules/module.py b/src/rookify/modules/module.py index 8d1a75a..bffacd5 100644 --- a/src/rookify/modules/module.py +++ b/src/rookify/modules/module.py @@ -8,6 +8,8 @@ import kubernetes import fabric import jinja2 +import structlog +from rookify.logger import get_logger from typing import Any, Dict, List, Optional @@ -162,6 +164,9 @@ def __init__(self, config: Dict[str, Any], data: Dict[str, Any], module_path: st self.__ceph: Optional[ModuleHandler.__Ceph] = None self.__k8s: Optional[ModuleHandler.__K8s] = None self.__ssh: Optional[ModuleHandler.__SSH] = None + self.__logger = get_logger() + + self.__logger.debug("Executing {0}", self.__class__.__name__) @abc.abstractmethod def preflight(self) -> None: @@ -185,6 +190,10 @@ def ceph(self) -> __Ceph: self.__ceph = ModuleHandler.__Ceph(self._config["ceph"]) return self.__ceph + @property + def logger(self) -> structlog.getLogger: + return self.__logger + @property def k8s(self) -> __K8s: if self.__k8s is None: