From 73a21acb70d759c5a784a6e6587206d002a4f37e Mon Sep 17 00:00:00 2001
From: Tobias Wolf <wolf@b1-systems.de>
Date: Tue, 18 Jun 2024 16:59:49 +0200
Subject: [PATCH] Add logic to "migrate_osds" to add OSD devices to Rook
 CephCluster nodes

Signed-off-by: Tobias Wolf <wolf@b1-systems.de>
---
 src/rookify/modules/migrate_osds/main.py      | 124 ++++++++++++++----
 .../templates/nodes_osd_devices_patch.yaml.j2 |   6 +
 2 files changed, 104 insertions(+), 26 deletions(-)
 create mode 100644 src/rookify/modules/migrate_osds/templates/nodes_osd_devices_patch.yaml.j2

diff --git a/src/rookify/modules/migrate_osds/main.py b/src/rookify/modules/migrate_osds/main.py
index 170ddbe..389d17b 100644
--- a/src/rookify/modules/migrate_osds/main.py
+++ b/src/rookify/modules/migrate_osds/main.py
@@ -1,35 +1,107 @@
 # -*- coding: utf-8 -*-
 
-from typing import Any, Dict
+from typing import Any, Dict, List
+from ..exception import ModuleException
+from ..machine import Machine
 from ..module import ModuleHandler
 
 
 class MigrateOSDsHandler(ModuleHandler):
-    REQUIRES = ["analyze_ceph"]
+    REQUIRES = ["analyze_ceph", "create_cluster"]
 
-    def execute(self) -> Any:
-        osd_config: Dict[str, Any] = {}
+    def _get_devices_of_hosts(self) -> Dict[str, List[str]]:
         state_data = self.machine.get_preflight_state("AnalyzeCephHandler").data
+        osd_devices: Dict[str, List[str]] = {}
 
-        for node, osds in state_data["node"]["ls"]["osd"].items():
-            osd_config[node] = {"osds": {}}
-            for osd in osds:
-                osd_config[node]["osds"][osd] = dict()
-
-        for osd in state_data["osd"]["dump"]["osds"]:
-            number = osd["osd"]
-            uuid = osd["uuid"]
-            for host in osd_config.values():
-                if number in host["osds"]:
-                    host["osds"][number]["uuid"] = uuid
-                    break
-
-        for node, values in osd_config.items():
-            devices = state_data["ssh"]["osd"][node]["devices"]
-            for osd in values["osds"].values():
-                for device in devices:
-                    if osd["uuid"] in device:
-                        osd["device"] = device
-                        break
-
-        self.logger.info(osd_config)
+        for osd_host, osds in state_data["node"]["ls"]["osd"].items():
+            osd_devices[osd_host] = []
+
+            """
+            Read OSD metadata to get OSD fsid UUID.
+            From there we try to get the encrypted volume partition UUID for later use in Rook.
+            """
+            for osd_id in osds:
+                osd_data = self.ceph.mon_command("osd metadata", id=osd_id)
+
+                result = self.ssh.command(
+                    osd_host,
+                    "sudo pvdisplay -c /dev/{0}".format(osd_data["devices"]),  # type:ignore
+                )
+
+                if result.failed:
+                    raise ModuleException("")
+
+                osd_vg_name = result.stdout.split(":")[1]
+
+                if osd_vg_name.startswith("ceph-"):
+                    osd_vg_name = osd_vg_name[5:]
+
+                osd_device_path = "/dev/ceph-{0}/osd-block-{0}".format(osd_vg_name)
+
+                if osd_device_path not in osd_devices[osd_host]:
+                    osd_devices[osd_host].append(osd_device_path)
+
+        return osd_devices
+
+    def preflight(self) -> None:
+        osd_host_devices = getattr(
+            self.machine.get_preflight_state("MigrateOSDsHandler"),
+            "osd_host_devices",
+            {},
+        )
+
+        if len(osd_host_devices) > 0:
+            return
+
+        self.machine.get_preflight_state(
+            "MigrateOSDsHandler"
+        ).osd_host_devices = self._get_devices_of_hosts()
+
+    def execute(self) -> None:
+        if getattr(
+            self.machine.get_execution_state("MigrateOSDsHandler"),
+            "migrated",
+            False,
+        ):
+            return
+
+        osd_host_devices = self.machine.get_preflight_state(
+            "MigrateOSDsHandler"
+        ).osd_host_devices
+
+        nodes_osd_devices = [
+            {"name": host, "devices": [{"name": device} for device in devices]}
+            for host, devices in osd_host_devices.items()
+        ]
+
+        cluster_patch_templated = self.load_template(
+            "nodes_osd_devices_patch.yaml.j2",
+            cluster_namespace=self._config["rook"]["cluster"]["namespace"],
+            cluster_name=self._config["rook"]["cluster"]["name"],
+            nodes_osd_devices_list=nodes_osd_devices,
+        )
+
+        crd_api = self.k8s.crd_api(api_version="ceph.rook.io/v1", kind="CephCluster")
+
+        crd_api.patch(
+            content_type="application/merge-patch+json",
+            body=cluster_patch_templated.yaml,
+        )
+
+        self.machine.get_execution_state("MigrateOSDsHandler").migrated = True
+
+    @staticmethod
+    def register_execution_state(
+        machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any
+    ) -> None:
+        ModuleHandler.register_execution_state(
+            machine, state_name, handler, tags=["migrated"]
+        )
+
+    @staticmethod
+    def register_preflight_state(
+        machine: Machine, state_name: str, handler: ModuleHandler, **kwargs: Any
+    ) -> None:
+        ModuleHandler.register_preflight_state(
+            machine, state_name, handler, tags=["osd_host_devices"]
+        )
diff --git a/src/rookify/modules/migrate_osds/templates/nodes_osd_devices_patch.yaml.j2 b/src/rookify/modules/migrate_osds/templates/nodes_osd_devices_patch.yaml.j2
new file mode 100644
index 0000000..7408bcb
--- /dev/null
+++ b/src/rookify/modules/migrate_osds/templates/nodes_osd_devices_patch.yaml.j2
@@ -0,0 +1,6 @@
+metadata:
+  name: {{ cluster_name }}
+  namespace: {{ cluster_namespace }}
+spec:
+  storage: # cluster level storage configuration and selection
+    nodes: {{ nodes_osd_devices_list }}