From 467987ff69c48a5dd0fd1e7c4798110a3ef0b230 Mon Sep 17 00:00:00 2001 From: Yongxue Hong Date: Thu, 27 Jul 2023 17:08:37 +0800 Subject: [PATCH] Add Virtual Machines Management(VMM) Signed-off-by: Yongxue Hong --- .../vt_agent/instance_drivers/__init__.py | 64 ++++++ virttest/vt_agent/instance_drivers/libvirt.py | 12 ++ virttest/vt_agent/instance_drivers/qemu.py | 56 ++++++ virttest/vt_agent/managers/__init__.py | 0 virttest/vt_agent/managers/monitor.py | 0 virttest/vt_agent/managers/vmm.py | 88 +++++++++ virttest/vt_agent/services/virt/vmm.py | 38 ++++ virttest/vt_vmm/__init__.py | 0 virttest/vt_vmm/api.py | 179 +++++++++++++++++ virttest/vt_vmm/conductor.py | 26 +++ virttest/vt_vmm/spec/__init__.py | 35 ++++ virttest/vt_vmm/spec/libvirt.py | 9 + virttest/vt_vmm/spec/qemu.py | 187 ++++++++++++++++++ 13 files changed, 694 insertions(+) create mode 100644 virttest/vt_agent/instance_drivers/__init__.py create mode 100644 virttest/vt_agent/instance_drivers/libvirt.py create mode 100644 virttest/vt_agent/instance_drivers/qemu.py create mode 100644 virttest/vt_agent/managers/__init__.py create mode 100644 virttest/vt_agent/managers/monitor.py create mode 100644 virttest/vt_agent/managers/vmm.py create mode 100644 virttest/vt_agent/services/virt/vmm.py create mode 100755 virttest/vt_vmm/__init__.py create mode 100755 virttest/vt_vmm/api.py create mode 100755 virttest/vt_vmm/conductor.py create mode 100644 virttest/vt_vmm/spec/__init__.py create mode 100644 virttest/vt_vmm/spec/libvirt.py create mode 100644 virttest/vt_vmm/spec/qemu.py diff --git a/virttest/vt_agent/instance_drivers/__init__.py b/virttest/vt_agent/instance_drivers/__init__.py new file mode 100644 index 0000000000..23109030d3 --- /dev/null +++ b/virttest/vt_agent/instance_drivers/__init__.py @@ -0,0 +1,64 @@ +import json +import signal +from functools import partial + +import six + +from abc import ABCMeta +from abc import abstractmethod + +import aexpect + +from . import qemu +from . import libvirt + + +@six.add_metaclass(ABCMeta) +class InstanceDriver(object): + def __init__(self, kind, spec): + self._kind = kind + self._params = json.loads(spec) + self._process = None + self._cmd = None + self._devices = None + + @abstractmethod + def create_devices(self): + raise NotImplementedError + + @abstractmethod + def make_cmdline(self): + raise NotImplementedError + + def run_cmdline(self, command, termination_func=None, output_func=None, + output_prefix="", timeout=1.0, auto_close=True, pass_fds=(), + encoding=None): + self._process = aexpect.run_tail( + command, termination_func, output_func, output_prefix, + timeout, auto_close, pass_fds, encoding) + + def get_pid(self): + return self._process.get_pid() + + def get_status(self): + return self._process.get_status() + + def get_output(self): + return self._process.get_output() + + def is_alive(self): + return self._process.is_alive() + + def kill(self, sig=signal.SIGKILL): + self._process.kill(sig) + + +def get_instance_driver(kind, spec): + instance_drivers = { + "qemu": qemu.QemuInstanceDriver(spec), + "libvirt": libvirt.LibvirtInstanceDriver(spec), + } + + if kind not in instance_drivers: + raise OSError("No support the %s instance driver" % kind) + return instance_drivers.get(kind, spec) diff --git a/virttest/vt_agent/instance_drivers/libvirt.py b/virttest/vt_agent/instance_drivers/libvirt.py new file mode 100644 index 0000000000..60f6851dd1 --- /dev/null +++ b/virttest/vt_agent/instance_drivers/libvirt.py @@ -0,0 +1,12 @@ +from . import InstanceDriver + + +class LibvirtInstanceDriver(InstanceDriver): + def __init__(self, spec): + super(LibvirtInstanceDriver, self).__init__("libvirt", spec) + + def create_devices(self): + pass + + def make_cmdline(self): + pass diff --git a/virttest/vt_agent/instance_drivers/qemu.py b/virttest/vt_agent/instance_drivers/qemu.py new file mode 100644 index 0000000000..58db9ee926 --- /dev/null +++ b/virttest/vt_agent/instance_drivers/qemu.py @@ -0,0 +1,56 @@ +import logging + +from virttest.qemu_devices import qdevices, qcontainer + +from . import InstanceDriver + +LOG = logging.getLogger("avocado.agent." + __name__) + + +class QemuInstanceDriver(InstanceDriver): + def __init__(self, spec): + super(QemuInstanceDriver, self).__init__("qemu", spec) + + def create_devices(self): + + def _add_name(name): + return " -name '%s'" % name + + def _process_sandbox(devices, action): + if action == "add": + if devices.has_option("sandbox"): + return " -sandbox on " + elif action == "rem": + if devices.has_option("sandbox"): + return " -sandbox off " + + qemu_binary = "/usr/libexec/qemu-kvm" + name = self._params.get("name") + self._devices = qcontainer.DevContainer(qemu_binary, name) + StrDev = qdevices.QStringDevice + + self._devices.insert(StrDev('qemu', cmdline=qemu_binary)) + + qemu_preconfig = self._params.get("qemu_preconfig") + if qemu_preconfig: + self._devices.insert(StrDev('preconfig', cmdline="--preconfig")) + + self._devices.insert(StrDev('vmname', cmdline=_add_name(name))) + + qemu_sandbox = self._params.get("qemu_sandbox") + if qemu_sandbox == "on": + self._devices.insert( + StrDev('qemu_sandbox', cmdline=_process_sandbox(self._devices, "add"))) + elif qemu_sandbox == "off": + self.devices.insert( + StrDev('qemu_sandbox', cmdline=_process_sandbox(self._devices, "rem"))) + + defaults = self._params.get("defaults", "no") + if self._devices.has_option("nodefaults") and defaults != "yes": + self._devices.insert(StrDev('nodefaults', cmdline=" -nodefaults")) + + return self._devices + + def make_cmdline(self): + self._cmd = self._devices.cmdline() + return self._cmd diff --git a/virttest/vt_agent/managers/__init__.py b/virttest/vt_agent/managers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/virttest/vt_agent/managers/monitor.py b/virttest/vt_agent/managers/monitor.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/virttest/vt_agent/managers/vmm.py b/virttest/vt_agent/managers/vmm.py new file mode 100644 index 0000000000..d103ec48af --- /dev/null +++ b/virttest/vt_agent/managers/vmm.py @@ -0,0 +1,88 @@ +import json +import logging + +from .. import instance_drivers + +LOG = logging.getLogger("avocado.agent." + __name__) + + +class VMMError(Exception): + pass + + +class VirtualMachinesManager(object): + def __init__(self): + self._filename = "/var/instances" + self._instances = self._load() + + @property + def instances(self): + return self._load() + + def _dump_instances(self): + with open(self._filename, "w") as details: + json.dump(self._instances, details) + + def _load_instances(self): + try: + with open(self._filename, 'r') as instances: + return json.load(instances) + except Exception: + return {} + + def _save(self): + self._dump_instances() + + def _load(self): + return self._load_instances() + + def register_instance(self, name, info): + if name in self._instances: + LOG.error("The instance %s is already registered.", name) + return False + self._instances[name] = info + self._save() + return True + + def unregister_instance(self, name): + if name in self._instances: + del self._instances[name] + self._save() + return True + LOG.error("The instance %s is not registered" % name) + return False + + def get_instance(self, name): + return self._instances.get(name) + + def update_instance(self, name, info): + self._instances.get(name).update(info) + self._save() + + @staticmethod + def build_instance(driver_kind, spec): + instance_info = {} + instance_driver = instance_drivers.get_instance_driver(driver_kind, spec) + instance_info["devices"] = instance_driver.create_devices() + instance_info["driver"] = instance_driver + return instance_info + + def run_instance(self, instance_id): + instance_info = self.get_instance(instance_id) + instance_driver = instance_info["driver"] + cmdline = instance_driver.make_cmdline() + instance_info["cmdline"] = cmdline + process = instance_driver.run_cmdline(cmdline) + instance_info["process"] = process + + def stop_instance(self, instance_id): + instance_info = self.get_instance(instance_id) + instance_driver = instance_info["driver"] + if instance_driver.is_alive(): + pass + + def get_instance_status(self, instance_id): + pass + + def get_instance_pid(self, instance_id): + pass diff --git a/virttest/vt_agent/services/virt/vmm.py b/virttest/vt_agent/services/virt/vmm.py new file mode 100644 index 0000000000..cd6e2f88e4 --- /dev/null +++ b/virttest/vt_agent/services/virt/vmm.py @@ -0,0 +1,38 @@ +import logging + +from ...managers import vmm + +VMM = vmm.VirtualMachinesManager() + +LOG = logging.getLogger('avocado.service.' + __name__) + + +def build_instance(instance_id, instance_driver, instance_spec): + if instance_id in VMM.instances: + raise vmm.VMMError(f"The instance {instance_id} was registered.") + + LOG.info(f"Build the instance {instance_id} by {instance_spec}") + + instance_info = VMM.build_instance(instance_driver, instance_spec) + VMM.register_instance(instance_id, instance_info) + + +def run_instance(instance_id): + VMM.run_instance(instance_id) + + +def stop_instance(instance_id): + VMM.stop_instance(instance_id) + + +def get_instance_status(instance_id): + return VMM.get_instance_status(instance_id) + + +def get_instance_pid(instance_id): + return VMM.get_instance_pid(instance_id) + + +def get_instance_monitors(instance_id): + return [] + diff --git a/virttest/vt_vmm/__init__.py b/virttest/vt_vmm/__init__.py new file mode 100755 index 0000000000..e69de29bb2 diff --git a/virttest/vt_vmm/api.py b/virttest/vt_vmm/api.py new file mode 100755 index 0000000000..0e198ccaa9 --- /dev/null +++ b/virttest/vt_vmm/api.py @@ -0,0 +1,179 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# See LICENSE for more details. +# +# Copyright: Red Hat Inc. 2023 +# Authors: Yongxue Hong + +import logging + +LOG = logging.getLogger('avocado.' + __name__) + +from .spec import get_spec_helper +from .conductor import Conductor + +from virttest import utils_misc + + +class VMMError(Exception): + pass + + +class _VirtualMachinesManager(object): + """ + The virtual machine manager(VMM) is to manager all the VM instances + on the cluster. + """ + + def __init__(self): + self._instances = {} + + def get_instance_status(self, instance_id): + conductor = self._instances.get(instance_id)["conductor"] + status = conductor.get_instance_status() + return status + + def define_instance(self, node, config): + """ + Define an instance by the configuration. + Note: this method just define the information, not start it + """ + # info is passed by calling instance.define_instance_info + instance_id = config["id"] + if instance_id in self._instances: + name = config["metadata"]["name"] + raise VMMError(f"The instance {name}({instance_id}) was defined") + + conductor = Conductor(node, instance_id, config) + conductor.build_instance() + self._instances[instance_id] = {"conductor": conductor, "config": config} + return instance_id + + def get_instance_spec(self, instance_id): + return self._instances[instance_id]["spec"].copy() + + def update_instance(self, instance_id, spec): + """ + Update the instance. + + :param instance_id: + :param spec: The spec instance + :type spec: Spec object + :return: + """ + pass + + def start_instance(self, instance_id): + """ + Start an instance. + """ + conductor = self._instances.get(instance_id)["conductor"] + conductor.run_instance() + + def create_instance(self, node, config): + """ + Create an instance. + """ + self.start_instance(self.define_instance(node, config)) + + def delete_instance(self, instance_id): + """ + Delete the instance from the VMM + + """ + try: + if self.get_instance_status(instance_id) == "running": + self.stop_instance(instance_id) + except: + conductor = self._instances.get(instance_id)["conductor"] + conductor.kill_instance() + + del self._instances[instance_id] + + def pause_instance(self, instance_id): + if self.get_instance_status(instance_id) != "paused": + raise VMMError("Failed to pause") + + def resume_instance(self, instance_id): + if self.get_instance_status(instance_id) != "running": + raise VMMError("Failed to resume") + + def reboot_instance(self, instance_id): + pass + + def get_consoles(self, instance_id): + consoles = [] + console_info = {} + console_info["type"] = "" + console_info["uri"] = "" + + return consoles + + def get_device_metadata(self, instance_id): + return [] + + def get_access_ipv4(self, instance_id): + return "" + + def get_access_ipv6(self, instance_id): + return "" + + def list_instances(self): + instances = [] + + for instance_id, info in self._instances.items(): + instance_info = dict() + instance_info["id"] = instance_id + instance_info["name"] = info.get("config")["name"] + instance_info["type"] = info.get("config")["kind"] + instance_info["status"] = self.get_instance_status(instance_id) + instance_info["consoles"] = self.get_consoles(instance_id) + instance_info["access_ipv4"] = self.get_access_ipv4(instance_id) + instance_info["access_ipv6"] = self.get_access_ipv6(instance_id) + instance_info["device_metadata"] = self.get_device_metadata(instance_id) + instances.append(instance_info) + + return instances + + def stop_instance(self, instance_id): + conductor = self._instances.get(instance_id)["conductor"] + conductor.stop_instance() + + def get_instance_pid(self, instance_id): + conductor = self._instances.get(instance_id)["conductor"] + return conductor.get_instance_pid() + + +def define_instance_config(name, vm_params): + """ + This interface is to handle the resource allocation and define the config + The resource allocation should be done before generating the spec. + # TODO: rename the interface. + + :param name: + :param vm_params: + :return: + """ + metadata = dict() + metadata["name"] = name + # Unique VT ID of the whole cluster + metadata["id"] = utils_misc.generate_random_string(16) + kind = vm_params.get("vm_type") + # add the resource allocation. + spec_helper = get_spec_helper(kind) + # suggestion: return a spec instance(class) by build spec: + # decouple spec and json. + spec = spec_helper.to_json(vm_params) + config = {"kind": kind, "metadata": metadata, "spec": spec} + LOG.debug(f"The config of the instance {name}: {config}") + return config + + +vmm = _VirtualMachinesManager() diff --git a/virttest/vt_vmm/conductor.py b/virttest/vt_vmm/conductor.py new file mode 100755 index 0000000000..2c222e9b74 --- /dev/null +++ b/virttest/vt_vmm/conductor.py @@ -0,0 +1,26 @@ +class Conductor(object): + def __init__(self, node, instance_id, instance_config): + self._instance_server = node.proxy.virt.vmm + self._instance_id = instance_id + self._instance_config = instance_config + self._instance_driver = instance_config["kind"] + + def build_instance(self): + self._instance_server.build_instance(self._instance_id, + self._instance_driver, + self._instance_config["spec"]) + + def run_instance(self): + self._instance_server.run_instance(self._instance_id) + + def stop_instance(self): + self._instance_server.stop_instance(self._instance_id) + + def get_instance_status(self): + self._instance_server.get_instance_status(self._instance_id) + + def get_instance_pid(self): + self._instance_server.get_instance_pid(self._instance_id) + + def get_consoles(self): + return self._instance_server.get_consoles(self._instance_id) diff --git a/virttest/vt_vmm/spec/__init__.py b/virttest/vt_vmm/spec/__init__.py new file mode 100644 index 0000000000..58821cc98a --- /dev/null +++ b/virttest/vt_vmm/spec/__init__.py @@ -0,0 +1,35 @@ +import json + +import six + +from abc import ABCMeta +from abc import abstractmethod + +from . import qemu +from . import libvirt + + +@six.add_metaclass(ABCMeta) +class SpecHelper(object): + def __init__(self, kind=None): + self._kind = kind + self._spec = None + + @abstractmethod + def _parse_params(self, params): + raise NotImplementedError + + def to_json(self, params): + self._spec = self._parse_params(params) + return json.dumps(self._spec) + + +def get_spec_helper(kind): + spec_helpers = { + "qemu": qemu.QemuSpecHelper(), + "libvirt": libvirt.LibvirtSpecHelper(), + } + + if kind not in spec_helpers: + raise OSError("No support the %s spec" % kind) + return spec_helpers.get(kind) diff --git a/virttest/vt_vmm/spec/libvirt.py b/virttest/vt_vmm/spec/libvirt.py new file mode 100644 index 0000000000..20f823ea13 --- /dev/null +++ b/virttest/vt_vmm/spec/libvirt.py @@ -0,0 +1,9 @@ +from . import SpecHelper + + +class LibvirtSpecHelper(SpecHelper): + def __init__(self): + super(LibvirtSpecHelper, self).__init__("libvirt") + + def _parse_params(self, params): + raise NotImplementedError diff --git a/virttest/vt_vmm/spec/qemu.py b/virttest/vt_vmm/spec/qemu.py new file mode 100644 index 0000000000..f3885a4f24 --- /dev/null +++ b/virttest/vt_vmm/spec/qemu.py @@ -0,0 +1,187 @@ +import arch + +from . import SpecHelper + + +class QemuSpecHelper(SpecHelper): + def __init__(self): + super(QemuSpecHelper, self).__init__("qemu") + self._params = None + + def _define_uuid(self): + return self._params.get("uuid") + + def _define_preconfig(self): + return self._params.get_boolean("qemu_preconfig") + + def _define_sandbox(self): + return self._params.get("qemu_sandbox") + + def _define_defaults(self): + return self._params.get("defaults", "no") + + def _define_machine(self): + machine = {} + machine["type"] = self._params.get("machine_type") + machine["accel"] = self._params.get("vm_accelerator") + return machine + + def _define_launch_security(self): + launch_security = {} + + if self._params.get("vm_secure_guest_type"): + launch_security["id"] = "lsec0" + launch_security["type"] = self._params.get("vm_secure_guest_type") + if launch_security["type"] == "sev": + launch_security["policy"] = int(self._params.get("vm_sev_policy")) + launch_security['cbitpos'] = int(self._params.get("vm_sev_cbitpos")) + launch_security['reduced_phys_bits'] = int(self._params.get("vm_sev_reduced_phys_bits")) + launch_security['session_file'] = self._params.get("vm_sev_session_file") + launch_security['dh_cert_file'] = self._params.get("vm_sev_dh_cert_file") + launch_security['kernel_hashes'] = self._params.get("vm_sev_kernel_hashes") + elif launch_security["type"] == "tdx": + pass + return launch_security + + def _define_iommu(self): + iommu = {} + + if self._params.get("intel_iommu"): + iommu["type"] = "intel_iommu" + elif self._params.get("virtio_iommu"): + iommu["type"] = "virtio_iommu" + iommu["prps"] = {} + iommu["bus"] = "pci.0" + + return iommu + + def _define_vga(self): + vga = {} + + if self._params.get("vga"): + vga["type"] = self._params.get("vga") + vga["bus"] = "pci.0" + + return vga + + def _define_watchdog(self): + watchdog = {} + + if self._params.get("enable_watchdog", "no") == "yes": + watchdog["type"] = self._params.get("watchdog_device_type") + watchdog["bus"] = "pci.0" + watchdog["action"] = self._params.get("watchdog_action", "reset") + + return watchdog + + def _define_pci_controller(self): + pci_controller = {} + return pci_controller + + def _define_memory(self): + memory = {} + return memory + + def _define_cpu(self): + cpu = {} + return cpu + + def _define_numa(self): + numa = [] + return numa + + def _define_soundcards(self): + soundcards = [] + + for sound_device in self._params.get("soundcards").split(","): + soundcard = {} + if "hda" in sound_device: + soundcard["type"] = "intel-hba" + elif sound_device in ("es1370", "ac97"): + soundcard["type"] = sound_device.upper() + else: + soundcard["type"] = sound_device + + soundcard["bus"] = {'aobject': self._params.get('pci_bus', 'pci.0')} + soundcards.append(soundcard) + + return soundcards + + def _define_monitors(self): + monitors = [] + + for monitor_name in self._params.objects("monitors"): + monitor_params = self._params.object_params(monitor_name) + monitor = {} + monitor["type"] = monitor_params.get("monitor_type") + monitor["bus"] = {'aobject': self._params.get('pci_bus', 'pci.0')} + monitors.append(monitor) + + return monitors + + def _define_pvpanic(self): + pvpanic = {} + + if self._params.get("enable_pvpanic") == "yes": + if 'aarch64' in self._params.get('vm_arch_name', arch.ARCH): + pvpanic["type"] = 'pvpanic-pci' + else: + pvpanic["type"] = 'pvpanic' + pvpanic["bus"] = "" + pvpanic["props"] = "" + + return pvpanic + + def _define_vmcoreinfo(self): + return "" + + def _define_serials(self): + serials = [] + for serial_id in self._params.objects('serials'): + serial = {} + serial_params = self._params.object_params(serial_id) + serial["id"] = serial_id + serial["type"] = serial_params.get('serial_type') + serial["bus"] = "" + serial["props"] = {} + if serial["type"] == "spapr-vty": + serial["props"]["serial_reg"] = "" + + backend = serial_params.get('chardev_backend', + 'unix_socket') + + if backend in ['udp', 'tcp_socket']: + serial_params['chardev_host'] = "" + serial_params['chardev_port'] = "" + + serial["props"]["name"] = serial_params.get('serial_name') + serials.append(serial) + return serials + + def _define_rngs(self): + rngs = [] + return rngs + + def _parse_params(self, params): + self._params = params + spec = {} + spec["uuid"] = self._define_uuid() + spec["preconfig"] = self._define_preconfig() + spec["sandbox"] = self._define_sandbox() + spec["defaults"] = self._define_defaults() + spec["machine"] = self._define_machine() + spec["launch_security"] = self._define_launch_security() + spec["iommu"] = self._define_iommu() + spec["vga"] = self._define_vga() + spec["watchdog"] = self._define_watchdog() + spec["pci_controller"] = self._define_pci_controller() + spec["memory"] = self._define_memory() + spec["cpu"] = self._define_cpu() + spec["numa"] = self._define_numa() + spec["soundcards"] = self._define_soundcards() + spec["monitors"] = self._define_monitors() + spec["pvpanic"] = self._define_pvpanic() + spec["vmcoreinfo"] = self._define_vmcoreinfo() + spec["serials"] = self._define_serials() + spec["rngs"] = self._define_rngs() +