From 599c68188c50a717450006cf96f3fef63d37be6b Mon Sep 17 00:00:00 2001
From: Zhenchao Liu <zhencliu@redhat.com>
Date: Thu, 31 Oct 2024 10:40:21 +0800
Subject: [PATCH 1/2] virtio_fs_nfs_migration: Virtiofs migration testing over
 NFS

The following testing matrix is covered:
cache mode:
  auto, never, always, metadata
writeback:
  enabled and disabled
allow-direct-io
  enabled and disabled
file-handles
  mandatory and never
virtiofs
  single fs and two fs

Note we usually use memory-backend-file on s390x for the virtio-fs
testing on RHEL, but for the live migration scenario, we have to use
memory-backend-memfd due to RHEL-58831.

Signed-off-by: Zhenchao Liu <zhencliu@redhat.com>
---
 qemu/tests/cfg/virtio_fs_nfs_migration.cfg | 172 +++++++++++
 qemu/tests/virtio_fs_nfs_migration.py      | 320 +++++++++++++++++++++
 2 files changed, 492 insertions(+)
 create mode 100644 qemu/tests/cfg/virtio_fs_nfs_migration.cfg
 create mode 100644 qemu/tests/virtio_fs_nfs_migration.py

diff --git a/qemu/tests/cfg/virtio_fs_nfs_migration.cfg b/qemu/tests/cfg/virtio_fs_nfs_migration.cfg
new file mode 100644
index 0000000000..0624829909
--- /dev/null
+++ b/qemu/tests/cfg/virtio_fs_nfs_migration.cfg
@@ -0,0 +1,172 @@
+- virtio_fs_nfs_migration:
+    no RHEL.6 RHEL.7 RHEL.8.0 RHEL.8.1
+    no Win2008 Win7 Win2012 Win2012..r2 Win8 Win8.1
+
+    type = virtio_fs_nfs_migration
+    virt_test_type = qemu
+    required_qemu = [9.0.0,)
+    kill_vm = yes
+    start_vm = yes
+    not_preprocess = yes
+
+    # Setup the 1st nfs server
+    setup_local_nfs = yes
+    export_options = 'rw,insecure,no_root_squash,async'
+    export_dir = /var/tmp/virtio_fs_test_nfs
+    nfs_mount_options = rw
+    nfs_mount_dir = /var/mnt/fs
+    nfs_mount_dir_target = /var/mnt/targetfs
+
+    filesystems = fs
+    filesystems_migration = targetfs
+    fs_driver = virtio-fs
+    fs_source_type = mount
+    fs_source_dir_fs = ${nfs_mount_dir}
+    fs_source_dir_targetfs = ${nfs_mount_dir_target}
+    fs_target = myfs
+    fs_dest = /mnt/${fs_target}
+    force_create_fs_source = no
+    remove_fs_source = no
+    fs_driver_props = {"queue-size": 1024}
+    fs_binary_extra_options = ""
+    fs_binary_extra_options_targetfs = ""
+    driver_name = viofs
+
+    test_file = test_file
+    guest_file = "%s/${test_file}"
+    pre_command = "mkdir -p ${nfs_mount_dir_target}"
+    post_command = "rm -rf ${nfs_mount_dir_target} ${export_dir}"
+    cmd_md5 = 'md5sum %s/${test_file}'
+    cmd_dd = 'dd if=/dev/urandom of=%s bs=1M count=100 oflag=direct'
+    fio_name = this_is_fiotest
+    fio_filename = %s/${test_file}.fio
+    numjobs = 4
+    fio_options = '--name=${fio_name} --filename=%s --ioengine=libaio'
+    fio_options += ' --rw=rw --bs=1k --size=200M --numjobs=${numjobs} --runtime=1800 --time_based'
+    cmd_chk_fio = "ps -ef | grep fio | grep ${fio_name} | grep -v grep"
+
+    share_mem = yes
+    vm_mem_share = yes
+    # Note that memory-backend-file can cause error when doing live migration
+    vm_mem_backend = memory-backend-memfd
+    Win10.i386:
+        mem = 4096
+    !s390, s390x:
+        mem_devs = mem1
+        backend_mem_mem1 = memory-backend-memfd
+        size_mem1 = ${mem}M
+        use_mem_mem1 = no
+        guest_numa_nodes = shm0
+        numa_memdev_shm0 = mem-mem1
+        numa_nodeid_shm0 = 0
+    Windows:
+        # install winfsp tool
+        i386, i686:
+            install_winfsp_path = 'C:\Program Files'
+            devcon_dirname = 'x86'
+        x86_64:
+            install_winfsp_path = 'C:\Program Files (x86)'
+            devcon_dirname = 'amd64'
+        install_winfsp_cmd = 'msiexec /i WIN_UTILS:\winfsp.msi /qn'
+        check_installed_cmd = 'dir "%s" |findstr /I winfsp'
+        viofs_log_file = C:\viofs_log.txt
+        viofs_svc_name = VirtioFsSvc
+        viofs_exe_path = C:\virtiofs.exe
+        viofs_exe_copy_cmd = xcopy %s C:\ /Y
+        viofs_sc_create_cmd = 'sc create ${viofs_svc_name} binpath=${viofs_exe_path} start=auto'
+        viofs_sc_create_cmd += ' depend="WinFsp.Launcher/VirtioFsDrv" DisplayName="Virtio FS Service"'
+        viofs_sc_start_cmd = 'sc start ${viofs_svc_name}'
+        viofs_sc_query_cmd = 'sc query ${viofs_svc_name}'
+        viofs_sc_delete_cmd = 'sc delete ${viofs_svc_name}'
+        debug_log_operation = 'enable'
+        viofs_debug_enable_cmd = 'reg add HKLM\Software\VirtIO-FS /v DebugFlags /d 0xFFFFFFFF /t REG_DWORD'
+        viofs_log_enable_cmd = 'reg add HKLM\Software\VirtIO-FS /v DebugLogFile /d ${viofs_log_file} /t REG_SZ'
+        viofs_debug_delete_cmd = 'reg delete HKLM\Software\VirtIO-FS /v DebugFlags /f'
+        viofs_log_delete_cmd = 'reg delete HKLM\Software\VirtIO-FS /v DebugLogFile /f'
+        viofs_reg_query_cmd = 'reg query HKLM\Software\VirtIO-FS'
+        virtio_win_media_type = iso
+        cdroms += " virtio"
+        cmd_md5 = "%s: && md5sum.exe ${test_file}"
+        cmd_dd = 'dd if=/dev/random of=%s bs=1M count=100'
+        guest_file = "%s:\${test_file}"
+        fio_options = '--name=${fio_name} --filename=%s --ioengine=windowsaio'
+        fio_options += ' --rw=rw --bs=1k --size=200M --numjobs=${numjobs} --runtime=1800 --time_based --thread'
+        fio_name = fio.exe
+        cmd_chk_fio = 'TASKLIST /FI "IMAGENAME eq FIO.EXE'
+        fio_filename = "%s\:\${test_file}.fio"
+    variants:
+        - cache_mode_auto:
+            fs_binary_extra_options += " --cache auto"
+            fs_binary_extra_options_targetfs += " --cache auto"
+        - cache_mode_always:
+            fs_binary_extra_options += " --cache always"
+            fs_binary_extra_options_targetfs += " --cache always"
+        - cache_mode_never:
+            fs_binary_extra_options += " --cache never"
+            fs_binary_extra_options_targetfs += " --cache never"
+        - cache_mode_metadata:
+            fs_binary_extra_options += " --cache metadata"
+            fs_binary_extra_options_targetfs += " --cache metadata"
+    variants:
+        - @default:
+        - writeback:
+            only cache_mode_auto cache_mode_always
+            migrate_parameters = "{'downtime-limit': 30000}"
+            fs_binary_extra_options += " --writeback"
+            fs_binary_extra_options_targetfs += " --writeback"
+        - dio:
+            only cache_mode_never
+            fs_binary_extra_options += " --allow-direct-io"
+            fs_binary_extra_options_targetfs += " --allow-direct-io"
+    variants:
+        - find_paths:
+        - file_handles:
+            required_virtiofsd_version = [1.13.0,)
+            virtiofsd_version_cmd = rpm -q virtiofsd | cut -d- -f2
+            fs_binary_extra_options_targetfs = " --modcaps=+dac_read_search"
+            variants:
+                - never:
+                    fs_binary_extra_options += " --inode-file-handles=never --migration-mode=file-handles --modcaps=+dac_read_search"
+                - mandatory:
+                    fs_binary_extra_options += " --inode-file-handles=mandatory --migration-mode=file-handles --modcaps=+dac_read_search"
+    variants:
+        - @default:
+        - multifs:
+            only find_paths.default.cache_mode_auto
+
+            filesystems += " fs2"
+            filesystems_migration += " targetfs2"
+            fs_binary_extra_options_targetfs2 = " --cache auto --modcaps=+dac_read_search"
+
+            # Setup the 2nd nfs server
+            export_dir_fs2 = /var/tmp/virtio_fs_test_nfs2
+            nfs_mount_dir_fs2 = /var/mnt/fs2
+            nfs_mount_dir_target_fs2 = /var/mnt/targetfs2
+
+            fs_source_dir_fs2 = ${nfs_mount_dir_fs2}
+            fs_target_fs2 = myfs2
+            fs_dest_fs2 = /mnt/${fs_target_fs2}
+            fs_source_dir_targetfs2 = ${nfs_mount_dir_target_fs2}
+            fs_target_targetfs2 = ${fs_target_fs2}
+            fs_dest_targetfs2 = ${fs_dest_fs2}
+
+            pre_command += " && mkdir -p ${nfs_mount_dir_target_fs2}"
+            post_command += " && rm -rf ${nfs_mount_dir_target_fs2} ${export_dir_fs2}"
+            fio_filename = "%s/${test_file}.fio:%s/${test_file}.fio"
+            Windows:
+                clone_master = yes
+                master_images_clone = image1
+                remove_image_image1 = yes
+                viofs_svc_name = WinFSP.Launcher
+                i386, i686:
+                    cmd_path = 'C:\Program Files'
+                    win_type = x86
+                x86_64:
+                    cmd_path = 'C:\Program Files (x86)'
+                    win_type = x64
+                viofs_sc_create_cmd = '"${cmd_path}\WinFsp\bin\fsreg.bat" virtiofs "${viofs_exe_path}" "-t %1 -m %2"'
+                instance_start_cmd = '"${cmd_path}\WinFsp\bin\launchctl-${win_type}.exe" start virtiofs viofs%s %s %s'
+                instance_stop_cmd = '"${cmd_path}\WinFsp\bin\launchctl-${win_type}.exe" stop virtiofs viofs%s'
+                volume_label_fs = X:
+                volume_label_fs2 = Y:
+                fio_filename = "%s\:\${test_file}.fio:%s\:\${test_file}.fio"
diff --git a/qemu/tests/virtio_fs_nfs_migration.py b/qemu/tests/virtio_fs_nfs_migration.py
new file mode 100644
index 0000000000..8b6dc54f94
--- /dev/null
+++ b/qemu/tests/virtio_fs_nfs_migration.py
@@ -0,0 +1,320 @@
+import ast
+import re
+
+from avocado.utils import process
+from virttest import env_process, error_context, nfs, utils_disk, utils_misc, utils_test
+from virttest.utils_version import VersionInterval
+
+from provider import virtio_fs_utils
+from provider.storage_benchmark import generate_instance
+
+
+@error_context.context_aware
+def run(test, params, env):
+    """
+    Basic migration test with diffirent cache modes and i/o over nfs
+    Steps:
+        1. Setup one or two local nfs servers and mount each exported dir
+           to two different mountpoints, one is for source vm and the other
+           one is for the target vm, e.g. fs and targetfs
+        2. Run the virtiofsd daemon to share fs with different cache modes
+        3. Boot the source vm with the virtiofs device in step2
+        4. Mount the virtiofs targets inside the guest
+        5. Create a file under each virtiofs and get the md5, run fio
+        6. Run the virtiofsd daemon to share the targetfs
+        7. Boot the target vm with the virtiofs device in step6
+        8. Do live migration from the source vm to the target vm
+        9. No error occurs, the virtiofs should be mounted automatically,
+           the file md5 should keep the same, fio should still be running
+
+    :param test: QEMU test object.
+    :param params: Dictionary with the test parameters.
+    :param env: Dictionary with test environment.
+    """
+
+    def create_service(s):
+        if os_type == "windows":
+            error_context.context("Create virtiofs service in guest.", test.log.info)
+
+            driver_name = params["driver_name"]
+            s = utils_test.qemu.windrv_check_running_verifier(s, vm, test, driver_name)
+            viofs_svc_name = params["viofs_svc_name"]
+            virtio_fs_utils.create_viofs_service(
+                test, params, s, service=viofs_svc_name
+            )
+        return s
+
+    def delete_service():
+        if os_type == "windows":
+            error_context.context("Delete virtiofs service in guest.", test.log.info)
+            s = vm.wait_for_login()
+            virtio_fs_utils.delete_viofs_serivce(test, params, s)
+            if s:
+                s.close()
+
+    def start_service(s):
+        def start_multifs_instance(fs_tag, fs_target, fs_volume_label):
+            """
+            Only for windows and only for multiple shared directory.
+            """
+            error_context.context(
+                "MultiFS-%s: Start virtiofs instance with"
+                " tag %s to %s." % (fs_tag, fs_target, fs_volume_label),
+                test.log.info,
+            )
+            instance_start_cmd = params["instance_start_cmd"]
+            output = s.cmd_output(
+                instance_start_cmd % (fs_target, fs_target, fs_volume_label)
+            )
+            if re.search("KO.*error", output, re.I):
+                test.fail(
+                    "MultiFS-%s: Start virtiofs instance failed, "
+                    "output is %s." % (fs_tag, output)
+                )
+
+        for fs in params.objects("filesystems"):
+            fs_params = params.object_params(fs)
+
+            fs_target = fs_params["fs_target"]
+            fs_dest = fs_params["fs_dest"]
+
+            if os_type == "linux":
+                utils_misc.make_dirs(fs_dest, s)
+                error_context.context(
+                    "Mount virtiofs target %s to %s inside"
+                    " guest." % (fs_target, fs_dest),
+                    test.log.info,
+                )
+                if not utils_disk.mount(fs_target, fs_dest, "virtiofs", session=s):
+                    utils_misc.safe_rmdir(fs_dest, session=s)
+                    test.fail(f"Failed to mount virtiofs {fs_target}")
+            else:
+                if params["viofs_svc_name"] == "VirtioFsSvc":
+                    error_context.context(
+                        "Start virtiofs service in guest.", test.log.info
+                    )
+                    debug_log_operation = params.get("debug_log_operation")
+                    if debug_log_operation:
+                        s = virtio_fs_utils.operate_debug_log(
+                            test, params, s, vm, debug_log_operation
+                        )
+                    virtio_fs_utils.start_viofs_service(test, params, s)
+                else:
+                    error_context.context(
+                        "Start winfsp.launcher instance in guest.", test.log.info
+                    )
+                    fs_volume_label = fs_params["volume_label"]
+                    start_multifs_instance(fs, fs_target, fs_volume_label)
+
+                fs_dest = "%s" % virtio_fs_utils.get_virtiofs_driver_letter(
+                    test, fs_target, s
+                )
+
+            guest_mnts[fs_target] = fs_dest
+        return s
+
+    def stop_service(s):
+        error_context.context("Stop virtiofs service in guest.", test.log.info)
+        if os_type == "linux":
+            for fs_target, fs_dest in guest_mnts.items():
+                utils_disk.umount(fs_target, fs_dest, "virtiofs", session=s)
+                utils_misc.safe_rmdir(fs_dest, session=s)
+        else:
+            if params["viofs_svc_name"] == "WinFSP.Launcher":
+                for fs_target in guest_mnts.keys():
+                    error_context.context(
+                        "Unmount fs with WinFsp.Launcher.z", test.log.info
+                    )
+                    instance_stop_cmd = params["instance_stop_cmd"]
+                    s.cmd(instance_stop_cmd % fs_target)
+            else:
+                if guest_mnts:
+                    virtio_fs_utils.stop_viofs_service(test, params, s)
+        if s:
+            s.close()
+
+    def start_io(s):
+        def do_fio(fio, dirs):
+            error_context.context("Start fio process", test.log.info)
+            tmo = params.get_numeric("fio_runtime", 1800)
+            f = (
+                params["fio_filename"] % tuple(dirs)
+                if len(dirs) > 1
+                else params["fio_filename"] % dirs[0]
+            )
+            bg_test = utils_test.BackgroundTest(
+                fio.run, (params["fio_options"] % f, tmo)
+            )
+            bg_test.start()
+
+        def dd_file():
+            fs_dest = guest_mnts[fs_target]
+            guest_file = fs_params["guest_file"] % fs_dest
+
+            error_context.context(
+                "Create the file %s get its md5" % guest_file, test.log.info
+            )
+            io_timeout = params.get_numeric("io_timeout", 300)
+            s.cmd(params["cmd_dd"] % guest_file, io_timeout)
+
+            cmd_md5 = params["cmd_md5"] % fs_dest
+            md5 = s.cmd_output(cmd_md5, io_timeout).strip().split()[0]
+            guest_files_md5[fs_target] = md5
+            test.log.debug("The guest file md5: %s", md5)
+
+        fs_list = list()
+        for fs in params.objects("filesystems"):
+            fs_params = params.object_params(fs)
+            fs_target = fs_params["fs_target"]
+            fs_list.append(guest_mnts[fs_target])
+            dd_file()
+
+        fio = generate_instance(params, vm, "fio")
+        do_fio(fio, fs_list)
+        return fio
+
+    def stop_io():
+        if guest_fio_object:
+            error_context.context("Stop fio process", test.log.info)
+            guest_fio_object.clean(force=True)
+
+    def test_migration():
+        def check_fio_running():
+            error_context.context("Check fio is running after migration", test.log.info)
+            fio_name = params["fio_name"]
+            out = session.cmd_output(params["cmd_chk_fio"])
+            test.log.debug("Status of fio process: %s", out)
+
+            procs = re.findall(fio_name, out, re.M | re.I)
+            if not procs:
+                test.fail("Failed to get any running fio process")
+            else:
+                # On linux, the count should be numjobs+1
+                # because ps lists fio and all its subprocess
+                count_fio_processes = params.get_numeric("numjobs") + 1
+                if os_type == "windows":
+                    # On Windows, the count should be 1
+                    # because tasklist only lists the fio.exe process
+                    count_fio_processes = 1
+
+                if len(procs) != count_fio_processes:
+                    test.fail("Failed to get all running fio process")
+
+        def check_service_activated():
+            error_context.context(
+                "Check virtiofs service activated after migration",
+                test.log.info,
+            )
+            tmo = params.get_numeric("active_timeout", 10)
+            if os_type == "linux":
+                for fs_target, fs_dest in guest_mnts.items():
+                    if not utils_misc.wait_for(
+                        lambda: utils_disk.is_mount(
+                            fs_target, fs_dest, "virtiofs", None, True, session
+                        ),
+                        tmo,
+                    ):
+                        test.fail(f"Failed to mount {fs_target}")
+            else:
+                for fs_target in guest_mnts.keys():
+                    vol_lable = virtio_fs_utils.get_virtiofs_driver_letter(
+                        test, fs_target, session
+                    )
+                    test.log.debug(
+                        "Fs target %s mounted on volume %s", fs_target, vol_lable
+                    )
+
+        def check_file_md5():
+            error_context.context("Check file md5 after migration", test.log.info)
+            for fs_target, original_md5 in guest_files_md5.items():
+                fs_dest = guest_mnts[fs_target]
+                cmd_md5 = params["cmd_md5"] % fs_dest
+                md5 = session.cmd_output(cmd_md5).strip().split()[0]
+                test.log.debug("File md5 after migration: %s", md5)
+
+                if md5 != original_md5:
+                    test.fail(f"Wrong file md5 found: {md5}")
+
+        # FIXME: Replace the vm's params to use a different shared virtio fs
+        vm.params["filesystems"] = vm.params["filesystems_migration"]
+        src_params = ast.literal_eval(params.get("migrate_parameters", "None"))
+        tgt_params = ast.literal_eval(params.get("target_migrate_parameters", "None"))
+        migrate_parameters = (src_params, tgt_params)
+        vm.migrate(migrate_parameters=migrate_parameters)
+        session = vm.wait_for_login()
+
+        check_service_activated()
+        check_fio_running()
+        check_file_md5()
+
+        return session
+
+    def setup_local_nfs():
+        for fs in params.objects("filesystems"):
+            fs_params = params.object_params(fs)
+            nfs_config[fs] = dict()
+
+            error_context.context(
+                "Setup the nfs server, mount it to two dirs", test.log.info
+            )
+
+            # Setup the local nfs server and mount it to nfs_mount_dir
+            nfs_obj = nfs.Nfs(fs_params)
+            nfs_obj.setup()
+            nfs_config[fs]["server"] = nfs_obj
+
+            # Mount the local nfs server to nfs_mount_dir_target
+            target_params = fs_params.copy()
+            target_params["nfs_mount_dir"] = fs_params["nfs_mount_dir_target"]
+            target_params["setup_local_nfs"] = "no"
+            nfs_target = nfs.Nfs(target_params)
+            test.log.debug("Mount %s to %s", nfs_target.mount_src, nfs_target.mount_dir)
+            nfs_target.mount()
+            nfs_config[fs]["target"] = nfs_target
+
+    def cleanup_local_nfs():
+        error_context.context("Umount all and stop nfs server", test.log.info)
+        for obj in nfs_config.values():
+            if "target" in obj:
+                obj["target"].umount()
+            if "server" in obj:
+                obj["server"].cleanup()
+
+    def check_vertiofsd_version():
+        version = params.get("required_virtiofsd_version")
+        if version:
+            v = process.getoutput(params["virtiofsd_version_cmd"], shell=True).strip()
+            test.log.debug("The virtiofsd version: %s", v)
+            if v not in VersionInterval(version):
+                test.cancel(f"The required virtiofsd version >= {version}")
+
+    guest_mnts = dict()
+    guest_files_md5 = dict()
+    guest_fio_object = None
+    os_type = params["os_type"]
+    nfs_config = dict()
+    vm = None
+    session = None
+
+    check_vertiofsd_version()
+    try:
+        setup_local_nfs()
+        env_process.process(
+            test, params, env, env_process.preprocess_image, env_process.preprocess_vm
+        )
+        vm = env.get_vm(params.get("main_vm"))
+        vm.verify_alive()
+        session = vm.wait_for_login()
+        session = create_service(session)
+        session = start_service(session)
+        guest_fio_object = start_io(session)
+        session = test_migration()
+    finally:
+        try:
+            stop_io()
+            stop_service(session)
+            delete_service()
+        finally:
+            if vm:
+                vm.destroy()
+            cleanup_local_nfs()

From 1b8661d90b6cc3750ae9faa8647067d7f082c2e5 Mon Sep 17 00:00:00 2001
From: Zhenchao Liu <zhencliu@redhat.com>
Date: Tue, 5 Nov 2024 11:28:31 +0800
Subject: [PATCH 2/2] storage_benchmark: Exe suffix is needed when killing
 process on windows

A full process name is required for the TASKKILL command, e.g.
TASKKILL /F /IM fio.exe /T

Signed-off-by: Zhenchao Liu <zhencliu@redhat.com>
---
 provider/storage_benchmark.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/provider/storage_benchmark.py b/provider/storage_benchmark.py
index 06b71b0493..62dfaea142 100644
--- a/provider/storage_benchmark.py
+++ b/provider/storage_benchmark.py
@@ -134,8 +134,9 @@ def __kill_procs(self, session):
         :param session: vm session
         :type session: aexpect.client.ShellSession
         """
-        LOG_JOB.info("Killing all %s processes by force.", self.name)
-        session.cmd_output(self._kill_pid % self.name, timeout=120)
+        proc_name = self.name if self.os_type == "linux" else f"{self.name}.exe"
+        LOG_JOB.info("Killing all %s processes by force.", proc_name)
+        session.cmd_output(self._kill_pid % proc_name, timeout=120)
 
     def __remove_env_files(self, session, timeout=300):
         """