From a49d408c9349fdf2bc5914785e9e28a9a559fc3e Mon Sep 17 00:00:00 2001
From: xuemin <xuli@redhat.com>
Date: Thu, 18 May 2023 10:41:00 +0800
Subject: [PATCH] Add basic nested hyperv on kvm test case

1. Start L1 Windows VM (w2022,2019,2016,win10)
2. Start L2 BIOS/UEFI VM via booting up fedora image

ID: 2219365
Signed-off-by: xuemin <xuli@redhat.com>
---
 qemu/deps/nested_hyperv/hyperv_env.ps1  | 170 +++++++++
 qemu/deps/nested_hyperv/hyperv_run.ps1  |  70 ++++
 qemu/deps/nested_hyperv/hyperv_sets.ps1 | 436 ++++++++++++++++++++++++
 qemu/tests/cfg/nested_hyperv_on_kvm.cfg |  17 +
 qemu/tests/nested_hyperv_on_kvm.py      |  97 ++++++
 5 files changed, 790 insertions(+)
 create mode 100644 qemu/deps/nested_hyperv/hyperv_env.ps1
 create mode 100644 qemu/deps/nested_hyperv/hyperv_run.ps1
 create mode 100644 qemu/deps/nested_hyperv/hyperv_sets.ps1
 create mode 100644 qemu/tests/cfg/nested_hyperv_on_kvm.cfg
 create mode 100644 qemu/tests/nested_hyperv_on_kvm.py

diff --git a/qemu/deps/nested_hyperv/hyperv_env.ps1 b/qemu/deps/nested_hyperv/hyperv_env.ps1
new file mode 100644
index 0000000000..032034cf9e
--- /dev/null
+++ b/qemu/deps/nested_hyperv/hyperv_env.ps1
@@ -0,0 +1,170 @@
+#######################################################################
+#
+#    Initialize, or provision Windows Server as follows:
+#  - The Hyper-V feature is installed.
+#  - The required Hyper-V vSwitche is created, e.g. Internal switch.
+#######################################################################
+
+Write-Host "Info: Start to build up enviroment for Hyper-V"
+
+$internalSwitchName = "Internal"
+$externalSwitchName = "External"
+
+function TestCommandExists ([String]$command){
+    try {
+        if(Get-Command $command -ErrorAction SilentlyContinue){
+            Write-Host "Info: $command exists";
+            return $true
+        }
+    }
+    Catch {
+        Write-Host "Info: $command does not exist";
+        return $false
+    }
+}
+
+function InstallHyperVPowershell(){
+    # Need to restart windows to take effect, restart action will be out of this script
+    if (TestCommandExists Get-WindowsFeature) {
+        $powershellFeature = Get-WindowsFeature -Name "Hyper-v-powershell"
+        if (-not $powershellFeature.Installed){
+            Install-windowsFeature -Name Hyper-v-powershell
+            if (-not $?){
+                Throw "Error: Unable to install Hyper-v module"
+            }
+            else{
+                Write-Host "Info: Have executed Install-windowsfeature successfully";
+            }
+        }
+    }
+}
+
+function CreateExternalSwitch()
+{
+    #
+    # We will only create an external switch if:
+    #  - The host only has a single physical NIC, and an external switch
+    #    does not already exist.
+    #  - The host has multiple physical NICs, but only one is connected,
+    #    and an external switch does not already exist.
+    #
+
+    Write-Host "Info: Checking for External vSwitch named '${externalSwitchName}'"
+    $externalSwitch = Get-VMSwitch -Name "${externalSwitchName}" -ErrorAction SilentlyContinue
+    if ($externalSwitch){
+        # A vSwitch named external already exists
+        Write-Host -f Yellow "Warning: The external vSwitch '${externalSwitchName}' already exists"
+        return
+    }
+
+    $adapters = Get-NetAdapter
+    $numPotentialNICs = 0
+    $potentialNIC = $null
+
+    foreach ($nic in $adapters){
+        # Make sure NIC is connected (MediaConnectState = 1)
+        # and the NIC is up (InterfaceOperationalStatus = 1)
+
+        if ($nic.InterfaceOperationalStatus -eq 1 -and $nic.MediaConnectState -eq 1){
+            $numPotentialNICs += 1
+            $potentialNIC = $nic.InterfaceDescription
+            Write-Host "Info: Potential NIC for external vSwitch = '${potentialNIC}'"
+        }
+    }
+
+    if ($numPotentialNICs -eq 0){
+        Write-Host -f Yellow "Warning: No potential NICs found to create an External vSwitch"
+        Write-Host -f Yellow "         You will need to manually create the external vSwitch"
+        exit 1
+    }
+    elseif ($numPotentialNICs -gt 1){
+        Write-Host -f Yellow "Warning: There are more than one physical NICs that could be used"
+        Write-Host -f Yellow "         with an external vSwitch.  You will need to manually"
+        Write-Host -f Yellow "         create the external vSwitch"
+        exit 1
+    }
+
+    #
+    # Create an External NIC using the one potential physical NIC
+    #
+    New-VMSwitch -Name "${externalSwitchName}" -NetAdapterInterfaceDescription "${potentialNIC}"
+    if (-not $?){
+        Write-Host "Error: Unable to create external vSwitch using NIC '${potentialNIC}'"
+        exit 1
+    }
+    Write-Host "Info: External vSwitch '${externalSwitchName}' was created, using NIC"
+    Write-Host "      ${potentialNIC}"
+}
+
+function CreateInternalSwitch(){
+    #
+    # See if an internal switch named 'Internal' already exists.
+    # If not, create it
+    #
+    Write-Host "Info: Checking for Internal vSwitch named '${internalSwitchName}'"
+    $internalSwitch = Get-VMSwitch -Name "${internalSwitchName}" -ErrorAction SilentlyContinue
+    if (-not $internalSwitch){
+        New-VMSwitch -Name "${internalSwitchName}"  -SwitchType Internal
+        if (-not $?){
+            Throw "Error: Unable to create Internal switch"
+        }
+
+        Get-NetAdapter -Name "${internalSwitchName}" | New-NetIPAddress -AddressFamily ipv4 -IPAddress 192.168.0.1 -PrefixLength 24
+        Write-Host "Info: Internal vSwitch '${internalSwitchName}' was created with IP range 192.168.0.1/24"
+    }
+    else{
+        Write-Host -f Yellow "Warning: The Internal vSwitch '${internalSwitchName}' already exists"
+    }
+}
+
+function InstallRolesAndFeatures(){
+    if ( TestCommandExists Get-WindowsFeature){
+        $hypervFeature = Get-WindowsFeature -Name "Hyper-V"
+        if (-not $hypervFeature.Installed){
+            $feature = Install-WindowsFeature -Name "Hyper-V" -IncludeAllSubfeature -IncludeManagementTools
+            if (-not $feature.Success){
+                Throw "Error: Unable to install the Hyper-V roles"
+            }else{
+                Write-Host "Info: Have executed Install-WindowsFeature successfully"
+            }
+        }
+    }
+    else{
+        # For windows 10 and 11
+        $hypervFeature = Get-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V-All
+        if (-not $hypervFeature.Enalbed){
+            $feature=Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V-All -NoRestart
+            if (-not $feature.Online){
+                Throw "Error: Unable to enable the Hyper-V roles }"
+            }
+            else{
+                Write-Host "Info: Have executed Enable-WindowsOptionalFeature successfully"
+            }
+        }
+    }
+}
+
+#######################################################################
+#
+# Main script body
+#
+#######################################################################
+
+try {
+    # Install any roles and features
+    Write-Host "Info: Start to install for Hyper-V Powershell"
+    InstallHyperVPowershell
+
+    Write-Host "Info: Start to install Hyper-V role"
+    InstallRolesAndFeatures
+    #CreateExternalSwitch
+    #CreateInternalSwitch
+}
+catch {
+    $msg = $_.Exception.Message
+    Write-Host -f Red "Error: Unable to provision the host"
+    Write-Host -f Red "${msg}"
+    exit 1
+}
+
+exit 0
diff --git a/qemu/deps/nested_hyperv/hyperv_run.ps1 b/qemu/deps/nested_hyperv/hyperv_run.ps1
new file mode 100644
index 0000000000..e87bebb8df
--- /dev/null
+++ b/qemu/deps/nested_hyperv/hyperv_run.ps1
@@ -0,0 +1,70 @@
+param([String] $vhdx_path='C:\fedora.vhdx')
+# Main script to get vhdx and start gen1/gen2 vm.
+
+$PrivateSwitchName = "Private"
+
+function CreatePrivateSwitches(){
+    #
+    # See if an Private switch named 'Private' already exists.
+    # If not, create it
+    #
+    Write-Host "Info : Checking for Private vSwitch named '${PrivateSwitchName}'"
+    $privateSwitch = Get-VMSwitch -Name "${privateSwitchName}" -ErrorAction SilentlyContinue
+    if (-not $privateSwitch){
+        New-VMSwitch -Name "${privateSwitchName}" -SwitchType Private
+        if (-not $?){
+            Throw "Error: Unable to create Private switch"
+        }
+        Write-Host "Info: Private vSwitch '${privateSwitchName}' is created"
+    }
+    else{
+        Write-Host -f Yellow "Warning: the vSwitch '$privateSwitchName}' already exists"
+    }
+}
+
+# Create internal switch if does not exist
+CreatePrivateSwitches
+
+$vhdx_name=[System.IO.Path]::GetFileNameWithoutExtension($vhdx_path)
+$vhdx_folder=[System.IO.Path]::GetDirectoryName($vhdx_path)
+
+# $vm_name_gen1="fedora-gen1"
+# $vm_name_gen2="fedora-gen2"
+$vm_name_gen1= $vhdx_name +"-gen1"
+$vm_name_gen2= $vhdx_name +"-gen2"
+
+# $vm_gen1_vhdx="C:\fedora-gen1.vhdx"
+# $vm_gen2_vhdx="C:\fedora-gen2.vhdx"
+$vm_vhdx_gen1=$vhdx_folder+$vm_name_gen1+".vhdx"
+$vm_vhdx_gen2=$vhdx_folder+$vm_name_gen2+".vhdx"
+
+# Remove old vm if have
+powershell C:\nested-hyperv-on-kvm\hyperv_sets.ps1 -action del -vm_name $vm_name_gen1
+powershell C:\nested-hyperv-on-kvm\hyperv_sets.ps1 -action del -vm_name $vm_name_gen2
+start-sleep 2
+
+if (-not (Test-Path $vhdx_path)){
+    write-host "Error: the vhdx file $vhdx_path does not exist."
+    exit 1
+}
+
+start-sleep 2
+Copy-Item $vhdx_path $vm_vhdx_gen1
+start-sleep 2
+Copy-Item $vhdx_path $vm_vhdx_gen2
+start-sleep 2
+Remove-Item $vhdx_path
+
+#Start new gen1 vm
+powershell C:\nested-hyperv-on-kvm\hyperv_sets.ps1 -action clone -vm_name $vm_name_gen1 -vhd_path $vhdx_folder
+if ($? -eq $false) {
+    write-host "Error: fail to start vm $vm_name_gen1"
+    exit 1
+}
+start-sleep 2
+powershell C:\nested-hyperv-on-kvm\hyperv_sets.ps1 -action clone -vm_name $vm_name_gen2 -vhd_path $vhdx_folder -gen2
+if ($? -eq $false) {
+    write-host "Error: fail to start vm $vm_name_gen2"
+    exit 1
+}
+exit 0
diff --git a/qemu/deps/nested_hyperv/hyperv_sets.ps1 b/qemu/deps/nested_hyperv/hyperv_sets.ps1
new file mode 100644
index 0000000000..f13b8ca2b5
--- /dev/null
+++ b/qemu/deps/nested_hyperv/hyperv_sets.ps1
@@ -0,0 +1,436 @@
+param (
+    [CmdletBinding()]
+    [String] $action,
+    [String] $vm_name,
+    [Parameter(Mandatory=$false)][String] $vhd_path,
+    [Parameter(Mandatory=$false)][Switch] $gen2,
+    [Parameter(Mandatory=$false)][String] $isoPath,
+    [Parameter(Mandatory=$false)][String] $isoKSPath
+)
+
+function VMStop([String] $vmName){
+    $vm = get-vm $vmName
+    if ( $vm.State -eq "Off"){
+        write-host "Info: $vmName is stopped"
+        return $true
+    }
+
+    stop-vm $vmName -force
+    if ( $? -ne $true ){
+        write-host "Error: stop-VM $vm failed"
+        return $false
+    }
+
+    $timeout = 100
+    while ($timeout -gt 0){
+        $vm = get-vm $vmName
+        if ($vm.state -eq "Off"){
+            write-host "Info: $vmName state is Off now"
+            break
+        }
+        else{
+            write-host "Info: $vmName state is not Off, waiting..."
+            start-sleep -seconds 1
+            $timeout -= 1
+        }
+    }
+    if ( $timeout -eq 0 -and $vm.state -ne "Off"){
+        write-host "Error: Stop $vm failed (timeout=$timeout)"
+        return $false
+    }
+
+    return $true
+}
+
+function VMRemove([String]$vmName){
+    # Check the vm, if exists, then delete
+    Get-VM -Name $vmName -ErrorAction "SilentlyContinue" | out-null
+    if ( $? ){
+        # check vm is not Running
+        if ( $(Get-VM -Name $vmName).State -eq "Running"){
+            VMStop($vmName)
+        }
+        write-host "Info: Remove $vmName"
+        # Get latest snapshot
+        Get-VMSnapshot -VMName $vmName | Remove-VMSnapshot
+        start-sleep -s 2
+        $vhdpath = (get-vm -vmName $vmName | Select-Object vmid | get-vhd ).ParentPath
+        if ( -not $vhdpath ) {
+            $vhdpath = (get-vm -vmName $vmName | Select-Object vmid | get-vhd ).Path
+        }
+
+        Remove-VM -Name $vmName  -Confirm:$false -Force
+        start-sleep -s 2
+        remove-item -Path $vhdpath -force
+
+        if ($?){
+            write-output "Info: Remove VM succussfully"
+        }
+        else{
+            Write-Output "Error: Remove VM failed"
+        }
+    }
+}
+
+function GetIPv4ViaKVP([String] $vmName){
+    $vmObj = Get-WmiObject -Namespace root\virtualization\v2 -Query "Select * From Msvm_ComputerSystem Where ElementName=`'$vmName`'"
+    if (-not $vmObj){
+        write-error -Message "GetIPv4ViaKVP: Unable to create Msvm_ComputerSystem object" -Category ObjectNotFound -ErrorAction SilentlyContinue
+        return $null
+    }
+
+    $kvp = Get-WmiObject -Namespace root\virtualization\v2 -Query "Associators of {$vmObj} Where AssocClass=Msvm_SystemDevice ResultClass=Msvm_KvpExchangeComponent"
+    if (-not $kvp){
+        write-error -Message "GetIPv4ViaKVP: Unable to create KVP exchange component" -Category ObjectNotFound -ErrorAction SilentlyContinue
+        return $null
+    }
+
+    $rawData = $Kvp.GuestIntrinsicExchangeItems
+    if (-not $rawData){
+        write-error -Message "GetIPv4ViaKVP: No KVP Intrinsic data returned" -Category ReadError -ErrorAction SilentlyContinue
+        return $null
+    }
+
+    $addresses = $null
+
+    foreach ($dataItem in $rawData){
+        $found = 0
+        $xmlData = [Xml] $dataItem
+        foreach ($p in $xmlData.INSTANCE.PROPERTY){
+            if ($p.Name -eq "Name" -and $p.Value -eq "NetworkAddressIPv4"){
+                $found += 1
+            }
+
+            if ($p.Name -eq "Data"){
+                $addresses = $p.Value
+                $found += 1
+            }
+
+            if ($found -eq 2){
+                $addrs = $addresses.Split(";")
+                foreach ($addr in $addrs){
+                    if ($addr.StartsWith("127.")){
+                        Continue
+                    }
+                    return $addr
+                }
+            }
+        }
+    }
+
+    write-error -Message "GetIPv4ViaKVP: No IPv4 address found for VM ${vmName}" -Category ObjectNotFound -ErrorAction SilentlyContinue
+    return $null
+}
+
+function WaitForVMToStartKVP([String] $vmName, [int] $timeout){
+    $waitTimeOut = $timeout
+    while ($waitTimeOut -gt 0){
+        $ipv4 = GetIPv4ViaKVP $vmName
+        if ($ipv4){
+            return $true
+        }
+
+        $waitTimeOut -= 10
+        Start-Sleep -s 10
+    }
+
+    write-error -Message "WaitForVMToStartKVP: VM ${vmName} did not start KVP within timeout period ($timeout)" -Category OperationTimeout -ErrorAction SilentlyContinue
+    return $false
+}
+
+function NewCheckpoint([String] $vmName, [String] $snapshotName){
+    write-host "Info: Stop $vmName before make snapshot"
+    VMStop $vmName
+
+    write-host "Info: make checkpoint of $vmName, checkpoint name is $snapshotName"
+    Checkpoint-VM -Name $vmName -SnapshotName $snapshotName
+
+    if ( $snapshotName -eq $(Get-VMSnapshot $vmName).Name){
+        write-host "Info: $snapshotName for $vmName is created successfully"
+        return $true
+    }
+    else{
+        return $false
+    }
+}
+
+function SetFirmwareProcessor([Switch]$gen2, [String]$vmName, [Int64]$cpuCount){
+    # If gen 2 vm, set vmfirmware secure boot disabled
+    if ($gen2){
+        # disable secure boot
+        Set-VMFirmware $vmName -EnableSecureBoot Off
+
+        if (-not $?){
+            write-host "Info: Set-VMFirmware $vmName secureboot failed"
+            return $false
+        }
+
+        write-host "Info: Set-VMFirmware $vmName secureboot successfully"
+    }
+
+    # set processor to 2, default is 1
+    Set-VMProcessor -vmName $vmName -Count $cpuCount
+
+    if (! $?){
+        write-host "Error: Set-VMProcessor $vmName  to $cpuCount failed"
+        return $false
+    }
+
+    if ((Get-VMProcessor -vmName $vmName).Count -eq $cpuCount){
+        write-host "Info: Set-VMProcessor $vmName to $cpuCount"
+    }
+}
+
+function NewVMFromVHDX([String]$vmPath, [Switch]$gen2, [String]$switchName, [String]$vmName, [Int64]$cpuCount, [Int64]$mem){
+    write-host "Info: Creating $vmName with $cpuCount CPU and ${mem}G memory."
+    # Convert GB to bytes because parameter -MemoryStartupByptes requires bytes
+    [Int64]$memory = 1GB * $mem
+
+    if ($gen2){
+        New-VM -Name "$vmName" -Generation 2 -BootDevice "VHD" -MemoryStartupBytes $memory -VHDPath $vmPath -SwitchName $switchName | Out-Null
+    }
+    else {
+        New-VM -Name "$vmName" -BootDevice "IDE" -MemoryStartupBytes $memory -VHDPath $vmPath -SwitchName $switchName | Out-Null
+    }
+
+    if (-not $?){
+        write-host "New-VM $vmName failed"
+        # rm new created disk
+        If (Test-Path $vmPath){
+            Remove-Item $vmPath
+        }
+        return $false
+    }
+    write-host "Info: New-VM $vmName successfully"
+
+    if ($gen2){
+        SetFirmwareProcessor -gen2 -vmName $vmName -cpuCount $cpuCount
+    }
+    else{
+            SetFirmwareProcessor -vmName $vmName -cpuCount $cpuCount
+        }
+    if (-not $?){
+        write-host "Error: fail to set the firmware and processor for VM $vmName"
+        return $false
+    }
+    return $true
+}
+
+function VMStart([String]$vmPath, [String]$vmName, [Bool]$gen2, [String]$switchName, [Int64]$cpuCount, [Int64]$mem){
+    write-host "Info: vmName is $vmName"
+
+    # Create vm based on new vhdx file
+    if ($gen2){
+        NewVMFromVHDX -vmPath "${vmPath}\${vmName}.vhdx" -gen2 $true -switchName $switchName -vmName $vmName -cpuCount $cpuCount -mem $mem
+    }
+    else{
+        NewVMFromVHDX -vmPath "${vmPath}\${vmName}.vhdx" -switchName $switchName -vmName $vmName -cpuCount $cpuCount -mem $mem
+    }
+    # Now Start the VM
+    write-host "Info: Starting VM $vmName."
+    Start-VM -Name $vmName
+    start-sleep -seconds 60
+
+    $timeout = 0
+    while ($timeout -lt 180) {
+    # Check the VMs heartbeat
+    $hb = Get-VMIntegrationService -VMName $VMName -Name "Heartbeat"
+    $vm = Get-VM $vmName
+    if ($($hb.Enabled) -eq "True" -and $($vm.Heartbeat) -eq "OkApplicationsUnknown") {
+        write-host "Info: Heartbeat detected for $vmName"
+        return $true
+    }
+    else {
+        start-sleep -seconds 10
+        $timeout = $timeout + 10
+    }
+    }
+
+    write-host "Test Failed: VM heartbeat not detected after wait for 4 mintus!"
+    write-host "Heartbeat not detected while the Heartbeat service is enabled, Heartbeat - $($vm.Heartbeat)"
+    return $false
+}
+
+function VMPowerOn([String]$vmName){
+    write-host "Info: vmName is $vmName"
+
+    # Now Start the VM
+    write-host "Info: Starting VM $vmName."
+    $timeout = 300
+    Start-VM -Name $vmName | out-null
+    WaitForVMToStartKVP -vmName $vmName -timeout $timeout
+    $vmIP = GetIPv4ViaKVP $vmName
+
+    if ($vmIP){
+        #Write-Output y | plink -l root -i ssh\3rd_id_rsa.ppk $vmIP "exit 0"
+        write-host "Info: Get $vmName IP = $vmIP"
+        return $vmIP
+    }
+    else{
+        ###############################################################
+        # Update for bug 2016 and 2012R2 Gen2 vm cannot install sometimes.
+        # 2019 host verison is 17763
+        ###############################################################
+
+        [int]$BuildNumber = (Get-CimInstance Win32_OperatingSystem).BuildNumber
+        write-host "INFO: current build number is $BuildNumber`n----------"
+        [int]$gen = (Get-VM -Name $vmName).Generation
+        if ( $gen -eq 2 -and  $BuildNumber -lt 17763 ) {
+            for ($count=1; $count -le 5; $count++){
+
+                write-host "Info: start to sleep 150 to check $vmName is really running or not"
+                start-sleep -seconds 150
+                $vmIP = GetIPv4ViaKVP $vmName
+                if ($vmIP) {
+                    return $vmIP
+                }
+                ##########################################################
+                # if $duration 0, try to start vm again, change for bug
+                ########################################################
+                else {
+                    write-host "Info : $vmName is being started again for bug - cannot enter kernel entry， retry $count"
+                    Start-VM -name $vmName | out-null
+                }
+            }
+            return $false
+        } # end of if gen2 vm
+    }
+    return $false
+}
+
+function VMInstall([String]$vmPath, [String]$vmName, [Bool]$gen2, [String]$switchName, [Int64]$cpuCount, [Int64]$mem, [String]$isopath, [String]$isoKsPath){
+    write-host "Info: Prepare to install vm, vmName is $vmName"
+    $startDTM = (Get-Date)
+
+    # Create vm based on new vhdx file by installing iso
+    if ($gen2){
+        NewVMFromISO -vmPath "${vmPath}\${vmName}.vhdx" -gen2 -switchName $switchName -vmName $vmName -cpuCount $cpuCount -mem $mem -isoPath $isopath -isoKsPath $isoKsPath
+    }
+    else{
+        NewVMFromISO -vmPath "${vmPath}\${vmName}.vhdx" -switchName $switchName -vmName $vmName -cpuCount $cpuCount -mem $mem -isoPath $isopath -isoKsPath $isoKsPath
+    }
+    # Now Start the VM
+    write-host "Info: Installing VM $vmName is started "
+
+    $timeout = 6000
+    Start-VM -Name $vmName
+    WaitForVMToStartKVP -vmName $vmName -timeout $timeout
+    $vmIP = GetIPv4ViaKVP $vmName
+
+    $endDTM = (Get-Date)
+
+    write-host "Summary: The installation executed $(($endDTM-$startDTM).TotalMinutes) minutes"
+    $vmVhdxPath= (get-vm -vmName $vmName | Select-Object vmid | get-vhd ).path
+
+    write-host "vmVhdxPath is $vmVhdxPath"
+    # VMStop -vmName $vmName
+
+    if ($vmIP){
+        #Write-Output y | plink -l root -i ssh\3rd_id_rsa.ppk $vmIP "exit 0"
+        write-host "Info: Get $vmName IP = $vmIP"
+        return $vmIP
+    }
+    else{
+        write-host "ERROR: Unable to get VM IP" -ErrorAction SilentlyContinue
+        #VMRemove -vmName $vmName
+        return $false
+    }
+}
+
+function NewVMFromISO([String]$vmPath, [switch]$gen2, [String]$switchName, [String]$vmName, [Int64]$cpuCount, [Int64]$mem, [String]$isoPath, [String]$isoKsPath){
+    write-host "Info: Creating $vmName with $cpuCount CPU and ${mem}G memory."
+    # Convert GB to bytes because parameter -MemoryStartupByptes requires bytes
+    [Int64]$memory = 1GB * $mem
+
+    # gen2 need specify -Generation
+    if ($gen2){
+        write-host "Info: New-VM -Name $vmName -MemoryStartupBytes $memory -Generation 2 -NewVHDSizeBytes 30GB -SwitchName $switchName -NewVHDPath $vmPath"
+
+        New-VM -Name $vmName -MemoryStartupBytes $memory -Generation 2 -NewVHDSizeBytes 30GB -SwitchName $switchName -NewVHDPath $vmPath
+    }
+    else
+    {
+        write-host "Info: New-VM -Name $vmName -MemoryStartupBytes $memory -NewVHDSizeBytes 30GB -SwitchName $switchName -NewVHDPath $vmPath -BootDevice IDE"
+
+        New-VM -Name $vmName -MemoryStartupBytes $memory -NewVHDSizeBytes 30GB -SwitchName $switchName -NewVHDPath $vmPath -BootDevice "IDE"
+    }
+
+    if (-not $?){
+        write-host "New-VM $vmName failed"
+        # rm new created disk
+        If (Test-Path $vmPath){
+            Remove-Item $vmPath
+        }
+        return $false
+    }
+    write-host "Info: New-VM $vmName successfully"
+
+    if ($gen2) {
+        SetFirmwareProcessor -gen2 -vmName $vmName -cpuCount $cpuCount
+    }
+    else{
+            SetFirmwareProcessor -vmName $vmName -cpuCount $cpuCount
+        }
+    if (-not $?){
+        write-host "Error: fail to set the firmware and processor for VM $vmName"
+        return $false
+    }
+    Get-VMDvdDrive -VMName $vmName -ControllerNumber 1 | Remove-VMDvdDrive
+    if (! $gen2) {
+        # put the two isos file in the same controller, to make the iso inject
+        Add-VMDvdDrive -VMName $vmName -Path $isoPath -ControllerNumber 1 -ControllerLocation 0 -Confirm:$False
+	    start-sleep -s 1
+        Add-VMDvdDrive -VMName $vmName -Path $isoKsPath -ControllerNumber 1 -ControllerLocation 1 -Confirm:$False
+    } else {
+        Add-VMDvdDrive -VMName $vmName -Path $isoPath -ControllerNumber 0 -ControllerLocation 1 -Confirm:$False
+	    start-sleep -s 1
+        Add-VMDvdDrive -VMName $vmName -Path $isoKsPath -ControllerNumber 0 -ControllerLocation 2 -Confirm:$False
+    }
+
+    if ( ! $? ){
+        write-host "Error: Set dvd drive to vm $vmName failed"
+        return $false
+    } else{
+        write-host "Info: Add dvd to vm $vmName $isoPath and $isoKsPath"
+    }
+
+    return $true
+}
+
+# Test command
+#.\hyperv_set.ps1 -action install -vm_name test_install -vhd_path .\vhdpath\
+#  -isopath C:\test.iso -isokspath C:\Temp\ks.iso
+#.\hyperv_set.ps1 -action clone -vm_name testvm -vhd_path C:\ -gen2
+# note: -vhd_path is folder path
+
+$DebugPreference = "Continue"
+
+#Set-Variable -Name switchName -Value "External" -Option constant -Scope Script
+Set-Variable -Name switchName -Value "Private" -Option constant -Scope Script
+Set-Variable -Name arch -Value "x86_64" -Option constant -Scope Script
+Set-Variable -Name snapshotName -Value "ICABase" -Option constant -Scope Script
+
+switch ($action){
+    "del"{
+        VMRemove $vm_name
+    }
+    "clone"{   # create 2 vcpu and 1G memory
+        $ret = VMStart $vhd_path $vm_name $gen2 $switchName 2 1
+        write-host "Infor: return value of VMStart for $vm_name is $ret"
+    }
+    "snapshot"{
+        NewCheckpoint -vmName $vm_name -snapshotName $snapshotName
+    }
+    "install"{
+        # return $ret
+        $ret = VMInstall $vhd_path $vm_name $gen2 $switchName 2 2 $isopath $isoKSPath
+        $ret = "$($ret[-1])".trim()
+        #write-host "$($ret[-1])"
+        return $ret
+    }
+    "poweron"{
+        # poweron
+        $ret = VMPowerOn $vm_name
+        return "$($ret[-1])".trim()
+    }
+}
diff --git a/qemu/tests/cfg/nested_hyperv_on_kvm.cfg b/qemu/tests/cfg/nested_hyperv_on_kvm.cfg
new file mode 100644
index 0000000000..46bbc9e48c
--- /dev/null
+++ b/qemu/tests/cfg/nested_hyperv_on_kvm.cfg
@@ -0,0 +1,17 @@
+- nested_hyperv_on_kvm:
+    type = nested_hyperv_on_kvm
+    only x86_64
+    only Windows
+    download_url = "https://download.fedoraproject.org/pub/fedora/linux/releases/38/Cloud/x86_64/images/Fedora-Cloud-Base-38-1.6.x86_64.qcow2"
+    md5value = "53ddfe7b28666d5ddc55e93ff06abad2"
+    vhdx_dest = "C:\\fedora.vhdx"
+    nested_dest = "C:\\nested-hyperv-on-kvm"
+    cpu_model_flags += ",hv_reset,hv_crash,hv-no-nonarch-coresharing=auto"
+    Host_RHEL.m9:
+        cpu_model_flags += ",hv_emsr_bitmap"
+
+    HostCpuVendor.intel:
+        cpu_model_flags += ",hv_evmcs,+vmx"
+
+    HostCpuVendor.amd:
+        cpu_model_flags += ",+svm"
diff --git a/qemu/tests/nested_hyperv_on_kvm.py b/qemu/tests/nested_hyperv_on_kvm.py
new file mode 100644
index 0000000000..811cb79685
--- /dev/null
+++ b/qemu/tests/nested_hyperv_on_kvm.py
@@ -0,0 +1,97 @@
+import time
+import os
+
+from avocado.utils import download
+from avocado.utils import process
+from virttest import data_dir
+from virttest import error_context
+
+
+@error_context.context_aware
+def run(test, params, env):
+    """
+    Nested Hyper-V on KVM:
+    This case is used to test in L1 Windows VM, start L2 BIOS/UEFI Linux VM via Fedora image
+
+    :param test: QEMU test object
+    :param params: Dictionary with the test parameters
+    :param env: Dictionary with test environment
+    """
+
+    def get_vhdx():
+        test.log.info("start download fedora image")
+
+        download_url = params.get("download_url")
+        image_dir = params.get("images_base_dir", data_dir.get_data_dir())
+        md5value = params.get("md5value")
+        vhdx_dest = params.get("vhdx_dest")
+        test.log.info("Parameters: %s %s %s %s" % (download_url, image_dir, md5value, vhdx_dest))
+        image_name = os.path.basename(download_url)
+        image_path = os.path.join(image_dir, image_name)
+        vhdx_name = image_name.replace('qcow2', 'vhdx')
+        vhdx_path = os.path.join(image_dir, vhdx_name)
+
+        download.get_file(download_url, image_path, hash_expected=md5value)
+
+        test.log.info("Covert fedora image to vhdx")
+        cmd_covert = "qemu-img convert -O vhdx " + image_path + " " + vhdx_path
+
+        status, output = process.getstatusoutput(cmd_covert, timeout)
+        if status != 0:
+            test.error("qemu-img convert failed, status: %s, output: %s" % (status, output))
+        vm.copy_files_to(vhdx_path, vhdx_dest, timeout=300)
+
+    vm = env.get_vm(params["main_vm"])
+    vm.verify_alive()
+    timeout = int(params.get("login_timeout", 600))
+    session = vm.wait_for_login(timeout=timeout)
+
+    test.log.info("Prepare vhdx file")
+    get_vhdx()
+
+    need_reboot = 0
+    status = session.cmd_status("powershell Get-VM \
+        -ErrorAction SilentlyContinue", timeout=120, safe=True)
+
+    if status:
+        need_reboot = 1
+        test.log.info("Hyper-V powershell module does not install")
+    else:
+        test.log.info("Hyper-V powershell module has been installed")
+
+    nested_dest = params.get("nested_dest")
+    path_cmd = r"powershell Remove-Item %s -recurse -force -ErrorAction SilentlyContinue" % nested_dest
+
+    try:
+        session.cmd(path_cmd)
+    except:
+        test.log.info("catch error when remove folder, ignore it")
+
+    time.sleep(10)
+
+    # nested_dest=r"C:\nested-hyperv-on-kvm"
+    # copy via deps
+    hyperv_source = os.path.join(data_dir.get_deps_dir(), "nested_hyperv")
+    vm.copy_files_to(hyperv_source, nested_dest)
+
+    # set RemoteSigned policy mainly for windows 10/11, it is default for windows server
+    session.cmd("powershell Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Force")
+    # powershell C:\nested-hyperv-on-kvm\hyperv_env.ps1
+    status, output = session.cmd_status_output(r"powershell %s\hyperv_env.ps1" % nested_dest, timeout=1200)
+    if status != 0:
+        test.error("Setup Hyper-v enviroment error: %s", output)
+    else:
+        test.log.info("Setup Hyper-v enviroment pass: %s", output)
+
+    if need_reboot:
+        test.log.info("VM will reboot to make Hyper-V powershell module installation work")
+        session = vm.reboot(session, timeout=360)
+
+    time.sleep(5)
+    # powershell C:\nested-hyperv-on-kvm\hyperv_run.ps1
+    status, output = session.cmd_status_output(r"powershell %s\hyperv_run.ps1" % nested_dest, timeout=1800)
+    if status != 0:
+        test.fail("Test failed, script output is: %s", output)
+    else:
+        test.log.info("Test pass, script output is : %s", output)
+    session.close()