Add support for schduled kernel mode multi-threaded stress tests in g…

…h ci/cd (#2670)

Co-authored-by: Dhiren Vispute <[email protected]>

.github/workflows/cicd.yml

@@ -424,36 +424,36 @@ jobs:
       gather_dumps: true
       capture_etw: true
 
-  # Run multi-threaded stress tests against the kernel mode eBPF sub-system.
-  # The '-td' parameter specifies the run time per test, so the total run-time will be a multiple of the total number
-  # of tests, as each test is run sequentially.
-  kernel_mode_multi_threaded_stress_test:
+  # Run multi-threaded stress tests with 'restart extension' disabled (default behavior)
+  # against the kernel mode eBPF sub-system.
+  km_mt_stress_tests:
     needs: regular
     if: github.event_name == 'schedule'
     uses: ./.github/workflows/reusable-test.yml
     with:
-      name: kernel_mode_multi_threaded_stress
-      test_command: .\ebpf_stress_tests_km -tt=32 -td=10
+      name: km_mt_stress_tests
+      pre_test: .\setup_ebpf_cicd_stress_tests.ps1
+      test_command: .\execute_ebpf_cicd_tests.ps1 -RunKmStressTests $true
+      post_test: .\cleanup_ebpf_cicd_tests.ps1
       build_artifact: Build-x64
-      environment: windows-2022
+      environment: ebpf_cicd_tests
       code_coverage: false
-      leak_detection: false
-      gather_dumps: true
-      capture_etw: true
+      # For this test, we only want kernel mode dumps and not user mode dumps.
+      gather_dumps: false
 
-  # Run multi-threaded stress tests against the kernel mode eBPF sub-system with extension restart enabled.
-  # The '-td' parameter specifies the run time per test, so the total run-time will be a multiple of the total number
-  # of tests, as each test is run sequentially.
-  kernel_mode_multi_threaded_stress_extension_restart_test:
+  # Run multi-threaded stress tests with 'restart extension' enabled
+  # against the kernel mode eBPF sub-system.
+  km_mt_stress_tests_restart_extension:
     needs: regular
     if: github.event_name == 'schedule'
     uses: ./.github/workflows/reusable-test.yml
     with:
-      name: kernel_mode_multi_threaded_stress_extension_restart
-      test_command: .\ebpf_stress_tests_km -tt=32 -td=10 -er=true -erd=250
+      name: km_mt_stress_tests_restart_extension
+      pre_test: .\setup_ebpf_cicd_stress_tests.ps1
+      test_command: .\execute_ebpf_cicd_stress_tests.ps1 -RunKmStressTests $true -RestartExtension $true
+      post_test: .\cleanup_ebpf_cicd_tests.ps1
       build_artifact: Build-x64
-      environment: windows-2022
+      environment: ebpf_cicd_tests
       code_coverage: false
-      leak_detection: false
-      gather_dumps: true
-      capture_etw: true
+      # For this test, we only want kernel mode dumps and not user mode dumps.
+      gather_dumps: false

scripts/execute_ebpf_cicd_tests.ps1

@@ -7,6 +7,8 @@ param ([parameter(Mandatory=$false)][string] $AdminTarget = "TEST_VM",
        [parameter(Mandatory=$false)][string] $WorkingDirectory = $pwd.ToString(),
        [parameter(Mandatory=$false)][string] $TestExecutionJsonFileName = "test_execution.json",
        [parameter(Mandatory=$false)][bool] $Coverage = $false,
+       [parameter(Mandatory=$false)][bool] $RunKmStressTests = $false,
+       [parameter(Mandatory=$false)][bool] $RestartExtension = $false,
        [parameter(Mandatory=$false)][string] $SelfHostedRunnerName)
 
 Push-Location $WorkingDirectory
@@ -24,7 +26,11 @@ $VMList = $Config.VMMap.$SelfHostedRunnerName
 
 # Run tests on test VMs.
 foreach ($VM in $VMList) {
-    Invoke-CICDTestsOnVM -VMName $VM.Name -Coverage $Coverage
+    Invoke-CICDTestsOnVM `
+        -VMName $VM.Name `
+        -Coverage $Coverage `
+        -RunKmStressTests $RunKmStressTests `
+        -RestartExtension $RestartExtension
 }
 
 # Run XDP Tests.

scripts/run_driver_tests.psm1

@@ -238,4 +238,33 @@ function Invoke-ConnectRedirectTest
     popd
 }
 
+function Invoke-CICDStressTests
+{
+    param([parameter(Mandatory = $true)][bool] $VerboseLogs,
+          [parameter(Mandatory = $false)][bool] $Coverage = $false,
+          [parameter(Mandatory = $false)][bool] $RestartExtension = $false)
+
+    pushd $WorkingDirectory
+    $env:EBPF_ENABLE_WER_REPORT = "yes"
+
+    Write-Log "Executing eBPF kernel mode multi-threaded stress tests (restart extension:$RestartExtension)."
+
+    $LASTEXITCODE = 0
+    # The '-td' parameter specifies the run time per test, so the total run-time will be a multiple of the total
+    # number of tests, as each test is run sequentially.
+    if ($RestartExtension -eq $false) {
+        write-host "Test command-line: .\ebpf_stress_tests_km -tt=8 -td=5"
+        .\ebpf_stress_tests_km -tt=8 -td=5
+    } else {
+        write-host "Test command-line: .\ebpf_stress_tests_km -tt=8 -td=5 -erd=1000 -er=1"
+        .\ebpf_stress_tests_km -tt=8 -td=5 -erd=1000 -er=1
+    }
+
+    if ($LASTEXITCODE -ne 0) {
+        throw "*** ERROR *** eBPF kernel mode multi-threaded stress tests FAILED (restart extension:$RestartExtension)"
+    }
+
+    popd
+}
+
 Pop-Location

scripts/setup_build/CMakeLists.txt

@@ -38,6 +38,7 @@ set (misc_files "setup-ebpf.ps1"
     "common.psm1"
     "config_test_vm.psm1"
     "setup_ebpf_cicd_tests.ps1"
+    "setup_ebpf_cicd_stress_tests.ps1"
     "execute_ebpf_cicd_tests.ps1"
     "cleanup_ebpf_cicd_tests.ps1"
     "install_ebpf.psm1"

scripts/setup_build/setup_build.vcxproj

@@ -94,6 +94,13 @@ popd
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='NativeOnlyRelease|x64'">true</DeploymentContent>
     </CopyFileToFolders>
+    <CopyFileToFolders Include="..\setup_ebpf_cicd_stress_tests.ps1">
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='FuzzerDebug|x64'">true</DeploymentContent>
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='NativeOnlyDebug|x64'">true</DeploymentContent>
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
+      <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='NativeOnlyRelease|x64'">true</DeploymentContent>
+    </CopyFileToFolders>
     <CopyFileToFolders Include="..\execute_ebpf_cicd_tests.ps1">
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
       <DeploymentContent Condition="'$(Configuration)|$(Platform)'=='FuzzerDebug|x64'">true</DeploymentContent>

scripts/setup_build/setup_build.vcxproj.filters

@@ -34,6 +34,9 @@
     <CopyFileToFolders Include="..\setup_ebpf_cicd_tests.ps1">
       <Filter>Source Files</Filter>
     </CopyFileToFolders>
+    <CopyFileToFolders Include="..\setup_ebpf_cicd_stress_tests.ps1">
+      <Filter>Source Files</Filter>
+    </CopyFileToFolders>
     <CopyFileToFolders Include="..\execute_ebpf_cicd_tests.ps1">
       <Filter>Source Files</Filter>
     </CopyFileToFolders>

scripts/setup_ebpf_cicd_stress_tests.ps1

@@ -0,0 +1,42 @@
+# Copyright (c) Microsoft Corporation
+# SPDX-License-Identifier: MIT
+
+param ([parameter(Mandatory=$false)][string] $Target = "TEST_VM",
+       [parameter(Mandatory=$false)][string] $LogFileName = "TestLog.log",
+       [parameter(Mandatory=$false)][string] $WorkingDirectory = $pwd.ToString(),
+       [parameter(Mandatory=$false)][string] $TestExecutionJsonFileName = "test_execution.json",
+       [parameter(Mandatory=$false)][string] $SelfHostedRunnerName)
+
+Push-Location $WorkingDirectory
+
+$TestVMCredential = Get-StoredCredential -Target $Target -ErrorAction Stop
+
+# Load other utility modules.
+Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue
+Import-Module .\config_test_vm.psm1 -Force -ArgumentList ($TestVMCredential.UserName, $TestVMCredential.Password, $WorkingDirectory, $LogFileName) -WarningAction SilentlyContinue
+
+# Read the test execution json.
+$TestExecutionConfig = Get-Content ("{0}\{1}" -f $PSScriptRoot, $TestExecutionJsonFileName) | ConvertFrom-Json
+$VMList = $TestExecutionConfig.VMMap.$SelfHostedRunnerName
+
+# Delete old log files if any.
+Remove-Item "$env:TEMP\$LogFileName" -ErrorAction SilentlyContinue
+foreach($VM in $VMList) {
+    $VMName = $VM.Name
+    Remove-Item $env:TEMP\$LogFileName -ErrorAction SilentlyContinue
+}
+Remove-Item ".\TestLogs" -Recurse -Confirm:$false -ErrorAction SilentlyContinue
+
+# Get all VMs to ready state.
+Initialize-AllVMs -VMList $VMList -ErrorAction Stop
+
+# Export build artifacts to the test VMs.
+Export-BuildArtifactsToVMs -VMList $VMList -ErrorAction Stop
+
+# Install eBPF Components on the test VM.
+foreach($VM in $VMList) {
+    $VMName = $VM.Name
+    Install-eBPFComponentsOnVM -VMName $VMname -ErrorAction Stop
+}
+
+Pop-Location

scripts/vm_run_tests.psm1

@@ -15,13 +15,19 @@ Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction S
 #
 # Execute tests on VM.
 #
-
 function Invoke-CICDTestsOnVM
 {
     param([parameter(Mandatory=$true)] [string] $VMName,
           [parameter(Mandatory=$false)] [bool] $VerboseLogs = $false,
-          [parameter(Mandatory=$false)] [bool] $Coverage = $false)
-    Write-Log "Running eBPF CI/CD tests on $VMName"
+          [parameter(Mandatory=$false)] [bool] $Coverage = $false,
+          [parameter(Mandatory=$false)][bool] $RunKmStressTests = $false,
+          [parameter(Mandatory=$false)][bool] $RestartExtension = $false)
+
+    if ($RunKmStressTests -eq $true) {
+        Write-Log "Executing eBPF kernel mode multi-threaded stress tests on $VMName"
+    } else {
+        Write-Log "Running eBPF CI/CD tests on $VMName"
+    }
     $TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
 
     Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock {
@@ -33,7 +39,13 @@ function Invoke-CICDTestsOnVM
         Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
         Import-Module $WorkingDirectory\run_driver_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
 
-        Invoke-CICDTests -VerboseLogs $VerboseLogs -Coverage $Coverage 2>&1 | Write-Log
+        if ($RunKmStressTests -eq $true) {
+            Invoke-CICDStressTests -VerboseLogs $VerboseLogs -Coverage $Coverage `
+                -RestartExtension $RestartExtension 2>&1 | Write-Log
+        } else {
+            Invoke-CICDTests -VerboseLogs $VerboseLogs -Coverage $Coverage 2>&1 | Write-Log
+        }
+
     } -ArgumentList ("eBPF", $LogFileName, $VerboseLogs, $Coverage) -ErrorAction Stop
 }
 
@@ -434,9 +446,9 @@ function Invoke-ConnectRedirectTestsOnVM
            -ScriptBlock {param ($StandardUser) Get-LocalUser -Name "$StandardUser"} `
            -Argumentlist $StandardUser -ErrorAction SilentlyContinue
     if($UserId) {
-		Write-Host "Deleting existing standard user:" $StandardUser "on" $VMName
-		Remove-StandardUserOnVM -VM $VMName -UserName $StandardUser
-	}
+        Write-Host "Deleting existing standard user:" $StandardUser "on" $VMName
+        Remove-StandardUserOnVM -VM $VMName -UserName $StandardUser
+    }
 
     # Add a standard user on VM1.
     Add-StandardUserOnVM -VM $VMName -UserName $StandardUser -Password $UnsecurePassword