Bpf performance run (#2821)

* Run bpf_performance_runner on a scheduled run. Signed-off-by: Alan Jowett <[email protected]> * Capture CPU profile Signed-off-by: Alan Jowett <[email protected]> * Fix mandatory parameter Signed-off-by: Alan Jowett <[email protected]> * Pass list of options for test mode Signed-off-by: Alan Jowett <[email protected]> * Capture CPU profile Signed-off-by: Alan Jowett <[email protected]> * Handle empty options parameter Signed-off-by: Alan Jowett <[email protected]> * Switch to v0.0.2 of perf test and gather per test ETL Signed-off-by: Alan Jowett <[email protected]> * Switch perf to scheduled run Signed-off-by: Alan Jowett <[email protected]> * PR feedback Signed-off-by: Alan Jowett <[email protected]> * PR feedback Signed-off-by: Alan Jowett <[email protected]> --------- Signed-off-by: Alan Jowett <[email protected]> Co-authored-by: Alan Jowett <[email protected]>
microsoft · Sep 11, 2023 · b0c099b · b0c099b
1 parent 5cc488f
commit b0c099b
Show file tree

Hide file tree

Showing 7 changed files with 115 additions and 36 deletions.
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
@@ -17,6 +17,8 @@ on:
   # Run on pull request to validate code changes.
   pull_request:
   merge_group:
+  # Permit manual runs of the workflow.
+  workflow_dispatch:
   # Run on push so we can capture the baseline code coverage.
   push:
     branches: [ main ]
@@ -40,7 +42,7 @@ jobs:
   # Perform the regular build.
   regular:
     # Always run this job.
-    if: github.event_name == 'schedule' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'merge_group'
+    if: github.event_name == 'schedule' || github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch'
     uses: ./.github/workflows/reusable-build.yml
     with:
       build_artifact: Build-x64
@@ -138,7 +140,7 @@ jobs:
     uses: ./.github/workflows/reusable-test.yml
     with:
       pre_test: .\setup_ebpf_cicd_tests.ps1
-      test_command: .\execute_ebpf_cicd_tests.ps1
+      test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "CI/CD"
       post_test: .\cleanup_ebpf_cicd_tests.ps1
       name: driver
       build_artifact: Build-x64
@@ -157,7 +159,7 @@ jobs:
     uses: ./.github/workflows/reusable-test.yml
     with:
       pre_test: .\setup_ebpf_cicd_tests.ps1
-      test_command: .\execute_ebpf_cicd_tests.ps1
+      test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "CI/CD"
       post_test: .\cleanup_ebpf_cicd_tests.ps1
       name: driver
       build_artifact: Build-x64-native-only
@@ -448,7 +450,7 @@ jobs:
     with:
       name: km_mt_stress_tests
       pre_test: .\setup_ebpf_cicd_stress_tests.ps1
-      test_command: .\execute_ebpf_cicd_tests.ps1 -RunKmStressTestsOnly $true
+      test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Stress"
       post_test: .\cleanup_ebpf_cicd_tests.ps1
       build_artifact: Build-x64
       environment: ebpf_cicd_tests
@@ -465,10 +467,34 @@ jobs:
     with:
       name: km_mt_stress_tests_restart_extension
       pre_test: .\setup_ebpf_cicd_stress_tests.ps1
-      test_command: .\execute_ebpf_cicd_tests.ps1 -RunKmStressTestsOnly $true -RestartExtension $true
+      test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Stress" -Options @("RestartExtension")
       post_test: .\cleanup_ebpf_cicd_tests.ps1
       build_artifact: Build-x64
       environment: ebpf_cicd_tests
       code_coverage: false
       # For this test, we only want kernel mode dumps and not user mode dumps.
       gather_dumps: false
+
+  performance:
+    needs: regular
+    if: github.event_name == 'schedule'
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      name: km_performance
+      pre_test: .\setup_ebpf_cicd_tests.ps1
+      test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Performance"
+      post_test: .\cleanup_ebpf_cicd_tests.ps1
+      build_artifact: Build-x64
+      environment: ebpf_cicd_tests
+
+  performance_with_profile:
+    needs: regular
+    if: github.event_name == 'workflow_dispatch'
+    uses: ./.github/workflows/reusable-test.yml
+    with:
+      name: km_performance
+      pre_test: .\setup_ebpf_cicd_tests.ps1
+      test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Performance" -Options @("CaptureProfile")
+      post_test: .\cleanup_ebpf_cicd_tests.ps1
+      build_artifact: Build-x64
+      environment: ebpf_cicd_tests
diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml
@@ -198,13 +198,13 @@ jobs:
       working-directory: ${{env.GITHUB_WORKSPACE}}
       run: Invoke-WebRequest https://github.com/microsoft/ebpf-for-windows-demo/releases/download/v0.0.1/${{env.BUILD_PLATFORM}}-Release-cilium-xdp.zip -OutFile x64-${{env.BUILD_CONFIGURATION}}-cilium-xdp.zip
 
-    # Download the bpf_performance repository artifacts for the Release build.
+    # Download the bpf_performance repository artifacts.
     - name: Download bpf_performance repository artifacts
-      if: steps.skip_check.outputs.should_skip != 'true' && (matrix.configurations == 'Release' || matrix.configurations == 'NativeOnlyRelease')
+      if: steps.skip_check.outputs.should_skip != 'true'
       working-directory: ${{env.GITHUB_WORKSPACE}}
       run: |
         cd ${{github.workspace}}/${{env.BUILD_PLATFORM}}/${{env.BUILD_CONFIGURATION}}
-        Invoke-WebRequest https://github.com/Alan-Jowett/bpf_performance/releases/download/v0.0.1/build-RelWithDebInfo-windows-2022.zip -OutFile bpf_performance.zip
+        Invoke-WebRequest https://github.com/Alan-Jowett/bpf_performance/releases/download/v0.0.2/build-RelWithDebInfo-windows-2022.zip -OutFile bpf_performance.zip
 
     - name: Extract artifacts to build path
       if: steps.skip_check.outputs.should_skip != 'true' && matrix.configurations != 'FuzzerDebug'

diff --git a/docs/AutomatedTests.md b/docs/AutomatedTests.md
@@ -75,6 +75,4 @@ Tests in this category currently include:
 ## Performance tests
 Performance tests check for performance regressions across builds.
 
-Tests in this category currently include:
-* ebpf_performance.exe: Currently this only outputs performance numbers, rather than
-  checking for regressions.  Improving this is issue #1915.
+This invokes the bpf_performance test suite with a variety of test cases.
diff --git a/scripts/config_test_vm.psm1 b/scripts/config_test_vm.psm1
@@ -275,6 +275,14 @@ function Import-ResultsFromVM
         # Copy ETL from Test VM.
         Write-Log ("Copy $EtlFile from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs")
         Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\$EtlFile" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+
+        # Copy performance results from Test VM.
+        Write-Log ("Copy performance results from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs")
+        Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\*.csv" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+
+        # Copy the performance profile if present.
+        Write-Log ("Copy performance profile from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs")
+        Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\bpf_performance*.etl" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
     }
     # Move runner test logs to TestLogs folder.
     Write-Host ("Copy $LogFileName from $env:TEMP on host runner to $pwd\TestLogs")

diff --git a/scripts/execute_ebpf_cicd_tests.ps1 b/scripts/execute_ebpf_cicd_tests.ps1
@@ -7,8 +7,8 @@ param ([parameter(Mandatory=$false)][string] $AdminTarget = "TEST_VM",
        [parameter(Mandatory=$false)][string] $WorkingDirectory = $pwd.ToString(),
        [parameter(Mandatory=$false)][string] $TestExecutionJsonFileName = "test_execution.json",
        [parameter(Mandatory=$false)][bool] $Coverage = $false,
-       [parameter(Mandatory=$false)][bool] $RunKmStressTestsOnly = $false,
-       [parameter(Mandatory=$false)][bool] $RestartExtension = $false,
+       [parameter(Mandatory=$false)][string] $TestMode = "CI/CD",
+       [parameter(Mandatory=$false)][string[]] $Options = @("None"),
        [parameter(Mandatory=$false)][string] $SelfHostedRunnerName)
 
 Push-Location $WorkingDirectory
@@ -29,16 +29,13 @@ foreach ($VM in $VMList) {
     Invoke-CICDTestsOnVM `
         -VMName $VM.Name `
         -Coverage $Coverage `
-        -RunKmStressTestsOnly $RunKmStressTestsOnly `
-        -RestartExtension $RestartExtension
+        -TestMode $TestMode `
+        -Options $Options
 }
 
-# This script is used to execute the regular kernel mode tests as well as the scheduled kernel mode stress tests. The
-# required behavior is selected by the $RunKmStressTestsOnly parameter which is set to 'true' only for the scheduled
-# runs.  The other tests i.e., Invoke-XDPTestsOnVM, Invoke-ConnectRedirectTestsOnVM and
-# Invoke-ConnectRedirectTestsOnVM are already handled by other jobs, so re-executing them again along with the stress
-# tests is redundant.
-if ($RunKmStressTestsOnly -eq $false) {
+# This script is used to execute the various kernel mode tests. The required behavior is selected by the $TestMode
+# parameter.
+if ($TestMode -eq "CI/CD") {
 
     # Run XDP Tests.
     Invoke-XDPTestsOnVM $Config.Interfaces $VMList[0].Name

diff --git a/scripts/run_driver_tests.psm1 b/scripts/run_driver_tests.psm1
@@ -267,4 +267,43 @@ function Invoke-CICDStressTests
     popd
 }
 
+function Invoke-CICDPerformanceTests
+{
+    param(
+        [parameter(Mandatory = $true)][bool] $VerboseLogs,
+        [parameter(Mandatory = $true)][bool] $CaptureProfile)
+    Push-Location $WorkingDirectory
+
+    Write-Log "Executing eBPF kernel mode performance tests."
+
+    $LASTEXITCODE = 0
+
+    # Stop the services, remove the driver from verifier, and restart the services.
+    net.exe stop ebpfsvc
+    net.exe stop ebpfcore
+    # Remove the global verifier settings (this will remove the verifer interceptions that can degrade performance).
+    verifier.exe /volatile 0
+    # Remove the ebpfcore.sys driver from the verifier.
+    verifier.exe /volatile /removedriver ebpfcore.sys
+    net.exe start ebpfcore
+    net.exe start ebpfsvc
+
+    # Extract the performance test zip file.
+    Expand-Archive -Path .\bpf_performance.zip -DestinationPath .\bpf_performance -Force
+    Set-Location bpf_performance
+    # Stop any existing tracing.
+    wpr.exe -cancel
+
+    if ($CaptureProfile) {
+        $pre_command = 'wpr.exe -start CPU'
+        $post_command = 'wpr.exe -stop ""' + $WorkingDirectory + '\bpf_performance_%NAME%.etl""'
+        RelWithDebInfo\bpf_performance_runner.exe -i tests.yml -e .sys -r --pre "$pre_command" --post "$post_command" | Tee-Object -FilePath $WorkingDirectory\bpf_performance_native.csv
+    }
+    else {
+        RelWithDebInfo\bpf_performance_runner.exe -i tests.yml -e .sys -r | Tee-Object -FilePath $WorkingDirectory\bpf_performance_native.csv
+    }
+
+    Pop-Location
+}
+
 Pop-Location
diff --git a/scripts/vm_run_tests.psm1 b/scripts/vm_run_tests.psm1
@@ -20,36 +20,47 @@ function Invoke-CICDTestsOnVM
     param([parameter(Mandatory=$true)] [string] $VMName,
           [parameter(Mandatory=$false)] [bool] $VerboseLogs = $false,
           [parameter(Mandatory=$false)] [bool] $Coverage = $false,
-          [parameter(Mandatory=$false)][bool] $RunKmStressTestsOnly = $false,
-          [parameter(Mandatory=$false)][bool] $RestartExtension = $false)
+          [parameter(Mandatory=$false)][string] $TestMode = "CI/CD",
+          [parameter(Mandatory=$false)][string[]] $Options = @())
 
-    if ($RunKmStressTestsOnly -eq $true) {
-        Write-Log "Executing eBPF kernel mode multi-threaded stress tests on $VMName"
-    } else {
-        Write-Log "Running eBPF CI/CD tests on $VMName"
-    }
+    Write-Log "Running eBPF $TestMode tests on $VMName"
     $TestCredential = New-Credential -Username $Admin -AdminPassword $AdminPassword
 
     Invoke-Command -VMName $VMName -Credential $TestCredential -ScriptBlock {
         param([Parameter(Mandatory=$True)] [string] $WorkingDirectory,
               [Parameter(Mandatory=$True)] [string] $LogFileName,
               [Parameter(Mandatory=$True)] [bool] $VerboseLogs,
               [Parameter(Mandatory=$True)] [bool] $Coverage,
-              [parameter(Mandatory=$True)][bool] $RunKmStressTestsOnly,
-              [parameter(Mandatory=$True)][bool] $RestartExtension)
+              [parameter(Mandatory=$True)][string] $TestMode,
+              [parameter(Mandatory=$True)][string[]] $Options)
 
         $WorkingDirectory = "$Env:SystemDrive\$WorkingDirectory"
         Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
         Import-Module $WorkingDirectory\run_driver_tests.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue
 
-        if ($RunKmStressTestsOnly -eq $true) {
-            Invoke-CICDStressTests -VerboseLogs $VerboseLogs -Coverage $Coverage `
-                -RestartExtension $RestartExtension 2>&1 | Write-Log
-        } else {
-            Invoke-CICDTests -VerboseLogs $VerboseLogs -Coverage $Coverage 2>&1 | Write-Log
+        $TestMode = $TestMode.ToLower()
+        switch ($TestMode)
+        {
+            "ci/cd" {
+                Invoke-CICDTests -VerboseLogs $VerboseLogs -Coverage $Coverage 2>&1 | Write-Log
+            }
+            "stress" {
+                # Set RestartExtension to true if options contains that string
+                $RestartExtension = $Options -contains "RestartExtension"
+                Invoke-CICDStressTests -VerboseLogs $VerboseLogs -Coverage $Coverage `
+                    -RestartExtension $RestartExtension 2>&1 | Write-Log
+            }
+            "performance" {
+                # Set CaptureProfle to true if options contains that string
+                $CaptureProfile = $Options -contains "CaptureProfile"
+                Invoke-CICDPerformanceTests -VerboseLogs $VerboseLogs -CaptureProfile $CaptureProfile 2>&1 | Write-Log
+            }
+            default {
+                throw "Invalid test mode: $TestMode"
+            }
         }
 
-    } -ArgumentList ("eBPF", $LogFileName, $VerboseLogs, $Coverage, $RunKmStressTestsOnly, $RestartExtension) -ErrorAction Stop
+    } -ArgumentList ("eBPF", $LogFileName, $VerboseLogs, $Coverage, $TestMode, $Options) -ErrorAction Stop
 }
 
 function Add-eBPFProgramOnVM