Run Integration Tests #29
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Integration Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| run_id: | |
| description: 'Run ID of the build workflow (all_solutions.yml) to use the agent and integration tests from. ID can be found in URL for run.' | |
| required: true | |
| solutions: | |
| type: choice | |
| description: Which test solutions (IntegrationTests, UnboundedIntegrationTests, or both) to run | |
| required: false | |
| options: | |
| - IntegrationTests | |
| - UnboundedIntegrationTests | |
| - Both | |
| integration-test-namespaces: | |
| description: 'A comma-separated, single-quoted list of integration test namespaces to run (e.g. "''AgentFeatures'', ''BasicInstrumentation''"). If not specified, all integration tests will be run.' | |
| required: true | |
| default: 'ALL' | |
| unbounded-test-namespaces: | |
| description: 'A comma-separated, single-quoted list of unbounded integration test namespaces to run (e.g. "''MsSql'', ''Couchbase''"). If not specified, all unbounded tests will be run.' | |
| required: true | |
| default: 'ALL' | |
| parallelize: | |
| type: boolean | |
| description: 'Run tests in parallel within each namespace' | |
| required: false | |
| permissions: | |
| contents: read | |
| env: | |
| DOTNET_NOLOGO: true | |
| jobs: | |
| setup-matrices: | |
| #needs: [get-external-artifacts] | |
| name: Configure namespaces for test matrices | |
| runs-on: ubuntu-latest | |
| outputs: | |
| integration-tests-matrix: ${{ steps.configure_integration_tests_matrix.outputs.matrix }} | |
| unbounded-tests-matrix: ${{ steps.configure_unbounded_tests_matrix.outputs.matrix }} | |
| steps: | |
| - name: Harden Runner | |
| uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 | |
| with: | |
| egress-policy: audit | |
| - id: configure_integration_tests_matrix | |
| run: | | |
| if [ "${{ github.event.inputs.integration-test-namespaces }}" == "ALL" ] ; then | |
| # Use the full list of namespaces | |
| namespaces="[ 'AgentFeatures', 'AgentLogs', 'AgentMetrics', 'Api', 'AppDomainCaching', 'AspNetCore', 'BasicInstrumentation', 'CatInbound', 'CatOutbound', 'CodeLevelMetrics', 'Configuration', \ | |
| 'CSP', 'CustomAttributes', 'CustomInstrumentation', 'DataTransmission', 'DistributedTracing', 'Errors', 'HttpClientInstrumentation', 'InfiniteTracing', 'Logging.ContextData', \ | |
| 'Logging.HsmAndCsp', 'Logging.LocalDecoration', 'Logging.LogLevelDetection', 'Logging.MaxSamplesStored', 'Logging.MetricsAndForwarding', 'Logging.ZeroMaxSamplesStored', \ | |
| 'Owin', 'MassTransit', 'ReJit.NetCore', 'ReJit.NetFramework', 'RequestHandling', 'RequestHeadersCapture.AspNet', 'RequestHeadersCapture.AspNetCore', 'RequestHeadersCapture.EnvironmentVariables', \ | |
| 'RequestHeadersCapture.Owin', 'RequestHeadersCapture.WCF', 'RestSharp', 'WCF.Client.IIS.ASPDisabled', 'WCF.Client.IIS.ASPEnabled', 'WCF.Client.Self', \ | |
| 'WCF.Service.IIS.ASPDisabled', 'WCF.Service.IIS.ASPEnabled', 'WCF.Service.Self' ]" | |
| else | |
| # Just use the supplied list of namespaces | |
| namespaces="[ ${{ github.event.inputs.integration-test-namespaces }} ]" | |
| fi | |
| echo "matrix=$namespaces" >> $GITHUB_OUTPUT | |
| shell: bash | |
| - id: configure_unbounded_tests_matrix | |
| run: | | |
| if [ "${{ github.event.inputs.unbounded-test-namespaces }}" == "ALL" ] ; then | |
| # Use the full list of namespaces | |
| # Oracle is disabled temporarily | |
| #namespaces="[ 'CosmosDB', 'Couchbase', 'Elasticsearch', 'MongoDB', 'Msmq', 'MsSql', 'MySql', 'NServiceBus', 'NServiceBus5', 'Oracle', 'Postgres', 'RabbitMq', 'Redis' ]" | |
| namespaces="[ 'CosmosDB', 'Couchbase', 'Elasticsearch', 'MongoDB', 'Msmq', 'MsSql', 'MySql', 'NServiceBus', 'NServiceBus5', 'Postgres', 'RabbitMq', 'Redis' ]" | |
| else | |
| # Just use the supplied list of namespaces | |
| namespaces="[ ${{ github.event.inputs.unbounded-test-namespaces }} ]" | |
| fi | |
| echo "matrix=$namespaces" >> $GITHUB_OUTPUT | |
| shell: bash | |
| run-integration-tests: | |
| needs: [setup-matrices] | |
| if: github.event.inputs.solutions == 'Both' || github.event.inputs.solutions == 'IntegrationTests' | |
| name: Run IntegrationTests | |
| runs-on: windows-2022 # TODO: make this a input so we can test different OSes? | |
| strategy: | |
| matrix: | |
| namespace: ${{fromJSON(needs.setup-matrices.outputs.integration-tests-matrix)}} | |
| fail-fast: false # we don't want one test failure in one namespace to kill the other runs | |
| env: | |
| integration_tests_shared_project: ${{ github.workspace }}/tests/Agent/IntegrationTests/Shared | |
| integration_tests_path: ${{ github.workspace }}/tests/Agent/IntegrationTests/IntegrationTests/bin/Release/net462 | |
| # Make this variable true to enable extra data-gathering and logging to help troubleshoot test failures, at the cost of additional time and resources | |
| enhanced_logging: false | |
| NR_DOTNET_TEST_SAVE_WORKING_DIRECTORY : 1 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Disable TLS 1.3 | |
| run: | | |
| $registryPath = "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\TLS 1.3\Client" | |
| if(!(Test-Path $registryPath)) { | |
| New-Item -Path $registryPath -Force | |
| } | |
| New-ItemProperty -Path $registryPath -Name "DisabledByDefault" -Value "1" -PropertyType DWORD -Force | |
| New-ItemProperty -Path $registryPath -Name "Enabled" -Value "0" -PropertyType DWORD -Force | |
| shell: powershell | |
| - name: Create and trust .NET development SSL certificate | |
| run: | | |
| dotnet dev-certs https --clean | |
| dotnet dev-certs https --export-path ./devcert.pfx --password "password1" | |
| $pwd = ConvertTo-SecureString -String "password1" -Force -AsPlainText | |
| Import-PfxCertificate -FilePath ./devcert.pfx -CertStoreLocation Cert:\LocalMachine\Root -Password $pwd | |
| dotnet dev-certs https --check --trust | |
| shell: powershell | |
| - name: Set up secrets | |
| env: | |
| INTEGRATION_TEST_SECRETS: ${{ secrets.TEST_SECRETS }} | |
| run: | | |
| "$Env:INTEGRATION_TEST_SECRETS" | dotnet user-secrets set --project ${{ env.integration_tests_shared_project }} | |
| shell: pwsh #this doesn't work with normal powershell due to UTF-8 BOM handling | |
| - name: Download Agent Home Folders | |
| uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0 | |
| #uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d # v3.0.0 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| workflow: all_solutions.yml | |
| run-id: ${{ github.event.inputs.run_id }} | |
| name: homefolders | |
| path: ${{ github.workspace }}/src/Agent | |
| repo: ${{ github.repository }} | |
| - name: Download Integration Tests | |
| uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0 | |
| #uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d # v3.0.0 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| workflow: all_solutions.yml | |
| run-id: ${{ github.event.inputs.run_id }} | |
| name: integrationtests | |
| path: ${{ github.workspace }} | |
| repo: ${{ github.repository }} | |
| - name: Install dependencies | |
| run: | | |
| Enable-WindowsOptionalFeature -Online -FeatureName IIS-HostableWebCore | |
| pip install aiohttp | |
| shell: powershell | |
| - name: Run Integration Tests | |
| run: | | |
| if ($Env:enhanced_logging -eq $True) { | |
| Write-Host "List ports in use" | |
| netstat -no | |
| } | |
| Write-Host "Run tests" | |
| # Test parallelization is disabled until we can solve concurrent dotnet publish issues with ConsoleMF usage | |
| $json = Get-Content "${{ env.integration_tests_path }}/xunit.runner.json" | ConvertFrom-Json | |
| $json | Add-Member -Name "parallelizeAssembly" -Value $false -MemberType NoteProperty | |
| $json | Add-Member -Name "parallelizeTestCollections" -Value $false -MemberType NoteProperty | |
| # if ("${{ matrix.namespace }}" -like "Logging.*" ) { | |
| # $json.parallelizeAssembly = $true | |
| # $json.parallelizeTestCollections = $true | |
| # } | |
| if ("${{ github.event.inputs.parallelize }}" -eq "true") { | |
| $json.parallelizeAssembly = $true | |
| $json.parallelizeTestCollections = $true | |
| } | |
| $json | ConvertTo-Json | Out-File "${{ env.integration_tests_path }}/xunit.runner.json" | |
| dotnet test ${{ env.integration_tests_path }}/NewRelic.Agent.IntegrationTests.dll --filter FullyQualifiedName~NewRelic.Agent.IntegrationTests.${{ matrix.namespace }} --no-build --nologo --logger "trx;LogFileName=C:\IntegrationTestWorkingDirectory\TestResults\${{ matrix.namespace }}_testResults.trx" | |
| if ($Env:enhanced_logging -eq $True) { | |
| Write-Host "Get HostableWebCore errors (if any)" | |
| Get-EventLog -LogName Application -Source HostableWebCore -ErrorAction:Ignore | |
| Write-Host "Get .NET Runtime errors (if any)" | |
| Get-EventLog -LogName Application -Source ".NET Runtime" -EntryType "Error","Warning" -ErrorAction:Ignore | |
| } | |
| shell: powershell | |
| - name: Archive IntegrationTestWorkingDirectory on Failure | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 | |
| with: | |
| name: integration-test-working-directory-${{ matrix.namespace }} | |
| path: | | |
| C:\IntegrationTestWorkingDirectory\**\*.log | |
| C:\IntegrationTestWorkingDirectory\**\*.config | |
| C:\IntegrationTestWorkingDirectory\**\appsettings.json | |
| if-no-files-found: error | |
| - name: Archive Test Artifacts on failure | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 | |
| with: | |
| name: integration-test-results-${{ matrix.namespace }} | |
| path: C:\IntegrationTestWorkingDirectory\TestResults\**\*TestResults.trx | |
| if-no-files-found: error | |
| run-unbounded-tests: | |
| needs: [setup-matrices] | |
| if: github.event.inputs.solutions == 'Both' || github.event.inputs.solutions == 'UnboundedIntegrationTests' | |
| name: Run Unbounded Tests | |
| runs-on: windows-2022 | |
| strategy: | |
| matrix: | |
| namespace: ${{fromJSON(needs.setup-matrices.outputs.unbounded-tests-matrix)}} | |
| fail-fast: false # we don't want one test failure in one namespace to kill the other runs | |
| env: | |
| integration_tests_shared_project: ${{ github.workspace }}/tests/Agent/IntegrationTests/Shared | |
| unbounded_tests_path: ${{ github.workspace }}/tests/Agent/IntegrationTests/UnboundedIntegrationTests/bin/Release/net462 | |
| NR_DOTNET_TEST_SAVE_WORKING_DIRECTORY: 1 | |
| # Make this variable true to enable extra data-gathering and logging to help troubleshoot test failures, at the cost of additional time and resources | |
| enhanced_logging: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download Agent Home Folders | |
| uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0 | |
| #uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d # v3.0.0 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| workflow: all_solutions.yml | |
| run-id: ${{ github.event.inputs.run_id }} | |
| name: homefolders | |
| path: ${{ github.workspace }}/src/Agent | |
| repo: ${{ github.repository }} | |
| - name: Download Integration Tests | |
| uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0 | |
| #uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d # v3.0.0 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| workflow: all_solutions.yml | |
| run-id: ${{ github.event.inputs.run_id }} | |
| name: unboundedintegrationtests | |
| path: ${{ github.workspace }} | |
| repo: ${{ github.repository }} | |
| - name: Disable TLS 1.3 | |
| run: | | |
| $registryPath = "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\TLS 1.3\Client" | |
| if(!(Test-Path $registryPath)) { | |
| New-Item -Path $registryPath -Force | |
| } | |
| New-ItemProperty -Path $registryPath -Name "DisabledByDefault" -Value "1" -PropertyType DWORD -Force | |
| New-ItemProperty -Path $registryPath -Name "Enabled" -Value "0" -PropertyType DWORD -Force | |
| shell: powershell | |
| - name: Install dependencies | |
| run: | | |
| Write-Host "Installing HostableWebCore Feature" | |
| Enable-WindowsOptionalFeature -Online -FeatureName IIS-HostableWebCore | |
| $namespacesThatNeedMsmq = 'Msmq','NServiceBus','NServiceBus5' | |
| if ($namespacesThatNeedMsmq -contains "${{ matrix.namespace }}") | |
| { | |
| Write-Host "Installing Msmq Features" | |
| Enable-WindowsOptionalFeature -Online -FeatureName MSMQ-Server -All | |
| Enable-WindowsOptionalFeature -Online -FeatureName MSMQ-HTTP -All | |
| Enable-WindowsOptionalFeature -Online -FeatureName MSMQ-Triggers -All | |
| } | |
| if ("${{ matrix.namespace }}" -eq "MsSql") { | |
| Write-Host "Installing MSSQL CLI" | |
| msiexec /i "${{ github.workspace }}\build\Tools\sqlncli.msi" IACCEPTSQLNCLILICENSETERMS=YES /quiet /qn /norestart | |
| Start-Sleep 20 # Need to wait for install to finish -- takes only a few seconds, but we need to be sure. | |
| } | |
| shell: powershell | |
| - name: Set up secrets | |
| env: | |
| INTEGRATION_TEST_SECRETS: ${{ secrets.UNBOUNDED_TEST_SECRETS }} | |
| run: | | |
| "$Env:INTEGRATION_TEST_SECRETS" | dotnet user-secrets set --project ${{ env.integration_tests_shared_project }} | |
| shell: pwsh #this doesn't work with normal powershell due to UTF-8 BOM handling | |
| - name: Start Local CosmosDB Emulator for CosmosDB Tests | |
| if: matrix.namespace == 'CosmosDB' | |
| uses: southpolesteve/cosmos-emulator-github-action@2b1168b52481f972890f5da2ff8f9d2cc3707804 # v1 | |
| - name: Run Unbounded Integration Tests | |
| run: | | |
| if ($Env:enhanced_logging -eq $True) { | |
| Write-Host "List ports in use" | |
| netstat -no | |
| } | |
| # Test parallelization is disabled until we can solve concurrent dotnet publish issues with ConsoleMF usage | |
| $json = Get-Content "${{ env.unbounded_tests_path }}/xunit.runner.json" | ConvertFrom-Json | |
| $json | Add-Member -Name "parallelizeAssembly" -Value $false -MemberType NoteProperty | |
| $json | Add-Member -Name "parallelizeTestCollections" -Value $false -MemberType NoteProperty | |
| if ("${{ github.event.inputs.parallelize }}" -eq "true") { | |
| $json.parallelizeAssembly = $true | |
| $json.parallelizeTestCollections = $true | |
| } | |
| $json | ConvertTo-Json | Out-File "${{ env.unbounded_tests_path }}/xunit.runner.json" | |
| dotnet test ${{ env.unbounded_tests_path }}/NewRelic.Agent.UnboundedIntegrationTests.dll --filter FullyQualifiedName~NewRelic.Agent.UnboundedIntegrationTests.${{ matrix.namespace }} --no-build --nologo --logger "trx;LogFileName=C:\IntegrationTestWorkingDirectory\TestResults\${{ matrix.namespace }}_testResults.trx" | |
| if ($Env:enhanced_logging -eq $True) { | |
| Write-Host "Get HostableWebCore errors (if any)" | |
| Get-EventLog -LogName Application -Source HostableWebCore -ErrorAction:Ignore | |
| Write-Host "Get .NET Runtime errors (if any)" | |
| Get-EventLog -LogName Application -Source ".NET Runtime" -EntryType "Error","Warning" -ErrorAction:Ignore | |
| } | |
| shell: powershell | |
| - name: Archive IntegrationTestWorkingDirectory on Failure | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 | |
| with: | |
| name: unbounded-test-working-directory-${{ matrix.namespace }} | |
| path: | | |
| C:\IntegrationTestWorkingDirectory\**\*.log | |
| C:\IntegrationTestWorkingDirectory\**\*.config | |
| C:\IntegrationTestWorkingDirectory\**\appsettings.json | |
| if-no-files-found: error | |
| - name: Archive Test Artifacts | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 | |
| with: | |
| name: unbounded-test-results-${{ matrix.namespace }} | |
| path: C:\IntegrationTestWorkingDirectory\TestResults\**\*TestResults.trx | |
| if-no-files-found: error | |