Skip to content

Commit

Permalink
Add Analyzer Benchmarks (#2217)
Browse files Browse the repository at this point in the history
* test

* Add Benchmarks

* Add Camera + MediaElement Analyzer Benchmarks

* Add `run_benchmarks`

* Revert to main

* Fix Formatting

* Use `$(NetVersion)`

* Rename Files

* Run all 3 benchmarks

* Add `/* language=C#-test */`

* Fix Failing Tests

* Update azure-pipelines.yml

* Add `public`

* Fix Analyzer Build Warnings

---------

Co-authored-by: Brandon Minnick <[email protected]>
  • Loading branch information
MartyIX and brminnick authored Sep 18, 2024
1 parent cf6360d commit b9c7854
Show file tree
Hide file tree
Showing 25 changed files with 294 additions and 100 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -265,3 +265,6 @@ paket-files/
# Visual Studio Code
.vscode
/samples/workload-install.ps1

# Benchmarkdot
BenchmarkDotNet.Artifacts/
12 changes: 9 additions & 3 deletions Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,16 @@
CS1711: XML comment has a typeparam tag, but there is no type parameter by that name
CS1712: Type parameter has no matching typeparam tag in the XML comment
CS1723: XML comment has cref attribute that refers to a type parameter
CS1734: XML comment has a paramref tag, but there is no parameter by that name
CS1734: XML comment has a paramref tag, but there is no parameter by that name
xUnit1012: Null should not be used for type parameter
xUnit2021: Assert.ThrowsAsync is async. The resulting task should be awaited
IL2***: Trim Warnings
IL3***: AOT Warnings -->
IL3***: AOT Warnings
RS2007: Analyzer release file 'AnalyzerReleases.Shipped.md' has a missing or invalid release header-->
<WarningsAsErrors>nullable,
CS0419,CS1570,CS1571,CS1572,CS1573,CS1574,CS1580,CS1581,CS1584,CS1587,CS1589,CS1590,CS1591,CS1592,CS1598,CS1658,CS1710,CS1711,CS1712,CS1723,CS1734,
xUnit1012,xUnit2021,
Expand All @@ -59,7 +64,8 @@
IL2100,IL2101,IL2102,IL2103,IL2104,IL2105,IL2106,IL2107,IL2108,IL2109,
IL2110,IL2111,IL2112,IL2113,IL2114,IL2115,IL2116,IL2117,IL2118,IL2119,
IL2120,IL2121,IL2122,
IL3050,IL3051,IL3052,IL3053,IL3054,IL3055,IL3056
IL3050,IL3051,IL3052,IL3053,IL3054,IL3055,IL3056,
RS2007
</WarningsAsErrors>
</PropertyGroup>

Expand Down
56 changes: 56 additions & 0 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ variables:
PathToCommunityToolkitCameraAnalyzersCodeFixCsproj: 'src/CommunityToolkit.Maui.Camera.Analyzers.CodeFixes/CommunityToolkit.Maui.Camera.Analyzers.CodeFixes.csproj'
PathToCommunityToolkitMediaElementAnalyzersCodeFixCsproj: 'src/CommunityToolkit.Maui.MediaElement.Analyzers.CodeFixes/CommunityToolkit.Maui.MediaElement.Analyzers.CodeFixes.csproj'
PathToCommunityToolkitAnalyzersUnitTestCsproj: 'src/CommunityToolkit.Maui.Analyzers.UnitTests/CommunityToolkit.Maui.Analyzers.UnitTests.csproj'
PathToCommunityToolkitAnalyzersBenchmarkCsproj: 'src/CommunityToolkit.Maui.Analyzers.Benchmarks/CommunityToolkit.Maui.Analyzers.Benchmarks.csproj'
DotNetMauiRollbackFile: 'https://maui.blob.core.windows.net/metadata/rollbacks/8.0.6.json'
CommunityToolkitSampleApp_Xcode_Version: '15.4'
CommunityToolkitLibrary_Xcode_Version: '15.4'
Expand Down Expand Up @@ -442,3 +443,58 @@ jobs:
inputs:
artifactName: nuget
pathToPublish: '$(Build.ArtifactStagingDirectory)'

- job: run_benchmarks
displayName: Run Benchmarks
strategy:
matrix:
'Windows':
image: 'windows-latest'
'macOS':
image: 'macos-14'
pool:
vmImage: $(image)
steps:
- script: |
sudo xcode-select -s /Applications/Xcode_$(CommunityToolkitLibrary_Xcode_Version).app
sudo xcode-select -p
displayName: 'Set Xcode v$(CommunityToolkitLibrary_Xcode_Version)'
condition: eq(variables['Agent.OS'], 'Darwin') # Only run this step on macOS
- task: UseDotNet@2
displayName: 'Install .NET SDK'
inputs:
packageType: 'sdk'
version: '$(LATEST_NET_VERSION)'
includePreviewVersions: false

- task: CmdLine@2
displayName: 'Install .NET MAUI Workload'
inputs:
script : 'dotnet workload install maui'

- pwsh: |
Invoke-WebRequest 'https://raw.githubusercontent.com/Samsung/Tizen.NET/main/workload/scripts/workload-install.ps1' -OutFile 'workload-install.ps1'
.\workload-install.ps1
displayName: Install Tizen Workload
# Print Information on the .NET SDK Used By the CI Build Host
# These logs are useful information when debugging CI Builds
# Note: This step doesn't execute nor modify any code; it is strictly used for logging + debugging purposes
- task: CmdLine@2
displayName: 'Display dotnet --info'
inputs:
script: dotnet --info

- task: CmdLine@2
displayName: 'Run Benchmarks'
inputs:
script : 'dotnet run --project $(PathToCommunityToolkitAnalyzersBenchmarkCsproj) -c Release -- -a $(Build.ArtifactStagingDirectory)'

# publish the Benchmark Results
- task: PublishBuildArtifacts@1
condition: eq(variables['Agent.OS'], 'Windows_NT') # Only run this step on Windows
displayName: 'Publish Benchmark Artifacts'
inputs:
artifactName: benchmarks
pathToPublish: '$(Build.ArtifactStagingDirectory)'
13 changes: 11 additions & 2 deletions samples/CommunityToolkit.Maui.Sample.sln
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# 17
# Visual Studio Version 17
VisualStudioVersion = 17.0.31521.260
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommunityToolkit.Maui.UnitTests", "..\src\CommunityToolkit.Maui.UnitTests\CommunityToolkit.Maui.UnitTests.csproj", "{7D49CC16-93CF-471B-B1FA-0BA44DECC15D}"
Expand Down Expand Up @@ -49,6 +49,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommunityToolkit.Maui.Camer
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommunityToolkit.Maui.Camera.Analyzers", "..\src\CommunityToolkit.Maui.Camera.Analyzers\CommunityToolkit.Maui.Camera.Analyzers.csproj", "{02C5B93A-B8D6-421D-B0EA-D0CC41A00F0B}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Benchmarks", "Benchmarks", "{ED5A9C0B-D270-442D-BABE-F4FF622926C8}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommunityToolkit.Maui.Analyzers.Benchmarks.csproj", "..\src\CommunityToolkit.Maui.Analyzers.Benchmarks\CommunityToolkit.Maui.Analyzers.Benchmarks.csproj", "{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -123,6 +127,10 @@ Global
{02C5B93A-B8D6-421D-B0EA-D0CC41A00F0B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{02C5B93A-B8D6-421D-B0EA-D0CC41A00F0B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{02C5B93A-B8D6-421D-B0EA-D0CC41A00F0B}.Release|Any CPU.Build.0 = Release|Any CPU
{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{B80F59B7-276C-4A55-A8DD-54587C8BC3D2}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand All @@ -137,8 +145,9 @@ Global
{85B875BD-62F6-4EC3-BCFF-4DC25E94BCAE} = {9BFC4026-BC8F-43E2-BAA9-5BC2D764D37D}
{215083C8-D9CA-48FA-8B0A-1D21A989D055} = {9BFC4026-BC8F-43E2-BAA9-5BC2D764D37D}
{8CDCF66E-D969-4BFD-A6E3-816BBE5F80B6} = {9BFC4026-BC8F-43E2-BAA9-5BC2D764D37D}
{02C5B93A-B8D6-421D-B0EA-D0CC41A00F0B} = {9BFC4026-BC8F-43E2-BAA9-5BC2D764D37D}
{372D6A40-A4E0-434A-A463-C001441C68EB} = {9BFC4026-BC8F-43E2-BAA9-5BC2D764D37D}
{02C5B93A-B8D6-421D-B0EA-D0CC41A00F0B} = {9BFC4026-BC8F-43E2-BAA9-5BC2D764D37D}
{B80F59B7-276C-4A55-A8DD-54587C8BC3D2} = {ED5A9C0B-D270-442D-BABE-F4FF622926C8}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {1E9E61C1-5CB7-4C8E-87BA-6C1D38238679}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>$(NetVersion)</TargetFramework>
<OutputType>Exe</OutputType>
</PropertyGroup>
<PropertyGroup>
<PlatformTarget>AnyCPU</PlatformTarget>
<DebugType>pdbonly</DebugType>
<DebugSymbols>true</DebugSymbols>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<Optimize>true</Optimize>
<Configuration>Release</Configuration>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
<PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.14.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\CommunityToolkit.Maui.Analyzers.UnitTests\CommunityToolkit.Maui.Analyzers.UnitTests.csproj" />
</ItemGroup>
</Project>
15 changes: 15 additions & 0 deletions src/CommunityToolkit.Maui.Analyzers.Benchmarks/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Running;

namespace CommunityToolkit.Maui.Analyzers.Benchmarks;

public class Program
{
public static void Main(string[] args)
{
var config = DefaultConfig.Instance;
BenchmarkRunner.Run<UseMauiCommunityToolkitAnalyzerBenchmarks>(config, args);
BenchmarkRunner.Run<UseMauiCommunityToolkitCameraAnalyzerBenchmarks>(config, args);
BenchmarkRunner.Run<UseMauiCommunityToolkitMediaElementInitializationAnalyzerBenchmarks>(config, args);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
using BenchmarkDotNet.Attributes;
using CommunityToolkit.Maui.Analyzers.UnitTests;

namespace CommunityToolkit.Maui.Analyzers.Benchmarks;

[MemoryDiagnoser]
public class UseMauiCommunityToolkitAnalyzerBenchmarks
{
static readonly UseCommunityToolkitInitializationAnalyzerTests useCommunityToolkitInitializationAnalyzerTests = new();

[Benchmark]
public Task VerifyNoErrorsWhenUseMauiCommunityToolkit()
{
return useCommunityToolkitInitializationAnalyzerTests.VerifyNoErrorsWhenUseMauiCommunityToolkit();
}

[Benchmark]
public Task VerifyNoErrorsWhenUseMauiCommunityToolkitHasAdditonalWhitespace()
{
return useCommunityToolkitInitializationAnalyzerTests.VerifyNoErrorsWhenUseMauiCommunityToolkitHasAdditonalWhitespace();
}

[Benchmark]
public Task VerifyErrorsWhenMissingUseMauiCommunityToolkit()
{
return useCommunityToolkitInitializationAnalyzerTests.VerifyErrorsWhenMissingUseMauiCommunityToolkit();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
using BenchmarkDotNet.Attributes;
using CommunityToolkit.Maui.Analyzers.UnitTests;

namespace CommunityToolkit.Maui.Analyzers.Benchmarks;

[MemoryDiagnoser]
public class UseMauiCommunityToolkitCameraAnalyzerBenchmarks
{
static readonly UseCommunityToolkitCameraInitializationAnalyzerTests useCommunityToolkitCameraInitializationAnalyzerTests = new();

[Benchmark]
public Task VerifyNoErrorsWhenUseMauiCommunityToolkitCamera()
{
return useCommunityToolkitCameraInitializationAnalyzerTests.VerifyNoErrorsWhenUseMauiCommunityToolkitCamera();
}

[Benchmark]
public Task VerifyNoErrorsWhenUseMauiCommunityToolkitCameraHasAdditonalWhitespace()
{
return useCommunityToolkitCameraInitializationAnalyzerTests.VerifyNoErrorsWhenUseMauiCommunityToolkitCameraHasAdditonalWhitespace();
}

[Benchmark]
public Task VerifyErrorsWhenMissingUseMauiCommunityToolkitCamera()
{
return useCommunityToolkitCameraInitializationAnalyzerTests.VerifyErrorsWhenMissingUseMauiCommunityToolkitCamera();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
using BenchmarkDotNet.Attributes;
using CommunityToolkit.Maui.Analyzers.UnitTests;

namespace CommunityToolkit.Maui.Analyzers.Benchmarks;

[MemoryDiagnoser]
public class UseMauiCommunityToolkitMediaElementInitializationAnalyzerBenchmarks
{
static readonly UseCommunityToolkitMediaElementInitializationAnalyzerTests useCommunityToolkitMediaElementInitializationAnalyzerTests = new();

[Benchmark]
public Task VerifyNoErrorsWhenUseMauiCommunityToolkitMediaElement()
{
return useCommunityToolkitMediaElementInitializationAnalyzerTests.VerifyNoErrorsWhenUseMauiCommunityToolkitMediaElement();
}

[Benchmark]
public Task VerifyNoErrorsWhenUseMauiCommunityToolkitMediaElementHasAdditonalWhitespace()
{
return useCommunityToolkitMediaElementInitializationAnalyzerTests.VerifyNoErrorsWhenUseMauiCommunityToolkitMediaElementHasAdditonalWhitespace();
}

[Benchmark]
public Task VerifyErrorsWhenMissingUseMauiCommunityToolkitMediaElement()
{
return useCommunityToolkitMediaElementInitializationAnalyzerTests.VerifyErrorsWhenMissingUseMauiCommunityToolkitMediaElement();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ public void UseCommunityToolkitMediaElementInitializationAnalyzerId()
[Fact]
public async Task VerifyNoErrorsWhenUseMauiCommunityToolkitCamera()
{
const string source = @"
const string source = /* language=C#-test */ """
namespace CommunityToolkit.Maui.Analyzers.UnitTests
{
using Microsoft.Maui.Controls.Hosting;
Expand All @@ -32,22 +32,23 @@ public static MauiApp CreateMauiApp()
.UseMauiCommunityToolkitCamera()
.ConfigureFonts(fonts =>
{
fonts.AddFont(""OpenSans-Regular.ttf"", ""OpenSansRegular"");
fonts.AddFont(""OpenSans-Semibold.ttf"", ""OpenSansSemibold"");
fonts.AddFont("OpenSans-Regular.ttf", "OpenSansRegular");
fonts.AddFont("OpenSans-Semibold.ttf", "OpenSansSemibold");
});

return builder.Build();
}
}
}";
}
""";

await VerifyCameraToolkitAnalyzer(source);
}

[Fact]
public async Task VerifyNoErrorsWhenUseMauiCommunityToolkitCameraHasAdditonalWhitespace()
{
const string source = @"
const string source = /* language=C#-test */ """
namespace CommunityToolkit.Maui.Analyzers.UnitTests
{
using Microsoft.Maui.Controls.Hosting;
Expand All @@ -63,22 +64,23 @@ public static MauiApp CreateMauiApp()
.UseMauiCommunityToolkitCamera ()
.ConfigureFonts(fonts =>
{
fonts.AddFont(""OpenSans-Regular.ttf"", ""OpenSansRegular"");
fonts.AddFont(""OpenSans-Semibold.ttf"", ""OpenSansSemibold"");
fonts.AddFont("OpenSans-Regular.ttf", "OpenSansRegular");
fonts.AddFont("OpenSans-Semibold.ttf", "OpenSansSemibold");
});

return builder.Build ();
}
}
}";
}
""";

await VerifyCameraToolkitAnalyzer(source);
}

[Fact]
public async Task VerifyErrorsWhenMissingUseMauiCommunityToolkitCamera()
{
const string source = @"
const string source = /* language=C#-test */ """
namespace CommunityToolkit.Maui.Analyzers.UnitTests
{
using Microsoft.Maui.Controls.Hosting;
Expand All @@ -93,16 +95,17 @@ public static MauiApp CreateMauiApp()
builder.UseMauiApp<Microsoft.Maui.Controls.Application>()
.ConfigureFonts(fonts =>
{
fonts.AddFont(""OpenSans-Regular.ttf"", ""OpenSansRegular"");
fonts.AddFont(""OpenSans-Semibold.ttf"", ""OpenSansSemibold"");
fonts.AddFont("OpenSans-Regular.ttf", "OpenSansRegular");
fonts.AddFont("OpenSans-Semibold.ttf", "OpenSansSemibold");
});

return builder.Build();
}
}
}";
}
""";

await VerifyCameraToolkitAnalyzer(source, Diagnostic().WithSpan(13, 4, 13, 61).WithSeverity(DiagnosticSeverity.Error));
await VerifyCameraToolkitAnalyzer(source, Diagnostic().WithSpan(12, 4, 12, 61).WithSeverity(DiagnosticSeverity.Error));
}

static Task VerifyCameraToolkitAnalyzer(string source, params DiagnosticResult[] diagnosticResults)
Expand Down
Loading

0 comments on commit b9c7854

Please sign in to comment.