From 9bd7ba8ef5a886c9a980f5df3d763cfecbcb3f6c Mon Sep 17 00:00:00 2001 From: Michael Lipin Date: Wed, 7 Jun 2023 02:37:46 +0300 Subject: [PATCH 01/10] Adding metrics to Azure Benchmark tool. --- .../Tools/Benchmark/BenchmarkConfig.cs | 16 ++++++ .../Tools/Benchmark/CosmosBenchmark.csproj | 8 +++ .../Tools/Benchmark/Fx/IExecutionStrategy.cs | 7 ++- .../Tools/Benchmark/Fx/IExecutor.cs | 6 ++- .../Benchmark/Fx/ParallelExecutionStrategy.cs | 16 ++++-- .../Benchmark/Fx/SerialOperationExecutor.cs | 44 ++++++++++++++-- .../Tools/Benchmark/Program.cs | 50 +++++++++++++++++-- .../Tools/Benchmark/ReservoirProvider.cs | 49 ++++++++++++++++++ 8 files changed, 184 insertions(+), 12 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ReservoirProvider.cs diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs index 6f89776370..fc075c38d2 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs @@ -11,6 +11,7 @@ namespace CosmosBenchmark using CommandLine; using Microsoft.Azure.Documents.Client; using Newtonsoft.Json; + using static CosmosBenchmark.ReservoirProvider; public class BenchmarkConfig { @@ -123,6 +124,21 @@ public class BenchmarkConfig [Option(Required = false, HelpText = "Container to publish results to")] public string ResultsContainer { get; set; } = "runsummary"; + [Option(Required = false, HelpText = "Indicates whether to write metrics to console.")] + public bool WriteMetricsToConsole { get; set; } + + [Option(Required = false, HelpText = "Application Insights instrumentation key")] + public string AppInsightsInstrumentationKey { get; set; } + + [Option(Required = false, HelpText = "Reporting interval in seconds")] + public int ReportingIntervalInSeconds { get; set; } + + [Option(Required = false, HelpText = "Defines the reservoir type. Valid values are: Uniform, SlidingWindow and ExponentialDecay. The default value is SlidingWindow.")] + public ReservoirTypes ReservoirType { get; set; } = ReservoirTypes.SlidingWindow; + + [Option(Required = false, HelpText = "The reservoir sample size.")] + public int ReservoirSampleSize { get; set; } = 1028; + internal int GetTaskCount(int containerThroughput) { int taskCount = this.DegreeOfParallelism; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj index c8ac26f5aa..0fad607d5a 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj @@ -17,9 +17,17 @@ + + + + + + + + diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs index b18fc34dc7..ed0bd2544e 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs @@ -6,6 +6,8 @@ namespace CosmosBenchmark { using System; using System.Threading.Tasks; + using App.Metrics; + using Microsoft.Extensions.Logging; internal interface IExecutionStrategy { @@ -19,7 +21,8 @@ public Task ExecuteAsync( BenchmarkConfig benchmarkConfig, int serialExecutorConcurrency, int serialExecutorIterationCount, - double warmupFraction); - + double warmupFraction, + ILogger logger, + IMetrics metrics); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs index be2ee2e761..54f68cefa0 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs @@ -6,6 +6,8 @@ namespace CosmosBenchmark { using System; using System.Threading.Tasks; + using App.Metrics; + using Microsoft.Extensions.Logging; internal interface IExecutor { @@ -17,6 +19,8 @@ public Task ExecuteAsync( int iterationCount, bool isWarmup, bool traceFailures, - Action completionCallback); + Action completionCallback, + ILogger logger, + IMetrics metrics); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs index e5dc7bb58d..bc7bf5fe27 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs @@ -10,6 +10,10 @@ namespace CosmosBenchmark using System.Linq; using System.Threading; using System.Threading.Tasks; + using App.Metrics; + using Microsoft.ApplicationInsights; + using Microsoft.ApplicationInsights.Extensibility; + using Microsoft.Extensions.Logging; using Newtonsoft.Json; internal class ParallelExecutionStrategy : IExecutionStrategy @@ -28,7 +32,9 @@ public async Task ExecuteAsync( BenchmarkConfig benchmarkConfig, int serialExecutorConcurrency, int serialExecutorIterationCount, - double warmupFraction) + double warmupFraction, + ILogger logger, + IMetrics metrics) { IExecutor warmupExecutor = new SerialOperationExecutor( executorId: "Warmup", @@ -37,7 +43,9 @@ await warmupExecutor.ExecuteAsync( (int)(serialExecutorIterationCount * warmupFraction), isWarmup: true, traceFailures: benchmarkConfig.TraceFailures, - completionCallback: () => { }); + completionCallback: () => { }, + logger, + metrics); IExecutor[] executors = new IExecutor[serialExecutorConcurrency]; for (int i = 0; i < serialExecutorConcurrency; i++) @@ -54,7 +62,9 @@ await warmupExecutor.ExecuteAsync( iterationCount: serialExecutorIterationCount, isWarmup: false, traceFailures: benchmarkConfig.TraceFailures, - completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount)); + completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount), + logger, + metrics); } return await this.LogOutputStats( diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs index 1086c31bd5..32d0e527a8 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs @@ -7,13 +7,21 @@ namespace CosmosBenchmark using System; using System.Diagnostics; using System.Threading.Tasks; + using App.Metrics; + using App.Metrics.Counter; + using App.Metrics.Logging; + using App.Metrics.Timer; using Microsoft.Azure.Cosmos; + using Microsoft.Extensions.Logging; internal class SerialOperationExecutor : IExecutor { private readonly IBenchmarkOperation operation; private readonly string executorId; + // TODO: Move to config. + private const string LoggingContextIdentifier = "CosmosDBBenchmarkLoggingContext"; + public SerialOperationExecutor( string executorId, IBenchmarkOperation benchmarkOperation) @@ -34,9 +42,31 @@ public async Task ExecuteAsync( int iterationCount, bool isWarmup, bool traceFailures, - Action completionCallback) + Action completionCallback, + ILogger logger, + IMetrics metrics) { - Trace.TraceInformation($"Executor {this.executorId} started"); + logger.LogInformation($"Executor {this.executorId} started"); + + logger.LogInformation("Initializing counters and metrics."); + CounterOptions readSuccessMeter = new CounterOptions { Name = "#Read Successful Operations", Context = LoggingContextIdentifier }; + CounterOptions readFailureMeter = new CounterOptions { Name = "#Read Unsuccessful Operations", Context = LoggingContextIdentifier }; + CounterOptions writeSuccessMeter = new CounterOptions { Name = "#Write Successful Operations", Context = LoggingContextIdentifier }; + CounterOptions writeFailureMeter = new CounterOptions { Name = "#Write Unsuccessful Operations", Context = LoggingContextIdentifier }; + CounterOptions querySuccessMeter = new CounterOptions { Name = "#Query Successful Operations", Context = LoggingContextIdentifier }; + CounterOptions queryFailureMeter = new CounterOptions { Name = "#Query Unsuccessful Operations", Context = LoggingContextIdentifier }; + + TimerOptions readLatencyTimer = new() + { + Name = "Read latency", + MeasurementUnit = Unit.Requests, + DurationUnit = TimeUnit.Milliseconds, + RateUnit = TimeUnit.Seconds, + Context = LoggingContextIdentifier, + + // TODO: Pass config. + Reservoir = () => ReservoirProvider.GetReservoir(new BenchmarkConfig()) + }; try { @@ -51,9 +81,17 @@ public async Task ExecuteAsync( () => operationResult.Value, disableTelemetry: isWarmup)) { + try { - operationResult = await this.operation.ExecuteOnceAsync(); + using (TimerContext timerContext = metrics.Measure.Timer.Time(readLatencyTimer)) + { + operationResult = await this.operation.ExecuteOnceAsync(); + } + + // TODO: Move to operation implementation. + // if (this.operation.GetType() == typeof(ReadTExistsV3BenchmarkOperation)) + metrics.Measure.Counter.Increment(readSuccessMeter); // Success case this.SuccessOperationCount++; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs index 99131262d7..6236f1bf05 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs @@ -12,9 +12,14 @@ namespace CosmosBenchmark using System.Net; using System.Net.Http; using System.Reflection; + using System.Text; using System.Threading; using System.Threading.Tasks; + using App.Metrics; + using App.Metrics.Formatters.Json; + using App.Metrics.Scheduling; using Microsoft.Azure.Cosmos; + using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; /// @@ -28,6 +33,11 @@ public sealed class Program /// command line arguments. public static async Task Main(string[] args) { + using ILoggerFactory loggerFactory = LoggerFactory.Create(builder => builder + .AddConsole()); + + ILogger logger = loggerFactory.CreateLogger(); + try { BenchmarkConfig config = BenchmarkConfig.From(args); @@ -45,7 +55,16 @@ public static async Task Main(string[] args) Program program = new Program(); - RunSummary runSummary = await program.ExecuteAsync(config); + IMetricsRoot metrics = ConfigureReporting(config, logger); + + AppMetricsTaskScheduler scheduler = new AppMetricsTaskScheduler( + TimeSpan.FromSeconds(3), + async () => await Task.WhenAll(metrics.ReportRunner.RunAllAsync())); + scheduler.Start(); + + RunSummary runSummary = await program.ExecuteAsync(config, logger, metrics); + + await Task.WhenAll(metrics.ReportRunner.RunAllAsync()); } finally { @@ -58,6 +77,31 @@ public static async Task Main(string[] args) } } + private static IMetricsRoot ConfigureReporting( + BenchmarkConfig config, + ILogger logger) + { + + if (config.WriteMetricsToConsole) + { + return new MetricsBuilder() + .Report.ToConsole( + options => + { + options.FlushInterval = TimeSpan.FromSeconds(config.ReportingIntervalInSeconds); + options.MetricsOutputFormatter = new MetricsJsonOutputFormatter(); + }) + .Build(); + } + + string connectionString = $"InstrumentationKey={config.AppInsightsInstrumentationKey}"; + MetricsOptions metricsOptions = new MetricsOptions(); + return new MetricsBuilder() + .Configuration.Configure(metricsOptions) + .Report.ToApplicationInsights(connectionString) + .Build(); + } + private static async Task AddAzureInfoToRunSummary() { using HttpClient httpClient = new HttpClient(); @@ -85,7 +129,7 @@ private static async Task AddAzureInfoToRunSummary() /// Executing benchmarks for V2/V3 cosmosdb SDK. /// /// a Task object. - private async Task ExecuteAsync(BenchmarkConfig config) + private async Task ExecuteAsync(BenchmarkConfig config, ILogger logger, IMetrics metrics) { // V3 SDK client initialization using (CosmosClient cosmosClient = config.CreateCosmosClient(config.Key)) @@ -137,7 +181,7 @@ private async Task ExecuteAsync(BenchmarkConfig config) } IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory); - runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01); + runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, logger, metrics); } if (config.CleanupOnFinish) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ReservoirProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ReservoirProvider.cs new file mode 100644 index 0000000000..5a3e2ab452 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ReservoirProvider.cs @@ -0,0 +1,49 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using System; + using App.Metrics.ReservoirSampling; + + /// + /// Returns the based on the CTL configuration. + /// + public class ReservoirProvider + { + /// + /// Create and returns a new instance of the based on the Benchmark configuration. + /// + /// An instance of containing the Benchmark config params. + /// An implementation of . + public static IReservoir GetReservoir(BenchmarkConfig benchmarkConfig) + { + return benchmarkConfig.ReservoirType switch + { + ReservoirTypes.Uniform => new App.Metrics.ReservoirSampling.Uniform.DefaultAlgorithmRReservoir( + sampleSize: benchmarkConfig.ReservoirSampleSize), + + ReservoirTypes.SlidingWindow => new App.Metrics.ReservoirSampling.SlidingWindow.DefaultSlidingWindowReservoir( + sampleSize: benchmarkConfig.ReservoirSampleSize), + + ReservoirTypes.ExponentialDecay => new App.Metrics.ReservoirSampling.ExponentialDecay.DefaultForwardDecayingReservoir( + sampleSize: benchmarkConfig.ReservoirSampleSize, + alpha: 0.015), + + _ => throw new ArgumentException( + message: "Invalid ReservoirType Specified."), + }; + } + + /// + /// An enum containing different reservoir types. + /// + public enum ReservoirTypes + { + Uniform, + SlidingWindow, + ExponentialDecay + } + } +} From b34ba6edc6829a8b79318b71ec200a88e35856e6 Mon Sep 17 00:00:00 2001 From: Michael Lipin Date: Wed, 7 Jun 2023 19:34:26 +0300 Subject: [PATCH 02/10] Adding 999 and 9999 quantiles. --- .../Benchmark/Fx/ParallelExecutionStrategy.cs | 14 ++++++++++++++ .../Tools/Benchmark/Fx/TelemetrySpan.cs | 10 ++++++++++ .../Tools/Benchmark/RunSummary.cs | 2 ++ 3 files changed, 26 insertions(+) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs index bc7bf5fe27..7516bb6569 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs @@ -112,6 +112,16 @@ private async Task LogOutputStats( perLoopCounters.Add((int)diff.Rps()); await Task.Delay(TimeSpan.FromSeconds(outputLoopDelayInSeconds)); + + Console.WriteLine("P50: " + TelemetrySpan.GetLatencyPercentile(50)); + Console.WriteLine("P75: " + TelemetrySpan.GetLatencyPercentile(75)); + Console.WriteLine("P90: " + TelemetrySpan.GetLatencyPercentile(90)); + Console.WriteLine("P95: " + TelemetrySpan.GetLatencyPercentile(95)); + Console.WriteLine("P98: " + TelemetrySpan.GetLatencyPercentile(98)); + Console.WriteLine("P99: " + TelemetrySpan.GetLatencyPercentile(99)); + Console.WriteLine("P999: " + TelemetrySpan.GetLatencyQuantile(0.999)); + Console.WriteLine("P9999: " + TelemetrySpan.GetLatencyQuantile(0.9999)); + Console.WriteLine("P100: " + TelemetrySpan.GetLatencyPercentile(100)); } while (!isLastIterationCompleted); @@ -145,6 +155,8 @@ private async Task LogOutputStats( runSummary.Top80PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.8 * summaryCounters.Length)).Average(), 0); runSummary.Top90PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.9 * summaryCounters.Length)).Average(), 0); runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.95 * summaryCounters.Length)).Average(), 0); + runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.999 * summaryCounters.Length)).Average(), 0); + runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.9999 * summaryCounters.Length)).Average(), 0); runSummary.Top99PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.99 * summaryCounters.Length)).Average(), 0); runSummary.AverageRps = Math.Round(summaryCounters.Average(), 0); @@ -154,6 +166,8 @@ private async Task LogOutputStats( runSummary.Top95PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(95); runSummary.Top98PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(98); runSummary.Top99PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(99); + runSummary.Top999PercentLatencyInMs = TelemetrySpan.GetLatencyQuantile(0.999); + runSummary.Top9999PercentLatencyInMs = TelemetrySpan.GetLatencyQuantile(0.9999); runSummary.MaxLatencyInMs = TelemetrySpan.GetLatencyPercentile(100); string summary = JsonConvert.SerializeObject(runSummary); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs index 13d971aa2a..c0a44a737b 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs @@ -79,6 +79,16 @@ internal static void ResetLatencyHistogram(int totalNumberOfIterations) return MathNet.Numerics.Statistics.Statistics.Percentile(latencyHistogram.Take(latencyIndex + 1), percentile); } + internal static double? GetLatencyQuantile(double quantile) + { + if (latencyHistogram == null) + { + return null; + } + + return MathNet.Numerics.Statistics.Statistics.Quantile(latencyHistogram.Take(latencyIndex + 1), quantile); + } + private class NoOpDisposable : IDisposable { public static readonly NoOpDisposable Instance = new NoOpDisposable(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs index 4f84322554..7b74b8702a 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/RunSummary.cs @@ -79,6 +79,8 @@ public RunSummary( public double? Top95PercentLatencyInMs { get; set; } public double? Top98PercentLatencyInMs { get; set; } public double? Top99PercentLatencyInMs { get; set; } + public double? Top999PercentLatencyInMs { get; set; } + public double? Top9999PercentLatencyInMs { get; set; } public double? MaxLatencyInMs { get; set; } public double AverageRps { get; set; } From 36308e6c2693eced29b668536e0632ebbfe1cefc Mon Sep 17 00:00:00 2001 From: Michael Lipin Date: Wed, 7 Jun 2023 19:48:48 +0300 Subject: [PATCH 03/10] Base classes for operations. --- .../Tools/Benchmark/BenchmarkConfig.cs | 2 +- .../Benchmark/BenchmarkConfigProvider.cs | 16 +++++ .../Tools/Benchmark/Fx/IExecutionStrategy.cs | 4 +- .../Tools/Benchmark/Fx/IMetricsCollector.cs | 15 +++++ .../Tools/Benchmark/Fx/InsertBenchmarkBase.cs | 11 ++++ .../Benchmark/Fx/ParallelExecutionStrategy.cs | 13 ++-- .../Tools/Benchmark/Fx/QueryBenchmarkBase.cs | 11 ++++ .../Tools/Benchmark/Fx/ReadBenchmarkBase.cs | 11 ++++ .../Benchmark/Fx/SerialOperationExecutor.cs | 38 +++++------- .../Tools/Benchmark/MetricsContext.cs | 51 ++++++++++++++++ .../Tools/Benchmark/Program.cs | 60 +++++++++++++++---- .../v2/InsertV2BenchmarkOperation.cs | 2 +- ...QueryStreamSinglePkV2BenchmarkOperation.cs | 2 +- .../v2/QueryTSinglePkV2BenchmarkOperation.cs | 2 +- .../v2/ReadFeedStreamV2BenchmarkOperation.cs | 2 +- .../v2/ReadNotExistsV2BenchmarkOperation.cs | 2 +- .../ReadStreamExistsV2BenchmarkOperation.cs | 2 +- .../v2/ReadTExistsV2BenchmarkOperation.cs | 2 +- .../v3/InsertV3BenchmarkOperation.cs | 2 +- .../v3/QueryTV3BenchmarkOperation.cs | 2 +- .../v3/ReadFeedStreamV3BenchmarkOperation.cs | 2 +- .../v3/ReadNotExistsV3BenchmarkOperation.cs | 2 +- .../ReadStreamExistsV3BenchmarkOperation.cs | 2 +- ...istsWithDiagnosticsV3BenchmarkOperation.cs | 2 +- .../v3/ReadTExistsV3BenchmarkOperation.cs | 2 +- 25 files changed, 204 insertions(+), 56 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfigProvider.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkBase.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkBase.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkBase.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs index fc075c38d2..5fd7a594f5 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs @@ -286,4 +286,4 @@ private static void HandleParseError(IEnumerable errors) Environment.Exit(errors.Count()); } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfigProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfigProvider.cs new file mode 100644 index 0000000000..ac54a8abdc --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfigProvider.cs @@ -0,0 +1,16 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using App.Metrics; + using App.Metrics.Counter; + using App.Metrics.Timer; + + // TODO: Temporary solution. Remove after adding DI. + public class BenchmarkConfigProvider + { + public static BenchmarkConfig CurrentBenchmarkConfig { get; set; } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs index ed0bd2544e..2670c7fa16 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs @@ -12,9 +12,9 @@ namespace CosmosBenchmark internal interface IExecutionStrategy { public static IExecutionStrategy StartNew( - Func benchmarkOperation) + Func benchmarkOperation, IMetricsCollector metricsCollector) { - return new ParallelExecutionStrategy(benchmarkOperation); + return new ParallelExecutionStrategy(benchmarkOperation, metricsCollector); } public Task ExecuteAsync( diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs new file mode 100644 index 0000000000..88bbb2e880 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs @@ -0,0 +1,15 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +using App.Metrics; + +namespace CosmosBenchmark +{ + public interface IMetricsCollector + { + void CollectMetricsOnSuccess(MetricsContext metricsContext, IMetrics metrics); + + void CollectMetricsOnFailure(MetricsContext metricsContext, IMetrics metrics); + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkBase.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkBase.cs new file mode 100644 index 0000000000..afd230e8f4 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkBase.cs @@ -0,0 +1,11 @@ +using App.Metrics; + +namespace CosmosBenchmark +{ + public class InsertBenchmarkBase : IMetricsCollector + { + public void CollectMetricsOnSuccess(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.WriteSuccessMeter); + + public void CollectMetricsOnFailure(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.WriteFailureMeter); + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs index 7516bb6569..db1653f93f 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs @@ -20,12 +20,15 @@ internal class ParallelExecutionStrategy : IExecutionStrategy { private readonly Func benchmarkOperation; + private readonly IMetricsCollector metricsCollector; + private volatile int pendingExecutorCount; public ParallelExecutionStrategy( - Func benchmarkOperation) + Func benchmarkOperation, IMetricsCollector metricsCollector) { this.benchmarkOperation = benchmarkOperation; + this.metricsCollector = metricsCollector; } public async Task ExecuteAsync( @@ -38,7 +41,8 @@ public async Task ExecuteAsync( { IExecutor warmupExecutor = new SerialOperationExecutor( executorId: "Warmup", - benchmarkOperation: this.benchmarkOperation()); + benchmarkOperation: this.benchmarkOperation(), + metricsCollector); await warmupExecutor.ExecuteAsync( (int)(serialExecutorIterationCount * warmupFraction), isWarmup: true, @@ -52,7 +56,8 @@ await warmupExecutor.ExecuteAsync( { executors[i] = new SerialOperationExecutor( executorId: i.ToString(), - benchmarkOperation: this.benchmarkOperation()); + benchmarkOperation: this.benchmarkOperation(), + metricsCollector); } this.pendingExecutorCount = serialExecutorConcurrency; @@ -184,4 +189,4 @@ private async Task LogOutputStats( } } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkBase.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkBase.cs new file mode 100644 index 0000000000..0a19d0ad5a --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkBase.cs @@ -0,0 +1,11 @@ +using App.Metrics; + +namespace CosmosBenchmark +{ + public class QueryBenchmarkBase : IMetricsCollector + { + public void CollectMetricsOnSuccess(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.QuerySuccessMeter); + + public void CollectMetricsOnFailure(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.QueryFailureMeter); + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkBase.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkBase.cs new file mode 100644 index 0000000000..624ed6640f --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkBase.cs @@ -0,0 +1,11 @@ +using App.Metrics; + +namespace CosmosBenchmark +{ + public class ReadBenchmarkBase : IMetricsCollector + { + public void CollectMetricsOnSuccess(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.ReadSuccessMeter); + + public void CollectMetricsOnFailure(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.ReadFailureMeter); + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs index 32d0e527a8..b7659d9980 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs @@ -17,6 +17,9 @@ namespace CosmosBenchmark internal class SerialOperationExecutor : IExecutor { private readonly IBenchmarkOperation operation; + + private readonly IMetricsCollector metricsCollector; + private readonly string executorId; // TODO: Move to config. @@ -24,13 +27,16 @@ internal class SerialOperationExecutor : IExecutor public SerialOperationExecutor( string executorId, - IBenchmarkOperation benchmarkOperation) + IBenchmarkOperation benchmarkOperation, + IMetricsCollector metricsCollector) { this.executorId = executorId; this.operation = benchmarkOperation; this.SuccessOperationCount = 0; this.FailedOperationCount = 0; + + this.metricsCollector = metricsCollector; } public int SuccessOperationCount { get; private set; } @@ -49,24 +55,8 @@ public async Task ExecuteAsync( logger.LogInformation($"Executor {this.executorId} started"); logger.LogInformation("Initializing counters and metrics."); - CounterOptions readSuccessMeter = new CounterOptions { Name = "#Read Successful Operations", Context = LoggingContextIdentifier }; - CounterOptions readFailureMeter = new CounterOptions { Name = "#Read Unsuccessful Operations", Context = LoggingContextIdentifier }; - CounterOptions writeSuccessMeter = new CounterOptions { Name = "#Write Successful Operations", Context = LoggingContextIdentifier }; - CounterOptions writeFailureMeter = new CounterOptions { Name = "#Write Unsuccessful Operations", Context = LoggingContextIdentifier }; - CounterOptions querySuccessMeter = new CounterOptions { Name = "#Query Successful Operations", Context = LoggingContextIdentifier }; - CounterOptions queryFailureMeter = new CounterOptions { Name = "#Query Unsuccessful Operations", Context = LoggingContextIdentifier }; - - TimerOptions readLatencyTimer = new() - { - Name = "Read latency", - MeasurementUnit = Unit.Requests, - DurationUnit = TimeUnit.Milliseconds, - RateUnit = TimeUnit.Seconds, - Context = LoggingContextIdentifier, - // TODO: Pass config. - Reservoir = () => ReservoirProvider.GetReservoir(new BenchmarkConfig()) - }; + var metricsContext = new MetricsContext(operation.GetType().Name, BenchmarkConfigProvider.CurrentBenchmarkConfig); try { @@ -84,14 +74,12 @@ public async Task ExecuteAsync( try { - using (TimerContext timerContext = metrics.Measure.Timer.Time(readLatencyTimer)) + using (TimerContext timerContext = metrics.Measure.Timer.Time(metricsContext.LatencyTimer)) { - operationResult = await this.operation.ExecuteOnceAsync(); + operationResult = await operation.ExecuteOnceAsync(); } - // TODO: Move to operation implementation. - // if (this.operation.GetType() == typeof(ReadTExistsV3BenchmarkOperation)) - metrics.Measure.Counter.Increment(readSuccessMeter); + metricsCollector.CollectMetricsOnSuccess(metricsContext, metrics); // Success case this.SuccessOperationCount++; @@ -109,6 +97,8 @@ public async Task ExecuteAsync( Console.WriteLine(ex.ToString()); } + metricsCollector.CollectMetricsOnFailure(metricsContext, metrics); + // failure case this.FailedOperationCount++; @@ -140,4 +130,4 @@ public async Task ExecuteAsync( } } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs new file mode 100644 index 0000000000..2e787990c0 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs @@ -0,0 +1,51 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using App.Metrics; + using App.Metrics.Counter; + using App.Metrics.Timer; + + public class MetricsContext + { + public string ContextId { get; } + + public CounterOptions ReadSuccessMeter { get; } + + public CounterOptions ReadFailureMeter { get; } + + public CounterOptions WriteSuccessMeter { get; } + + public CounterOptions WriteFailureMeter { get; } + + public CounterOptions QuerySuccessMeter { get; } + + public CounterOptions QueryFailureMeter { get; } + + public TimerOptions LatencyTimer { get; } + + public MetricsContext(string contextId, BenchmarkConfig benchmarkConfig) + { + ContextId = contextId; + + ReadSuccessMeter = new CounterOptions { Name = "#Read Successful Operations", Context = ContextId }; + ReadFailureMeter = new CounterOptions { Name = "#Read Unsuccessful Operations", Context = ContextId }; + WriteSuccessMeter = new CounterOptions { Name = "#Write Successful Operations", Context = ContextId }; + WriteFailureMeter = new CounterOptions { Name = "#Write Unsuccessful Operations", Context = ContextId }; + QuerySuccessMeter = new CounterOptions { Name = "#Query Successful Operations", Context = ContextId }; + QueryFailureMeter = new CounterOptions { Name = "#Query Unsuccessful Operations", Context = ContextId }; + + LatencyTimer = new() + { + Name = "Latency", + MeasurementUnit = Unit.Requests, + DurationUnit = TimeUnit.Milliseconds, + RateUnit = TimeUnit.Seconds, + Context = ContextId, + Reservoir = () => ReservoirProvider.GetReservoir(benchmarkConfig) + }; + } + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs index 6236f1bf05..7aa96be351 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs @@ -42,7 +42,7 @@ public static async Task Main(string[] args) { BenchmarkConfig config = BenchmarkConfig.From(args); await Program.AddAzureInfoToRunSummary(); - + ThreadPool.SetMinThreads(config.MinThreadPoolSize, config.MinThreadPoolSize); if (config.EnableLatencyPercentiles) @@ -62,6 +62,9 @@ public static async Task Main(string[] args) async () => await Task.WhenAll(metrics.ReportRunner.RunAllAsync())); scheduler.Start(); + // TODO: Temporary solution. Remove after adding DI. + BenchmarkConfigProvider.CurrentBenchmarkConfig = config; + RunSummary runSummary = await program.ExecuteAsync(config, logger, metrics); await Task.WhenAll(metrics.ReportRunner.RunAllAsync()); @@ -119,7 +122,7 @@ private static async Task AddAzureInfoToRunSummary() RunSummary.Location = jObject["compute"]["location"].ToString(); Console.WriteLine($"Azure VM Location:{RunSummary.Location}"); } - catch(Exception e) + catch (Exception e) { Console.WriteLine("Failed to get Azure VM info:" + e.ToString()); } @@ -174,14 +177,17 @@ private async Task ExecuteAsync(BenchmarkConfig config, ILogger logg cosmosClient, documentClient); + if (config.DisableCoreSdkLogging) { // Do it after client initialization (HACK) Program.ClearCoreSdkListeners(); } - IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory); - runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, logger, metrics); + IMetricsCollector metricsCollector = this.GetMetricsCollector(config); + + IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory, metricsCollector); + runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, logger, metrics); } if (config.CleanupOnFinish) @@ -203,8 +209,8 @@ private async Task ExecuteAsync(BenchmarkConfig config, ILogger logg { runSummary.Diagnostics = CosmosDiagnosticsLogger.GetDiagnostics(); await this.PublishResults( - config, - runSummary, + config, + runSummary, cosmosClient); } @@ -213,8 +219,8 @@ await this.PublishResults( } private async Task PublishResults( - BenchmarkConfig config, - RunSummary runSummary, + BenchmarkConfig config, + RunSummary runSummary, CosmosClient benchmarkClient) { if (string.IsNullOrEmpty(config.ResultsEndpoint)) @@ -288,6 +294,38 @@ private Func GetBenchmarkFactory( return () => (IBenchmarkOperation)ci.Invoke(ctorArguments); } + private IMetricsCollector GetMetricsCollector(BenchmarkConfig config) + { + Type metricsCollectorType = typeof(IMetricsCollector); + + var availableBenchmarks = typeof(Program).Assembly.GetTypes() + .Where(p => metricsCollectorType.IsAssignableFrom(p)) + .ToArray(); + + IEnumerable res = availableBenchmarks + .Where(e => e.Name.Equals(config.WorkloadType, StringComparison.OrdinalIgnoreCase) || e.Name.Equals(config.WorkloadType + "BenchmarkOperation", StringComparison.OrdinalIgnoreCase)); + + if (res.Count() != 1) + { + throw new NotImplementedException($"Unsupported workload type {config.WorkloadType}. Available ones are " + + string.Join(", \r\n", availableBenchmarks.Select(e => e.Name))); + } + + Type metricsCollectorTypeName = res.Single(); + + BindingFlags flags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance; + var ci = metricsCollectorTypeName + .BaseType + .GetConstructors(flags) + .Where(constructor => constructor.GetParameters().Length == 0) + .First(); + + var metricsCollector = (IMetricsCollector)ci.Invoke(null); + + + return metricsCollector; + } + private static Type[] AvailableBenchmarks() { Type benchmarkType = typeof(IBenchmarkOperation); @@ -310,8 +348,8 @@ private static async Task CreatePartitionedContainerAsync(Ben { return await container.ReadContainerAsync(); } - catch(CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) - { + catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { // Show user cost of running this test double estimatedCostPerMonth = 0.06 * options.Throughput; double estimatedCostPerHour = estimatedCostPerMonth / (24 * 30); @@ -332,4 +370,4 @@ private static void ClearCoreSdkListeners() traceSource.Listeners.Clear(); } } -} +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs index 744bb9f577..a910651216 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs @@ -10,7 +10,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class InsertV2BenchmarkOperation : IBenchmarkOperation + internal class InsertV2BenchmarkOperation : InsertBenchmarkBase, IBenchmarkOperation { private readonly DocumentClient documentClient; private readonly Uri containerUri; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs index 1bf3bf245e..659def21fe 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents.Client; using Microsoft.Azure.Documents.Linq; - internal class QueryStreamSinglePkV2BenchmarkOperation : IBenchmarkOperation + internal class QueryStreamSinglePkV2BenchmarkOperation : QueryBenchmarkBase, IBenchmarkOperation { private readonly DocumentClient documentClient; private readonly string partitionKeyPath; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs index 7d58447859..077b8fadf8 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents.Client; using Microsoft.Azure.Documents.Linq; - internal class QueryTSinglePkV2BenchmarkOperation : IBenchmarkOperation + internal class QueryTSinglePkV2BenchmarkOperation : QueryBenchmarkBase, IBenchmarkOperation { private readonly DocumentClient documentClient; private readonly string partitionKeyPath; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs index 379cd363e6..b81fab7756 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class ReadFeedStreamV2BenchmarkOperation : IBenchmarkOperation + internal class ReadFeedStreamV2BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation { private readonly DocumentClient documentClient; private readonly string partitionKeyPath; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs index f5e8f94c68..707709d9dc 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class ReadNotExistsV2BenchmarkOperation : IBenchmarkOperation + internal class ReadNotExistsV2BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation { private readonly string databsaeName; private readonly string containerName; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs index 7afd4cdf71..fdbbce51b5 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class ReadStreamExistsV2BenchmarkOperation : IBenchmarkOperation + internal class ReadStreamExistsV2BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation { private readonly string partitionKeyPath; private readonly Dictionary sampleJObject; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs index 061183c575..938a3a083e 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs @@ -13,7 +13,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents.Client; using Newtonsoft.Json.Linq; - internal class ReadTExistsV2BenchmarkOperation : IBenchmarkOperation + internal class ReadTExistsV2BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation { private readonly string partitionKeyPath; private readonly Dictionary sampleJObject; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs index 1ab5861242..f549ed4d4a 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Newtonsoft.Json; using Newtonsoft.Json.Serialization; - internal class InsertV3BenchmarkOperation : IBenchmarkOperation + internal class InsertV3BenchmarkOperation : InsertBenchmarkBase, IBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs index bdb0b38cf8..7c6506bc70 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal abstract class QueryTV3BenchmarkOperation : IBenchmarkOperation + internal abstract class QueryTV3BenchmarkOperation : QueryBenchmarkBase, IBenchmarkOperation { protected readonly Container container; protected readonly Dictionary sampleJObject; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs index 99789040b0..3d5559388d 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadFeedStreamV3BenchmarkOperation : IBenchmarkOperation + internal class ReadFeedStreamV3BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs index c97d1af4b7..2190e76ca3 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs @@ -9,7 +9,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadNotExistsV3BenchmarkOperation : IBenchmarkOperation + internal class ReadNotExistsV3BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation { private readonly Container container; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs index aba0a4ea7a..3abd14201e 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadStreamExistsV3BenchmarkOperation : IBenchmarkOperation + internal class ReadStreamExistsV3BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs index ebdc14924b..de4ba3ecd7 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadStreamExistsWithDiagnosticsV3BenchmarkOperation : IBenchmarkOperation + internal class ReadStreamExistsWithDiagnosticsV3BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs index 7fd40397be..e853862ef9 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Cosmos; using Newtonsoft.Json.Linq; - internal class ReadTExistsV3BenchmarkOperation : IBenchmarkOperation + internal class ReadTExistsV3BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; From f450f54143fc72b74ed830de7601b441ed66229d Mon Sep 17 00:00:00 2001 From: Michael Lipin Date: Thu, 8 Jun 2023 23:30:36 +0300 Subject: [PATCH 04/10] Separating operations and metrics collecting, refactoring, code clean-up. --- .../Tools/Benchmark/BenchmarkConfig.cs | 10 +-- .../Tools/Benchmark/Fx/BenchmarkOperation.cs | 15 ++++ .../Tools/Benchmark/Fx/IExecutionStrategy.cs | 4 +- .../Tools/Benchmark/Fx/IExecutor.cs | 3 +- .../Tools/Benchmark/Fx/IMetricsCollector.cs | 10 ++- .../Tools/Benchmark/Fx/InsertBenchmarkBase.cs | 11 --- .../Benchmark/Fx/InsertBenchmarkOperation.cs | 13 ++++ .../Fx/InsertOperationMetricsCollector.cs | 31 ++++++++ .../Tools/Benchmark/Fx/MetricsCollector.cs | 28 +++++++ .../Benchmark/Fx/MetricsCollectorProvider.cs | 33 ++++++++ .../Benchmark/Fx/ParallelExecutionStrategy.cs | 35 +++------ .../Tools/Benchmark/Fx/QueryBenchmarkBase.cs | 11 --- .../Benchmark/Fx/QueryBenchmarkOperation.cs | 10 +++ .../Fx/QueryOperationMetricsCollector.cs | 31 ++++++++ .../Tools/Benchmark/Fx/ReadBenchmarkBase.cs | 11 --- .../Benchmark/Fx/ReadBenchmarkOperation.cs | 10 +++ .../Fx/ReadOperationMetricsCollector.cs | 31 ++++++++ .../Benchmark/Fx/SerialOperationExecutor.cs | 24 +++--- .../Tools/Benchmark/IBenchmarkOperation.cs | 4 +- .../Tools/Benchmark/MetricsContext.cs | 52 +++++++++---- .../Tools/Benchmark/Program.cs | 76 +++++-------------- .../v2/InsertV2BenchmarkOperation.cs | 6 +- ...QueryStreamSinglePkV2BenchmarkOperation.cs | 6 +- .../v2/QueryTSinglePkV2BenchmarkOperation.cs | 6 +- .../v2/ReadFeedStreamV2BenchmarkOperation.cs | 6 +- .../v2/ReadNotExistsV2BenchmarkOperation.cs | 6 +- .../ReadStreamExistsV2BenchmarkOperation.cs | 6 +- .../v2/ReadTExistsV2BenchmarkOperation.cs | 6 +- .../v3/InsertV3BenchmarkOperation.cs | 6 +- .../v3/QueryTV3BenchmarkOperation.cs | 6 +- .../v3/ReadFeedStreamV3BenchmarkOperation.cs | 6 +- .../v3/ReadNotExistsV3BenchmarkOperation.cs | 6 +- .../ReadStreamExistsV3BenchmarkOperation.cs | 6 +- ...istsWithDiagnosticsV3BenchmarkOperation.cs | 6 +- .../v3/ReadTExistsV3BenchmarkOperation.cs | 6 +- 35 files changed, 338 insertions(+), 199 deletions(-) create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/BenchmarkOperation.cs delete mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkBase.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs delete mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkBase.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkOperation.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs delete mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkBase.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkOperation.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs index 5fd7a594f5..c2cdb056f1 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs @@ -124,21 +124,21 @@ public class BenchmarkConfig [Option(Required = false, HelpText = "Container to publish results to")] public string ResultsContainer { get; set; } = "runsummary"; - [Option(Required = false, HelpText = "Indicates whether to write metrics to console.")] - public bool WriteMetricsToConsole { get; set; } + [Option(Required = false, HelpText = "Metrics reporting interval in seconds")] + public int MetricsReportingIntervalInSec { get; set; } [Option(Required = false, HelpText = "Application Insights instrumentation key")] public string AppInsightsInstrumentationKey { get; set; } - [Option(Required = false, HelpText = "Reporting interval in seconds")] - public int ReportingIntervalInSeconds { get; set; } - [Option(Required = false, HelpText = "Defines the reservoir type. Valid values are: Uniform, SlidingWindow and ExponentialDecay. The default value is SlidingWindow.")] public ReservoirTypes ReservoirType { get; set; } = ReservoirTypes.SlidingWindow; [Option(Required = false, HelpText = "The reservoir sample size.")] public int ReservoirSampleSize { get; set; } = 1028; + [Option(Required = false, HelpText = "Logging context name. The default value is \"CosmosDBBenchmarkLoggingContext\"")] + public string LoggingContextIdentifier { get; set; } = "CosmosDBBenchmarkLoggingContext"; + internal int GetTaskCount(int containerThroughput) { int taskCount = this.DegreeOfParallelism; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/BenchmarkOperation.cs new file mode 100644 index 0000000000..8cf2da3278 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/BenchmarkOperation.cs @@ -0,0 +1,15 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using System.Threading.Tasks; + + internal abstract class BenchmarkOperation : IBenchmarkOperation + { + public abstract Task ExecuteOnceAsync(); + + public abstract Task PrepareAsync(); + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs index 2670c7fa16..ed0bd2544e 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs @@ -12,9 +12,9 @@ namespace CosmosBenchmark internal interface IExecutionStrategy { public static IExecutionStrategy StartNew( - Func benchmarkOperation, IMetricsCollector metricsCollector) + Func benchmarkOperation) { - return new ParallelExecutionStrategy(benchmarkOperation, metricsCollector); + return new ParallelExecutionStrategy(benchmarkOperation); } public Task ExecuteAsync( diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs index 54f68cefa0..a6d4131272 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs @@ -21,6 +21,7 @@ public Task ExecuteAsync( bool traceFailures, Action completionCallback, ILogger logger, - IMetrics metrics); + IMetrics metrics, + BenchmarkConfig benchmarkConfig); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs index 88bbb2e880..eb56ef6e7c 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs @@ -2,14 +2,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. //------------------------------------------------------------ -using App.Metrics; - namespace CosmosBenchmark { + using App.Metrics.Timer; + public interface IMetricsCollector { - void CollectMetricsOnSuccess(MetricsContext metricsContext, IMetrics metrics); + TimerContext GetTimer(); + + void CollectMetricsOnSuccess(); - void CollectMetricsOnFailure(MetricsContext metricsContext, IMetrics metrics); + void CollectMetricsOnFailure(); } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkBase.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkBase.cs deleted file mode 100644 index afd230e8f4..0000000000 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkBase.cs +++ /dev/null @@ -1,11 +0,0 @@ -using App.Metrics; - -namespace CosmosBenchmark -{ - public class InsertBenchmarkBase : IMetricsCollector - { - public void CollectMetricsOnSuccess(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.WriteSuccessMeter); - - public void CollectMetricsOnFailure(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.WriteFailureMeter); - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs new file mode 100644 index 0000000000..4eab719a77 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs @@ -0,0 +1,13 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using App.Metrics; + using App.Metrics.Timer; + + internal abstract class InsertBenchmarkOperation : BenchmarkOperation + { + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs new file mode 100644 index 0000000000..ce2ce10b56 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs @@ -0,0 +1,31 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using App.Metrics.Timer; + using App.Metrics; + + internal class InsertOperationMetricsCollector : MetricsCollector + { + public InsertOperationMetricsCollector(MetricsContext metricsContext, IMetrics metrics) : base(metricsContext, metrics) + { + } + + public override TimerContext GetTimer() + { + return this.metrics.Measure.Timer.Time(this.metricsContext.InsertLatencyTimer); + } + + public override void CollectMetricsOnSuccess() + { + this.metrics.Measure.Counter.Increment(this.metricsContext.WriteSuccessMeter); + } + + public override void CollectMetricsOnFailure() + { + this.metrics.Measure.Counter.Increment(this.metricsContext.WriteFailureMeter); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs new file mode 100644 index 0000000000..e47dc977ec --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs @@ -0,0 +1,28 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using App.Metrics.Timer; + using App.Metrics; + + internal abstract class MetricsCollector : IMetricsCollector + { + protected MetricsContext metricsContext; + + protected IMetrics metrics; + + public MetricsCollector(MetricsContext metricsContext, IMetrics metrics) + { + this.metricsContext = metricsContext; + this.metrics = metrics; + } + + public abstract TimerContext GetTimer(); + + public abstract void CollectMetricsOnFailure(); + + public abstract void CollectMetricsOnSuccess(); + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs new file mode 100644 index 0000000000..5043988da6 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs @@ -0,0 +1,33 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using System; + using App.Metrics; + + internal static class MetricsCollectorProvider + { + public static IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation, MetricsContext metricsContext, IMetrics metrics) + { + Type benchmarkOperationType = benchmarkOperation.GetType(); + if (typeof(InsertBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) + { + return new InsertOperationMetricsCollector(metricsContext, metrics); + } + else if (typeof(QueryBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) + { + return new QueryOperationMetricsCollector(metricsContext, metrics); + } + else if (typeof(ReadBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) + { + return new ReadOperationMetricsCollector(metricsContext, metrics); + } + else + { + throw new NotSupportedException($"The type {nameof(benchmarkOperationType)} is not supported for collecting metrics."); + } + } + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs index db1653f93f..e76dc41c56 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs @@ -11,8 +11,6 @@ namespace CosmosBenchmark using System.Threading; using System.Threading.Tasks; using App.Metrics; - using Microsoft.ApplicationInsights; - using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -20,15 +18,12 @@ internal class ParallelExecutionStrategy : IExecutionStrategy { private readonly Func benchmarkOperation; - private readonly IMetricsCollector metricsCollector; - private volatile int pendingExecutorCount; public ParallelExecutionStrategy( - Func benchmarkOperation, IMetricsCollector metricsCollector) + Func benchmarkOperation) { this.benchmarkOperation = benchmarkOperation; - this.metricsCollector = metricsCollector; } public async Task ExecuteAsync( @@ -41,23 +36,22 @@ public async Task ExecuteAsync( { IExecutor warmupExecutor = new SerialOperationExecutor( executorId: "Warmup", - benchmarkOperation: this.benchmarkOperation(), - metricsCollector); + benchmarkOperation: this.benchmarkOperation()); await warmupExecutor.ExecuteAsync( (int)(serialExecutorIterationCount * warmupFraction), isWarmup: true, traceFailures: benchmarkConfig.TraceFailures, completionCallback: () => { }, logger, - metrics); + metrics, + benchmarkConfig); IExecutor[] executors = new IExecutor[serialExecutorConcurrency]; for (int i = 0; i < serialExecutorConcurrency; i++) { executors[i] = new SerialOperationExecutor( executorId: i.ToString(), - benchmarkOperation: this.benchmarkOperation(), - metricsCollector); + benchmarkOperation: this.benchmarkOperation()); } this.pendingExecutorCount = serialExecutorConcurrency; @@ -69,17 +63,20 @@ await warmupExecutor.ExecuteAsync( traceFailures: benchmarkConfig.TraceFailures, completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount), logger, - metrics); + metrics, + benchmarkConfig); } return await this.LogOutputStats( benchmarkConfig, - executors); + executors, + metrics); } private async Task LogOutputStats( BenchmarkConfig benchmarkConfig, - IExecutor[] executors) + IExecutor[] executors, + IMetrics metrics) { const int outputLoopDelayInSeconds = 1; IList perLoopCounters = new List(); @@ -117,16 +114,6 @@ private async Task LogOutputStats( perLoopCounters.Add((int)diff.Rps()); await Task.Delay(TimeSpan.FromSeconds(outputLoopDelayInSeconds)); - - Console.WriteLine("P50: " + TelemetrySpan.GetLatencyPercentile(50)); - Console.WriteLine("P75: " + TelemetrySpan.GetLatencyPercentile(75)); - Console.WriteLine("P90: " + TelemetrySpan.GetLatencyPercentile(90)); - Console.WriteLine("P95: " + TelemetrySpan.GetLatencyPercentile(95)); - Console.WriteLine("P98: " + TelemetrySpan.GetLatencyPercentile(98)); - Console.WriteLine("P99: " + TelemetrySpan.GetLatencyPercentile(99)); - Console.WriteLine("P999: " + TelemetrySpan.GetLatencyQuantile(0.999)); - Console.WriteLine("P9999: " + TelemetrySpan.GetLatencyQuantile(0.9999)); - Console.WriteLine("P100: " + TelemetrySpan.GetLatencyPercentile(100)); } while (!isLastIterationCompleted); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkBase.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkBase.cs deleted file mode 100644 index 0a19d0ad5a..0000000000 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkBase.cs +++ /dev/null @@ -1,11 +0,0 @@ -using App.Metrics; - -namespace CosmosBenchmark -{ - public class QueryBenchmarkBase : IMetricsCollector - { - public void CollectMetricsOnSuccess(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.QuerySuccessMeter); - - public void CollectMetricsOnFailure(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.QueryFailureMeter); - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkOperation.cs new file mode 100644 index 0000000000..1308a9b83e --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkOperation.cs @@ -0,0 +1,10 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + internal abstract class QueryBenchmarkOperation : BenchmarkOperation + { + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs new file mode 100644 index 0000000000..1511cc5b39 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs @@ -0,0 +1,31 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using App.Metrics; + using App.Metrics.Timer; + + internal class QueryOperationMetricsCollector : MetricsCollector + { + public QueryOperationMetricsCollector(MetricsContext metricsContext, IMetrics metrics) : base(metricsContext, metrics) + { + } + + public override TimerContext GetTimer() + { + return this.metrics.Measure.Timer.Time(this.metricsContext.QueryLatencyTimer); + } + + public override void CollectMetricsOnSuccess() + { + this.metrics.Measure.Counter.Increment(this.metricsContext.QuerySuccessMeter); + } + + public override void CollectMetricsOnFailure() + { + this.metrics.Measure.Counter.Increment(this.metricsContext.QueryFailureMeter); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkBase.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkBase.cs deleted file mode 100644 index 624ed6640f..0000000000 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkBase.cs +++ /dev/null @@ -1,11 +0,0 @@ -using App.Metrics; - -namespace CosmosBenchmark -{ - public class ReadBenchmarkBase : IMetricsCollector - { - public void CollectMetricsOnSuccess(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.ReadSuccessMeter); - - public void CollectMetricsOnFailure(MetricsContext metricsContext, IMetrics metrics) => metrics.Measure.Counter.Increment(metricsContext.ReadFailureMeter); - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkOperation.cs new file mode 100644 index 0000000000..55e42af491 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkOperation.cs @@ -0,0 +1,10 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + internal abstract class ReadBenchmarkOperation : BenchmarkOperation + { + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs new file mode 100644 index 0000000000..17b2c25111 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs @@ -0,0 +1,31 @@ +//------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +//------------------------------------------------------------ + +namespace CosmosBenchmark +{ + using App.Metrics; + using App.Metrics.Timer; + + internal class ReadOperationMetricsCollector : MetricsCollector + { + public ReadOperationMetricsCollector(MetricsContext metricsContext, IMetrics metrics) : base(metricsContext, metrics) + { + } + + public override TimerContext GetTimer() + { + return this.metrics.Measure.Timer.Time(this.metricsContext.ReadLatencyTimer); + } + + public override void CollectMetricsOnSuccess() + { + this.metrics.Measure.Counter.Increment(this.metricsContext.ReadSuccessMeter); + } + + public override void CollectMetricsOnFailure() + { + this.metrics.Measure.Counter.Increment(this.metricsContext.ReadFailureMeter); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs index b7659d9980..92fb081828 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs @@ -22,24 +22,19 @@ internal class SerialOperationExecutor : IExecutor private readonly string executorId; - // TODO: Move to config. - private const string LoggingContextIdentifier = "CosmosDBBenchmarkLoggingContext"; - public SerialOperationExecutor( string executorId, - IBenchmarkOperation benchmarkOperation, - IMetricsCollector metricsCollector) + IBenchmarkOperation benchmarkOperation) { this.executorId = executorId; this.operation = benchmarkOperation; this.SuccessOperationCount = 0; this.FailedOperationCount = 0; - - this.metricsCollector = metricsCollector; } public int SuccessOperationCount { get; private set; } + public int FailedOperationCount { get; private set; } public double TotalRuCharges { get; private set; } @@ -50,13 +45,15 @@ public async Task ExecuteAsync( bool traceFailures, Action completionCallback, ILogger logger, - IMetrics metrics) + IMetrics metrics, + BenchmarkConfig benchmarkConfig) { logger.LogInformation($"Executor {this.executorId} started"); logger.LogInformation("Initializing counters and metrics."); - var metricsContext = new MetricsContext(operation.GetType().Name, BenchmarkConfigProvider.CurrentBenchmarkConfig); + MetricsContext metricsContext = new MetricsContext(benchmarkConfig); + IMetricsCollector metricsCollector = MetricsCollectorProvider.GetMetricsCollector(this.operation, metricsContext, metrics); try { @@ -71,15 +68,14 @@ public async Task ExecuteAsync( () => operationResult.Value, disableTelemetry: isWarmup)) { - try { - using (TimerContext timerContext = metrics.Measure.Timer.Time(metricsContext.LatencyTimer)) + using (TimerContext timerContext = metricsCollector.GetTimer()) { - operationResult = await operation.ExecuteOnceAsync(); + operationResult = await this.operation.ExecuteOnceAsync(); } - metricsCollector.CollectMetricsOnSuccess(metricsContext, metrics); + metricsCollector.CollectMetricsOnSuccess(); // Success case this.SuccessOperationCount++; @@ -97,7 +93,7 @@ public async Task ExecuteAsync( Console.WriteLine(ex.ToString()); } - metricsCollector.CollectMetricsOnFailure(metricsContext, metrics); + metricsCollector.CollectMetricsOnFailure(); // failure case this.FailedOperationCount++; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/IBenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/IBenchmarkOperation.cs index d6933fcce5..9a3ea70894 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/IBenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/IBenchmarkOperation.cs @@ -8,8 +8,8 @@ namespace CosmosBenchmark internal interface IBenchmarkOperation { - Task PrepareAsync(); - Task ExecuteOnceAsync(); + + Task PrepareAsync(); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs index 2e787990c0..e948b99483 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs @@ -10,7 +10,9 @@ namespace CosmosBenchmark public class MetricsContext { - public string ContextId { get; } + public CounterOptions QuerySuccessMeter { get; } + + public CounterOptions QueryFailureMeter { get; } public CounterOptions ReadSuccessMeter { get; } @@ -20,30 +22,50 @@ public class MetricsContext public CounterOptions WriteFailureMeter { get; } - public CounterOptions QuerySuccessMeter { get; } + public TimerOptions QueryLatencyTimer { get; } - public CounterOptions QueryFailureMeter { get; } + public TimerOptions ReadLatencyTimer { get; } - public TimerOptions LatencyTimer { get; } + public TimerOptions InsertLatencyTimer { get; } - public MetricsContext(string contextId, BenchmarkConfig benchmarkConfig) + public MetricsContext(BenchmarkConfig benchmarkConfig) { - ContextId = contextId; + this.QuerySuccessMeter = new CounterOptions { Name = "#Query Successful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; + this.QueryFailureMeter = new CounterOptions { Name = "#Query Unsuccessful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; + + this.ReadSuccessMeter = new CounterOptions { Name = "#Read Successful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; + this.ReadFailureMeter = new CounterOptions { Name = "#Read Unsuccessful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; + + this.WriteSuccessMeter = new CounterOptions { Name = "#Insert Successful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; + this.WriteFailureMeter = new CounterOptions { Name = "#Insert Unsuccessful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; + + this.QueryLatencyTimer = new() + { + Name = "Query latency", + MeasurementUnit = Unit.Requests, + DurationUnit = TimeUnit.Milliseconds, + RateUnit = TimeUnit.Seconds, + Context = benchmarkConfig.LoggingContextIdentifier, + Reservoir = () => ReservoirProvider.GetReservoir(benchmarkConfig) + }; - ReadSuccessMeter = new CounterOptions { Name = "#Read Successful Operations", Context = ContextId }; - ReadFailureMeter = new CounterOptions { Name = "#Read Unsuccessful Operations", Context = ContextId }; - WriteSuccessMeter = new CounterOptions { Name = "#Write Successful Operations", Context = ContextId }; - WriteFailureMeter = new CounterOptions { Name = "#Write Unsuccessful Operations", Context = ContextId }; - QuerySuccessMeter = new CounterOptions { Name = "#Query Successful Operations", Context = ContextId }; - QueryFailureMeter = new CounterOptions { Name = "#Query Unsuccessful Operations", Context = ContextId }; + this.ReadLatencyTimer = new() + { + Name = "Read latency", + MeasurementUnit = Unit.Requests, + DurationUnit = TimeUnit.Milliseconds, + RateUnit = TimeUnit.Seconds, + Context = benchmarkConfig.LoggingContextIdentifier, + Reservoir = () => ReservoirProvider.GetReservoir(benchmarkConfig) + }; - LatencyTimer = new() + this.InsertLatencyTimer = new() { - Name = "Latency", + Name = "Insert latency", MeasurementUnit = Unit.Requests, DurationUnit = TimeUnit.Milliseconds, RateUnit = TimeUnit.Seconds, - Context = ContextId, + Context = benchmarkConfig.LoggingContextIdentifier, Reservoir = () => ReservoirProvider.GetReservoir(benchmarkConfig) }; } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs index 7aa96be351..70d24eda31 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs @@ -58,16 +58,11 @@ public static async Task Main(string[] args) IMetricsRoot metrics = ConfigureReporting(config, logger); AppMetricsTaskScheduler scheduler = new AppMetricsTaskScheduler( - TimeSpan.FromSeconds(3), + TimeSpan.FromSeconds(config.MetricsReportingIntervalInSec), async () => await Task.WhenAll(metrics.ReportRunner.RunAllAsync())); scheduler.Start(); - // TODO: Temporary solution. Remove after adding DI. - BenchmarkConfigProvider.CurrentBenchmarkConfig = config; - RunSummary runSummary = await program.ExecuteAsync(config, logger, metrics); - - await Task.WhenAll(metrics.ReportRunner.RunAllAsync()); } finally { @@ -84,25 +79,26 @@ private static IMetricsRoot ConfigureReporting( BenchmarkConfig config, ILogger logger) { - - if (config.WriteMetricsToConsole) + MetricsBuilder metricsBuilder = new MetricsBuilder(); + if (config.AppInsightsInstrumentationKey.Trim().Length > 0) { - return new MetricsBuilder() - .Report.ToConsole( - options => - { - options.FlushInterval = TimeSpan.FromSeconds(config.ReportingIntervalInSeconds); - options.MetricsOutputFormatter = new MetricsJsonOutputFormatter(); - }) - .Build(); + string connectionString = $"InstrumentationKey={config.AppInsightsInstrumentationKey}"; + MetricsOptions metricsOptions = new MetricsOptions(); + metricsBuilder + .Configuration.Configure(metricsOptions) + .Report.ToApplicationInsights(connectionString); + } + else + { + metricsBuilder.Report.ToConsole( + options => + { + options.FlushInterval = TimeSpan.FromSeconds(config.MetricsReportingIntervalInSec); + options.MetricsOutputFormatter = new MetricsJsonOutputFormatter(); + }); } - string connectionString = $"InstrumentationKey={config.AppInsightsInstrumentationKey}"; - MetricsOptions metricsOptions = new MetricsOptions(); - return new MetricsBuilder() - .Configuration.Configure(metricsOptions) - .Report.ToApplicationInsights(connectionString) - .Build(); + return metricsBuilder.Build(); } private static async Task AddAzureInfoToRunSummary() @@ -184,9 +180,7 @@ private async Task ExecuteAsync(BenchmarkConfig config, ILogger logg Program.ClearCoreSdkListeners(); } - IMetricsCollector metricsCollector = this.GetMetricsCollector(config); - - IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory, metricsCollector); + IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory); runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, logger, metrics); } @@ -294,38 +288,6 @@ private Func GetBenchmarkFactory( return () => (IBenchmarkOperation)ci.Invoke(ctorArguments); } - private IMetricsCollector GetMetricsCollector(BenchmarkConfig config) - { - Type metricsCollectorType = typeof(IMetricsCollector); - - var availableBenchmarks = typeof(Program).Assembly.GetTypes() - .Where(p => metricsCollectorType.IsAssignableFrom(p)) - .ToArray(); - - IEnumerable res = availableBenchmarks - .Where(e => e.Name.Equals(config.WorkloadType, StringComparison.OrdinalIgnoreCase) || e.Name.Equals(config.WorkloadType + "BenchmarkOperation", StringComparison.OrdinalIgnoreCase)); - - if (res.Count() != 1) - { - throw new NotImplementedException($"Unsupported workload type {config.WorkloadType}. Available ones are " + - string.Join(", \r\n", availableBenchmarks.Select(e => e.Name))); - } - - Type metricsCollectorTypeName = res.Single(); - - BindingFlags flags = BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance; - var ci = metricsCollectorTypeName - .BaseType - .GetConstructors(flags) - .Where(constructor => constructor.GetParameters().Length == 0) - .First(); - - var metricsCollector = (IMetricsCollector)ci.Invoke(null); - - - return metricsCollector; - } - private static Type[] AvailableBenchmarks() { Type benchmarkType = typeof(IBenchmarkOperation); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs index a910651216..e01ab060e8 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/InsertV2BenchmarkOperation.cs @@ -10,7 +10,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class InsertV2BenchmarkOperation : InsertBenchmarkBase, IBenchmarkOperation + internal class InsertV2BenchmarkOperation : InsertBenchmarkOperation { private readonly DocumentClient documentClient; private readonly Uri containerUri; @@ -38,7 +38,7 @@ public InsertV2BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { ResourceResponse itemResponse = await this.documentClient.CreateDocumentAsync( this.containerUri, @@ -58,7 +58,7 @@ public async Task ExecuteOnceAsync() }; } - public Task PrepareAsync() + public override Task PrepareAsync() { string newPartitionKey = Guid.NewGuid().ToString(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs index 659def21fe..908850dd39 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryStreamSinglePkV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents.Client; using Microsoft.Azure.Documents.Linq; - internal class QueryStreamSinglePkV2BenchmarkOperation : QueryBenchmarkBase, IBenchmarkOperation + internal class QueryStreamSinglePkV2BenchmarkOperation : QueryBenchmarkOperation { private readonly DocumentClient documentClient; private readonly string partitionKeyPath; @@ -47,7 +47,7 @@ public QueryStreamSinglePkV2BenchmarkOperation( this.containerUri = UriFactory.CreateDocumentCollectionUri(this.databsaeName, this.containerName); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { IDocumentQuery query = this.documentClient.CreateDocumentQuery( this.containerUri, @@ -85,7 +85,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (this.initialized) { diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs index 077b8fadf8..3cc32d295e 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/QueryTSinglePkV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents.Client; using Microsoft.Azure.Documents.Linq; - internal class QueryTSinglePkV2BenchmarkOperation : QueryBenchmarkBase, IBenchmarkOperation + internal class QueryTSinglePkV2BenchmarkOperation : QueryBenchmarkOperation { private readonly DocumentClient documentClient; private readonly string partitionKeyPath; @@ -46,7 +46,7 @@ public QueryTSinglePkV2BenchmarkOperation( this.sampleJObject[this.partitionKeyPath] = this.executionItemPartitionKey; } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { IDocumentQuery> query = this.documentClient.CreateDocumentQuery>( this.containerUri, @@ -93,7 +93,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (this.initialized) { diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs index b81fab7756..dd687e0f8b 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadFeedStreamV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class ReadFeedStreamV2BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation + internal class ReadFeedStreamV2BenchmarkOperation : ReadBenchmarkOperation { private readonly DocumentClient documentClient; private readonly string partitionKeyPath; @@ -40,7 +40,7 @@ public ReadFeedStreamV2BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { Uri containerUri = UriFactory.CreateDocumentCollectionUri(this.databsaeName, this.containerName); FeedResponse feedResponse = await this.documentClient.ReadDocumentFeedAsync( @@ -57,7 +57,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs index 707709d9dc..4c66893495 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadNotExistsV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class ReadNotExistsV2BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation + internal class ReadNotExistsV2BenchmarkOperation : ReadBenchmarkOperation { private readonly string databsaeName; private readonly string containerName; @@ -35,7 +35,7 @@ public ReadNotExistsV2BenchmarkOperation( this.containerName = containerName; } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { Uri itemUri = UriFactory.CreateDocumentUri(this.databsaeName, this.containerName, this.nextExecutionItemId); @@ -65,7 +65,7 @@ public async Task ExecuteOnceAsync() } } - public Task PrepareAsync() + public override Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs index fdbbce51b5..fda0c58c43 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadStreamExistsV2BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; - internal class ReadStreamExistsV2BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation + internal class ReadStreamExistsV2BenchmarkOperation : ReadBenchmarkOperation { private readonly string partitionKeyPath; private readonly Dictionary sampleJObject; @@ -41,7 +41,7 @@ public ReadStreamExistsV2BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { Uri itemUri = UriFactory.CreateDocumentUri(this.databsaeName, this.containerName, this.nextExecutionItemId); ResourceResponse itemResponse = await this.documentClient.ReadDocumentAsync( @@ -61,7 +61,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs index 938a3a083e..fa9cd81c98 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v2/ReadTExistsV2BenchmarkOperation.cs @@ -13,7 +13,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Documents.Client; using Newtonsoft.Json.Linq; - internal class ReadTExistsV2BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation + internal class ReadTExistsV2BenchmarkOperation : ReadBenchmarkOperation { private readonly string partitionKeyPath; private readonly Dictionary sampleJObject; @@ -42,7 +42,7 @@ public ReadTExistsV2BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { Uri itemUri = UriFactory.CreateDocumentUri(this.databsaeName, this.containerName, this.nextExecutionItemId); DocumentResponse> itemResponse = await this.documentClient.ReadDocumentAsync>( @@ -62,7 +62,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs index f549ed4d4a..81f9da9c87 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/InsertV3BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Newtonsoft.Json; using Newtonsoft.Json.Serialization; - internal class InsertV3BenchmarkOperation : InsertBenchmarkBase, IBenchmarkOperation + internal class InsertV3BenchmarkOperation : InsertBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; @@ -37,7 +37,7 @@ public InsertV3BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { using (MemoryStream input = JsonHelper.ToStream(this.sampleJObject)) { @@ -60,7 +60,7 @@ public async Task ExecuteOnceAsync() } } - public Task PrepareAsync() + public override Task PrepareAsync() { string newPartitionKey = Guid.NewGuid().ToString(); this.sampleJObject["id"] = Guid.NewGuid().ToString(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs index 7c6506bc70..5eb8fba252 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/QueryTV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal abstract class QueryTV3BenchmarkOperation : QueryBenchmarkBase, IBenchmarkOperation + internal abstract class QueryTV3BenchmarkOperation : QueryBenchmarkOperation { protected readonly Container container; protected readonly Dictionary sampleJObject; @@ -55,7 +55,7 @@ public QueryTV3BenchmarkOperation( /// /// /// - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { if (this.IsQueryStream) { @@ -258,7 +258,7 @@ private async Task ExecuteOnceAsyncWithStreamsAndPagination() /// /// /// - public virtual async Task PrepareAsync() + public override async Task PrepareAsync() { if (this.initialized) { diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs index 3d5559388d..3917470691 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadFeedStreamV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadFeedStreamV3BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation + internal class ReadFeedStreamV3BenchmarkOperation : ReadBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; @@ -39,7 +39,7 @@ public ReadFeedStreamV3BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { FeedIterator feedIterator = this.container .GetItemQueryStreamIterator( @@ -63,7 +63,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs index 2190e76ca3..fcec5a46fd 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadNotExistsV3BenchmarkOperation.cs @@ -9,7 +9,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadNotExistsV3BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation + internal class ReadNotExistsV3BenchmarkOperation : ReadBenchmarkOperation { private readonly Container container; @@ -32,7 +32,7 @@ public ReadNotExistsV3BenchmarkOperation( this.container = cosmosClient.GetContainer(this.databsaeName, this.containerName); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { using (ResponseMessage itemResponse = await this.container.ReadItemStreamAsync( this.nextExecutionItemId, @@ -54,7 +54,7 @@ public async Task ExecuteOnceAsync() } } - public Task PrepareAsync() + public override Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs index 3abd14201e..6695f721b6 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadStreamExistsV3BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation + internal class ReadStreamExistsV3BenchmarkOperation : ReadBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; @@ -39,7 +39,7 @@ public ReadStreamExistsV3BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { using (ResponseMessage itemResponse = await this.container.ReadItemStreamAsync( this.nextExecutionItemId, @@ -61,7 +61,7 @@ public async Task ExecuteOnceAsync() } } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs index de4ba3ecd7..580b9cd375 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadStreamExistsWithDiagnosticsV3BenchmarkOperation.cs @@ -11,7 +11,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.Azure.Cosmos; - internal class ReadStreamExistsWithDiagnosticsV3BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation + internal class ReadStreamExistsWithDiagnosticsV3BenchmarkOperation : ReadBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; @@ -39,7 +39,7 @@ public ReadStreamExistsWithDiagnosticsV3BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { using (ResponseMessage itemResponse = await this.container.ReadItemStreamAsync( this.nextExecutionItemId, @@ -67,7 +67,7 @@ public async Task ExecuteOnceAsync() } } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs index e853862ef9..392a38691e 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/v3/ReadTExistsV3BenchmarkOperation.cs @@ -12,7 +12,7 @@ namespace CosmosBenchmark using Microsoft.Azure.Cosmos; using Newtonsoft.Json.Linq; - internal class ReadTExistsV3BenchmarkOperation : ReadBenchmarkBase, IBenchmarkOperation + internal class ReadTExistsV3BenchmarkOperation : ReadBenchmarkOperation { private readonly Container container; private readonly string partitionKeyPath; @@ -40,7 +40,7 @@ public ReadTExistsV3BenchmarkOperation( this.sampleJObject = JsonHelper.Deserialize>(sampleJson); } - public async Task ExecuteOnceAsync() + public override async Task ExecuteOnceAsync() { ItemResponse> itemResponse = await this.container.ReadItemAsync>( this.nextExecutionItemId, @@ -60,7 +60,7 @@ public async Task ExecuteOnceAsync() }; } - public async Task PrepareAsync() + public override async Task PrepareAsync() { if (string.IsNullOrEmpty(this.nextExecutionItemId) || string.IsNullOrEmpty(this.nextExecutionItemPartitionKey)) From f644c20ee7c42688a4ed21861db27f010795171b Mon Sep 17 00:00:00 2001 From: Mikhail Lipin Date: Wed, 14 Jun 2023 14:27:33 +0300 Subject: [PATCH 05/10] Tracking percentiles with AppInsights TelemetryClient. --- .../Tools/Benchmark/BenchmarkConfig.cs | 4 - .../Benchmark/BenchmarkConfigProvider.cs | 16 -- .../Tools/Benchmark/CosmosBenchmark.csproj | 4 - .../Tools/Benchmark/Fx/IExecutionStrategy.cs | 4 +- .../Tools/Benchmark/Fx/IExecutor.cs | 6 +- .../Tools/Benchmark/Fx/IMetricsCollector.cs | 4 +- .../Benchmark/Fx/InsertBenchmarkOperation.cs | 3 - .../Fx/InsertOperationMetricsCollector.cs | 22 +-- .../Tools/Benchmark/Fx/MetricsCollector.cs | 156 ++++++++++++++++-- .../Benchmark/Fx/MetricsCollectorProvider.cs | 38 ++++- .../Benchmark/Fx/ParallelExecutionStrategy.cs | 18 +- .../Fx/QueryOperationMetricsCollector.cs | 22 +-- .../Fx/ReadOperationMetricsCollector.cs | 22 +-- .../Benchmark/Fx/SerialOperationExecutor.cs | 24 +-- .../Tools/Benchmark/Fx/TelemetrySpan.cs | 7 +- .../Tools/Benchmark/MetricCollectionWindow.cs | 63 +++++++ .../Tools/Benchmark/MetricsContext.cs | 73 -------- .../Tools/Benchmark/Program.cs | 55 ++---- .../Tools/Benchmark/ReservoirProvider.cs | 49 ------ 19 files changed, 303 insertions(+), 287 deletions(-) delete mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfigProvider.cs create mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs delete mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs delete mode 100644 Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ReservoirProvider.cs diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs index c2cdb056f1..7f22090503 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs @@ -11,7 +11,6 @@ namespace CosmosBenchmark using CommandLine; using Microsoft.Azure.Documents.Client; using Newtonsoft.Json; - using static CosmosBenchmark.ReservoirProvider; public class BenchmarkConfig { @@ -130,9 +129,6 @@ public class BenchmarkConfig [Option(Required = false, HelpText = "Application Insights instrumentation key")] public string AppInsightsInstrumentationKey { get; set; } - [Option(Required = false, HelpText = "Defines the reservoir type. Valid values are: Uniform, SlidingWindow and ExponentialDecay. The default value is SlidingWindow.")] - public ReservoirTypes ReservoirType { get; set; } = ReservoirTypes.SlidingWindow; - [Option(Required = false, HelpText = "The reservoir sample size.")] public int ReservoirSampleSize { get; set; } = 1028; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfigProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfigProvider.cs deleted file mode 100644 index ac54a8abdc..0000000000 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfigProvider.cs +++ /dev/null @@ -1,16 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace CosmosBenchmark -{ - using App.Metrics; - using App.Metrics.Counter; - using App.Metrics.Timer; - - // TODO: Temporary solution. Remove after adding DI. - public class BenchmarkConfigProvider - { - public static BenchmarkConfig CurrentBenchmarkConfig { get; set; } - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj index 0fad607d5a..847453fa7f 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj @@ -17,10 +17,6 @@ - - - - diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs index ed0bd2544e..d49674ecca 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs @@ -6,7 +6,7 @@ namespace CosmosBenchmark { using System; using System.Threading.Tasks; - using App.Metrics; + using Microsoft.ApplicationInsights; using Microsoft.Extensions.Logging; internal interface IExecutionStrategy @@ -23,6 +23,6 @@ public Task ExecuteAsync( int serialExecutorIterationCount, double warmupFraction, ILogger logger, - IMetrics metrics); + TelemetryClient telemetryClient); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs index a6d4131272..efb84782b9 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs @@ -6,7 +6,7 @@ namespace CosmosBenchmark { using System; using System.Threading.Tasks; - using App.Metrics; + using Microsoft.ApplicationInsights; using Microsoft.Extensions.Logging; internal interface IExecutor @@ -20,8 +20,8 @@ public Task ExecuteAsync( bool isWarmup, bool traceFailures, Action completionCallback, + BenchmarkConfig benchmarkConfig, ILogger logger, - IMetrics metrics, - BenchmarkConfig benchmarkConfig); + TelemetryClient telemetryClient); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs index eb56ef6e7c..2fec49f210 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs @@ -4,11 +4,9 @@ namespace CosmosBenchmark { - using App.Metrics.Timer; - public interface IMetricsCollector { - TimerContext GetTimer(); + void RecordLatency(double milliseconds); void CollectMetricsOnSuccess(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs index 4eab719a77..117fe5ae09 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs @@ -4,9 +4,6 @@ namespace CosmosBenchmark { - using App.Metrics; - using App.Metrics.Timer; - internal abstract class InsertBenchmarkOperation : BenchmarkOperation { } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs index ce2ce10b56..dbb4cee2cd 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs @@ -4,28 +4,20 @@ namespace CosmosBenchmark { - using App.Metrics.Timer; - using App.Metrics; + using Microsoft.ApplicationInsights; internal class InsertOperationMetricsCollector : MetricsCollector { - public InsertOperationMetricsCollector(MetricsContext metricsContext, IMetrics metrics) : base(metricsContext, metrics) + public InsertOperationMetricsCollector(TelemetryClient telemetryClient) : base(telemetryClient) { } - public override TimerContext GetTimer() - { - return this.metrics.Measure.Timer.Time(this.metricsContext.InsertLatencyTimer); - } + protected override string AverageRpsMetricName => "InsertOperationAverageRps"; - public override void CollectMetricsOnSuccess() - { - this.metrics.Measure.Counter.Increment(this.metricsContext.WriteSuccessMeter); - } + protected override string LatencyInMsMetricName => "InsertOperationLatencyInMs"; - public override void CollectMetricsOnFailure() - { - this.metrics.Measure.Counter.Increment(this.metricsContext.WriteFailureMeter); - } + protected override string FailureOperationMetricName => "InsertOperationFailure"; + + protected override string SuccessOperationMetricName => "InsertOperationSuccess"; } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs index e47dc977ec..7de637600b 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs @@ -4,25 +4,161 @@ namespace CosmosBenchmark { - using App.Metrics.Timer; - using App.Metrics; + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.ApplicationInsights; + using Microsoft.ApplicationInsights.DataContracts; internal abstract class MetricsCollector : IMetricsCollector { - protected MetricsContext metricsContext; + private readonly List operationLatenciesInMs = new List(); - protected IMetrics metrics; + protected readonly TelemetryClient telemetryClient; - public MetricsCollector(MetricsContext metricsContext, IMetrics metrics) + protected int successCounter; + + protected int failureCounter; + + public MetricsCollector(TelemetryClient telemetryClient) + { + this.telemetryClient = telemetryClient; + this.successCounter = 0; + this.failureCounter = 0; + } + + public void CollectMetricsOnSuccess() + { + MetricTelemetry metricTelemetry = new MetricTelemetry(this.SuccessOperationMetricName, ++this.successCounter); + this.telemetryClient.TrackMetric(metricTelemetry); + } + + public void CollectMetricsOnFailure() + { + MetricTelemetry metricTelemetry = new MetricTelemetry(this.FailureOperationMetricName, ++this.failureCounter); + this.telemetryClient.TrackMetric(metricTelemetry); + } + + public void RecordLatency(double milliseconds) { - this.metricsContext = metricsContext; - this.metrics = metrics; + this.operationLatenciesInMs.Add(milliseconds); + this.TelemetryPercentiles(); } - public abstract TimerContext GetTimer(); + protected abstract string AverageRpsMetricName { get; } + + protected abstract string LatencyInMsMetricName { get; } + + protected abstract string FailureOperationMetricName { get; } + + protected abstract string SuccessOperationMetricName { get; } + + internal double GetLatencyPercentile(int percentile) + { + return MathNet.Numerics.Statistics.Statistics.Percentile(this.operationLatenciesInMs, percentile); + } + + internal double GetLatencyQuantile(double quantile) + { + return MathNet.Numerics.Statistics.Statistics.Quantile(this.operationLatenciesInMs, quantile); + } + + private void TelemetryPercentiles() + { + if (this.operationLatenciesInMs.Count > 10) + { + double top10PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.1 * this.operationLatenciesInMs.Count)).Average(), 0); + MetricTelemetry metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top10Percent", top10PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); - public abstract void CollectMetricsOnFailure(); + double top20PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.2 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top20Percent", top20PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); - public abstract void CollectMetricsOnSuccess(); + double top30PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.3 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top30Percent", top30PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top40PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.4 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top40Percent", top40PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top50PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.5 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top50Percent", top50PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top60PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.6 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top60Percent", top60PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top70PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.7 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top70Percent", top70PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top80PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.8 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top80Percent", top80PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top90PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.9 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top90Percent", top90PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top95PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.95 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top95Percent", top95PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top99PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.99 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top99Percent", top99PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top999PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.999 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top99.9Percent", top999PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top9999PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.9999 * this.operationLatenciesInMs.Count)).Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top99.99Percent", top9999PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double averageRps = Math.Round(this.operationLatenciesInMs.Average(), 0); + metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName, top9999PercentAverageRps); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top50PercentLatencyInMs = this.GetLatencyPercentile(50); + metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P50", top50PercentLatencyInMs); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top75PercentLatencyInMs = this.GetLatencyPercentile(75); + metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P75", top75PercentLatencyInMs); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top90PercentLatencyInMs = this.GetLatencyPercentile(90); + metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P90", top90PercentLatencyInMs); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top95PercentLatencyInMs = this.GetLatencyPercentile(95); + metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P95", top95PercentLatencyInMs); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top98PercentLatencyInMs = this.GetLatencyPercentile(98); + metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P98", top95PercentLatencyInMs); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top99PercentLatencyInMs = this.GetLatencyPercentile(99); + metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P99", top99PercentLatencyInMs); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top999PercentLatencyInMs = this.GetLatencyQuantile(0.999); + metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-Q999", top999PercentLatencyInMs); + this.telemetryClient.TrackMetric(metricTelemetry); + + double top9999PercentLatencyInMs = this.GetLatencyQuantile(0.9999); + metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-Q9999", top9999PercentLatencyInMs); + this.telemetryClient.TrackMetric(metricTelemetry); + + double maxLatencyInMs = this.GetLatencyPercentile(100); + metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P100", maxLatencyInMs); + this.telemetryClient.TrackMetric(metricTelemetry); + } + } } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs index 5043988da6..0f78d0348c 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs @@ -5,24 +5,52 @@ namespace CosmosBenchmark { using System; - using App.Metrics; + using Microsoft.ApplicationInsights; internal static class MetricsCollectorProvider { - public static IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation, MetricsContext metricsContext, IMetrics metrics) + private static MetricCollectionWindow metricCollectionWindow; + + private static readonly object metricCollectionWindowLock = new object(); + + private static MetricCollectionWindow CurrentMetricCollectionWindow + { + get + { + if (CheckMetricCollectionInvalid(metricCollectionWindow)) + { + lock (metricCollectionWindowLock) + { + if (CheckMetricCollectionInvalid(metricCollectionWindow)) + { + metricCollectionWindow = new MetricCollectionWindow(); + } + } + } + + return metricCollectionWindow; + } + } + + private static bool CheckMetricCollectionInvalid(MetricCollectionWindow metricCollectionWindow) + { + return metricCollectionWindow is null || DateTime.Now > metricCollectionWindow.DateTimeCreated.AddSeconds(5); + } + + public static IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation, TelemetryClient telemetryClient) { Type benchmarkOperationType = benchmarkOperation.GetType(); if (typeof(InsertBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) { - return new InsertOperationMetricsCollector(metricsContext, metrics); + return CurrentMetricCollectionWindow.GetInsertOperationMetricsCollector(telemetryClient); } else if (typeof(QueryBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) { - return new QueryOperationMetricsCollector(metricsContext, metrics); + return CurrentMetricCollectionWindow.GetQueryOperationMetricsCollector(telemetryClient); } else if (typeof(ReadBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) { - return new ReadOperationMetricsCollector(metricsContext, metrics); + return CurrentMetricCollectionWindow.GetReadOperationMetricsCollector(telemetryClient); } else { diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs index e76dc41c56..86b791ad24 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs @@ -10,7 +10,7 @@ namespace CosmosBenchmark using System.Linq; using System.Threading; using System.Threading.Tasks; - using App.Metrics; + using Microsoft.ApplicationInsights; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -32,7 +32,7 @@ public async Task ExecuteAsync( int serialExecutorIterationCount, double warmupFraction, ILogger logger, - IMetrics metrics) + TelemetryClient telemetryClient) { IExecutor warmupExecutor = new SerialOperationExecutor( executorId: "Warmup", @@ -42,9 +42,9 @@ await warmupExecutor.ExecuteAsync( isWarmup: true, traceFailures: benchmarkConfig.TraceFailures, completionCallback: () => { }, + benchmarkConfig, logger, - metrics, - benchmarkConfig); + telemetryClient); IExecutor[] executors = new IExecutor[serialExecutorConcurrency]; for (int i = 0; i < serialExecutorConcurrency; i++) @@ -62,21 +62,23 @@ await warmupExecutor.ExecuteAsync( isWarmup: false, traceFailures: benchmarkConfig.TraceFailures, completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount), + benchmarkConfig, logger, - metrics, - benchmarkConfig); + telemetryClient); } return await this.LogOutputStats( benchmarkConfig, executors, - metrics); + logger, + telemetryClient); } private async Task LogOutputStats( BenchmarkConfig benchmarkConfig, IExecutor[] executors, - IMetrics metrics) + ILogger logger, + TelemetryClient telemetryClient) { const int outputLoopDelayInSeconds = 1; IList perLoopCounters = new List(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs index 1511cc5b39..7512b3c77a 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs @@ -4,28 +4,20 @@ namespace CosmosBenchmark { - using App.Metrics; - using App.Metrics.Timer; + using Microsoft.ApplicationInsights; internal class QueryOperationMetricsCollector : MetricsCollector { - public QueryOperationMetricsCollector(MetricsContext metricsContext, IMetrics metrics) : base(metricsContext, metrics) + public QueryOperationMetricsCollector(TelemetryClient telemetryClient) : base(telemetryClient) { } - public override TimerContext GetTimer() - { - return this.metrics.Measure.Timer.Time(this.metricsContext.QueryLatencyTimer); - } + protected override string AverageRpsMetricName => "QueryOperationAverageRps"; - public override void CollectMetricsOnSuccess() - { - this.metrics.Measure.Counter.Increment(this.metricsContext.QuerySuccessMeter); - } + protected override string LatencyInMsMetricName => "QueryOperationLatencyInMs"; - public override void CollectMetricsOnFailure() - { - this.metrics.Measure.Counter.Increment(this.metricsContext.QueryFailureMeter); - } + protected override string FailureOperationMetricName => "QueryOperationFailure"; + + protected override string SuccessOperationMetricName => "QueryOperationSuccess"; } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs index 17b2c25111..3109b88d50 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs @@ -4,28 +4,20 @@ namespace CosmosBenchmark { - using App.Metrics; - using App.Metrics.Timer; + using Microsoft.ApplicationInsights; internal class ReadOperationMetricsCollector : MetricsCollector { - public ReadOperationMetricsCollector(MetricsContext metricsContext, IMetrics metrics) : base(metricsContext, metrics) + public ReadOperationMetricsCollector(TelemetryClient telemetryClient) : base(telemetryClient) { } - public override TimerContext GetTimer() - { - return this.metrics.Measure.Timer.Time(this.metricsContext.ReadLatencyTimer); - } + protected override string AverageRpsMetricName => "ReadOperationAverageRps"; - public override void CollectMetricsOnSuccess() - { - this.metrics.Measure.Counter.Increment(this.metricsContext.ReadSuccessMeter); - } + protected override string LatencyInMsMetricName => "ReadOperationLatencyInMs"; - public override void CollectMetricsOnFailure() - { - this.metrics.Measure.Counter.Increment(this.metricsContext.ReadFailureMeter); - } + protected override string FailureOperationMetricName => "ReadOperationFailure"; + + protected override string SuccessOperationMetricName => "ReadOperationFailure"; } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs index 92fb081828..2bca6291e6 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs @@ -7,10 +7,7 @@ namespace CosmosBenchmark using System; using System.Diagnostics; using System.Threading.Tasks; - using App.Metrics; - using App.Metrics.Counter; - using App.Metrics.Logging; - using App.Metrics.Timer; + using Microsoft.ApplicationInsights; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; @@ -18,8 +15,6 @@ internal class SerialOperationExecutor : IExecutor { private readonly IBenchmarkOperation operation; - private readonly IMetricsCollector metricsCollector; - private readonly string executorId; public SerialOperationExecutor( @@ -44,36 +39,33 @@ public async Task ExecuteAsync( bool isWarmup, bool traceFailures, Action completionCallback, + BenchmarkConfig benchmarkConfig, ILogger logger, - IMetrics metrics, - BenchmarkConfig benchmarkConfig) + TelemetryClient telemetryClient) { logger.LogInformation($"Executor {this.executorId} started"); logger.LogInformation("Initializing counters and metrics."); - MetricsContext metricsContext = new MetricsContext(benchmarkConfig); - IMetricsCollector metricsCollector = MetricsCollectorProvider.GetMetricsCollector(this.operation, metricsContext, metrics); - try { int currentIterationCount = 0; do { + IMetricsCollector metricsCollector = MetricsCollectorProvider.GetMetricsCollector(this.operation, telemetryClient); + OperationResult? operationResult = null; await this.operation.PrepareAsync(); using (IDisposable telemetrySpan = TelemetrySpan.StartNew( () => operationResult.Value, - disableTelemetry: isWarmup)) + disableTelemetry: isWarmup, + metricsCollector.RecordLatency)) { try { - using (TimerContext timerContext = metricsCollector.GetTimer()) - { - operationResult = await this.operation.ExecuteOnceAsync(); - } + operationResult = await this.operation.ExecuteOnceAsync(); metricsCollector.CollectMetricsOnSuccess(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs index c0a44a737b..564f3253e1 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/TelemetrySpan.cs @@ -18,11 +18,13 @@ internal struct TelemetrySpan : IDisposable private Stopwatch stopwatch; private Func lazyOperationResult; + private Action recordLatencyAction; private bool disableTelemetry; public static IDisposable StartNew( Func lazyOperationResult, - bool disableTelemetry) + bool disableTelemetry, + Action recordLatencyAction) { if (disableTelemetry || !TelemetrySpan.IncludePercentile) { @@ -33,6 +35,7 @@ public static IDisposable StartNew( { stopwatch = Stopwatch.StartNew(), lazyOperationResult = lazyOperationResult, + recordLatencyAction = recordLatencyAction, disableTelemetry = disableTelemetry }; } @@ -47,6 +50,8 @@ public void Dispose() if (TelemetrySpan.IncludePercentile) { RecordLatency(this.stopwatch.Elapsed.TotalMilliseconds); + + this.recordLatencyAction?.Invoke(this.stopwatch.Elapsed.TotalMilliseconds); } BenchmarkLatencyEventSource.Instance.LatencyDiagnostics( diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs new file mode 100644 index 0000000000..4bdf46efba --- /dev/null +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs @@ -0,0 +1,63 @@ +namespace CosmosBenchmark +{ + using System; + using Microsoft.ApplicationInsights; + + internal class MetricCollectionWindow + { + private InsertOperationMetricsCollector insertOperationMetricsCollector; + private readonly object insertOperationMetricsCollectorLock = new object(); + + private QueryOperationMetricsCollector queryOperationMetricsCollector; + private readonly object queryOperationMetricsCollectorLock = new object(); + + private ReadOperationMetricsCollector readOperationMetricsCollector; + private readonly object readOperationMetricsCollectorLock = new object(); + + public DateTime DateTimeCreated { get; init; } + + public MetricCollectionWindow() + { + this.DateTimeCreated = DateTime.Now; + } + + public InsertOperationMetricsCollector GetInsertOperationMetricsCollector(TelemetryClient telemetryClient) + { + if (this.insertOperationMetricsCollector is null) + { + lock (this.insertOperationMetricsCollectorLock) + { + this.insertOperationMetricsCollector ??= new InsertOperationMetricsCollector(telemetryClient); + } + } + + return this.insertOperationMetricsCollector; + } + + public QueryOperationMetricsCollector GetQueryOperationMetricsCollector(TelemetryClient telemetryClient) + { + if (this.queryOperationMetricsCollector is null) + { + lock (this.queryOperationMetricsCollectorLock) + { + this.queryOperationMetricsCollector ??= new QueryOperationMetricsCollector(telemetryClient); + } + } + + return this.queryOperationMetricsCollector; + } + + public ReadOperationMetricsCollector GetReadOperationMetricsCollector(TelemetryClient telemetryClient) + { + if (this.readOperationMetricsCollector is null) + { + lock (this.readOperationMetricsCollectorLock) + { + this.readOperationMetricsCollector ??= new ReadOperationMetricsCollector(telemetryClient); + } + } + + return this.readOperationMetricsCollector; + } + } +} diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs deleted file mode 100644 index e948b99483..0000000000 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricsContext.cs +++ /dev/null @@ -1,73 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace CosmosBenchmark -{ - using App.Metrics; - using App.Metrics.Counter; - using App.Metrics.Timer; - - public class MetricsContext - { - public CounterOptions QuerySuccessMeter { get; } - - public CounterOptions QueryFailureMeter { get; } - - public CounterOptions ReadSuccessMeter { get; } - - public CounterOptions ReadFailureMeter { get; } - - public CounterOptions WriteSuccessMeter { get; } - - public CounterOptions WriteFailureMeter { get; } - - public TimerOptions QueryLatencyTimer { get; } - - public TimerOptions ReadLatencyTimer { get; } - - public TimerOptions InsertLatencyTimer { get; } - - public MetricsContext(BenchmarkConfig benchmarkConfig) - { - this.QuerySuccessMeter = new CounterOptions { Name = "#Query Successful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; - this.QueryFailureMeter = new CounterOptions { Name = "#Query Unsuccessful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; - - this.ReadSuccessMeter = new CounterOptions { Name = "#Read Successful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; - this.ReadFailureMeter = new CounterOptions { Name = "#Read Unsuccessful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; - - this.WriteSuccessMeter = new CounterOptions { Name = "#Insert Successful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; - this.WriteFailureMeter = new CounterOptions { Name = "#Insert Unsuccessful Operations", Context = benchmarkConfig.LoggingContextIdentifier }; - - this.QueryLatencyTimer = new() - { - Name = "Query latency", - MeasurementUnit = Unit.Requests, - DurationUnit = TimeUnit.Milliseconds, - RateUnit = TimeUnit.Seconds, - Context = benchmarkConfig.LoggingContextIdentifier, - Reservoir = () => ReservoirProvider.GetReservoir(benchmarkConfig) - }; - - this.ReadLatencyTimer = new() - { - Name = "Read latency", - MeasurementUnit = Unit.Requests, - DurationUnit = TimeUnit.Milliseconds, - RateUnit = TimeUnit.Seconds, - Context = benchmarkConfig.LoggingContextIdentifier, - Reservoir = () => ReservoirProvider.GetReservoir(benchmarkConfig) - }; - - this.InsertLatencyTimer = new() - { - Name = "Insert latency", - MeasurementUnit = Unit.Requests, - DurationUnit = TimeUnit.Milliseconds, - RateUnit = TimeUnit.Seconds, - Context = benchmarkConfig.LoggingContextIdentifier, - Reservoir = () => ReservoirProvider.GetReservoir(benchmarkConfig) - }; - } - } -} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs index 70d24eda31..c324f81bfe 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs @@ -12,15 +12,13 @@ namespace CosmosBenchmark using System.Net; using System.Net.Http; using System.Reflection; - using System.Text; using System.Threading; using System.Threading.Tasks; - using App.Metrics; - using App.Metrics.Formatters.Json; - using App.Metrics.Scheduling; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; + using Microsoft.ApplicationInsights; using Newtonsoft.Json.Linq; + using Microsoft.ApplicationInsights.Extensibility; /// /// This sample demonstrates how to achieve high performance writes using Azure Comsos DB. @@ -33,11 +31,6 @@ public sealed class Program /// command line arguments. public static async Task Main(string[] args) { - using ILoggerFactory loggerFactory = LoggerFactory.Create(builder => builder - .AddConsole()); - - ILogger logger = loggerFactory.CreateLogger(); - try { BenchmarkConfig config = BenchmarkConfig.From(args); @@ -53,16 +46,15 @@ public static async Task Main(string[] args) config.Print(); - Program program = new Program(); + using ILoggerFactory loggerFactory = LoggerFactory.Create(builder => builder.AddConsole()); + ILogger logger = loggerFactory.CreateLogger(); - IMetricsRoot metrics = ConfigureReporting(config, logger); + TelemetryConfiguration telemetryConfiguration = new(config.AppInsightsInstrumentationKey); + TelemetryClient telemetryClient = new TelemetryClient(telemetryConfiguration); - AppMetricsTaskScheduler scheduler = new AppMetricsTaskScheduler( - TimeSpan.FromSeconds(config.MetricsReportingIntervalInSec), - async () => await Task.WhenAll(metrics.ReportRunner.RunAllAsync())); - scheduler.Start(); + Program program = new Program(); - RunSummary runSummary = await program.ExecuteAsync(config, logger, metrics); + RunSummary runSummary = await program.ExecuteAsync(config, logger, telemetryClient); } finally { @@ -75,32 +67,6 @@ public static async Task Main(string[] args) } } - private static IMetricsRoot ConfigureReporting( - BenchmarkConfig config, - ILogger logger) - { - MetricsBuilder metricsBuilder = new MetricsBuilder(); - if (config.AppInsightsInstrumentationKey.Trim().Length > 0) - { - string connectionString = $"InstrumentationKey={config.AppInsightsInstrumentationKey}"; - MetricsOptions metricsOptions = new MetricsOptions(); - metricsBuilder - .Configuration.Configure(metricsOptions) - .Report.ToApplicationInsights(connectionString); - } - else - { - metricsBuilder.Report.ToConsole( - options => - { - options.FlushInterval = TimeSpan.FromSeconds(config.MetricsReportingIntervalInSec); - options.MetricsOutputFormatter = new MetricsJsonOutputFormatter(); - }); - } - - return metricsBuilder.Build(); - } - private static async Task AddAzureInfoToRunSummary() { using HttpClient httpClient = new HttpClient(); @@ -128,7 +94,7 @@ private static async Task AddAzureInfoToRunSummary() /// Executing benchmarks for V2/V3 cosmosdb SDK. /// /// a Task object. - private async Task ExecuteAsync(BenchmarkConfig config, ILogger logger, IMetrics metrics) + private async Task ExecuteAsync(BenchmarkConfig config, ILogger logger, TelemetryClient telemetryClient) { // V3 SDK client initialization using (CosmosClient cosmosClient = config.CreateCosmosClient(config.Key)) @@ -173,7 +139,6 @@ private async Task ExecuteAsync(BenchmarkConfig config, ILogger logg cosmosClient, documentClient); - if (config.DisableCoreSdkLogging) { // Do it after client initialization (HACK) @@ -181,7 +146,7 @@ private async Task ExecuteAsync(BenchmarkConfig config, ILogger logg } IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory); - runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, logger, metrics); + runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, logger, telemetryClient); } if (config.CleanupOnFinish) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ReservoirProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ReservoirProvider.cs deleted file mode 100644 index 5a3e2ab452..0000000000 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ReservoirProvider.cs +++ /dev/null @@ -1,49 +0,0 @@ -//------------------------------------------------------------ -// Copyright (c) Microsoft Corporation. All rights reserved. -//------------------------------------------------------------ - -namespace CosmosBenchmark -{ - using System; - using App.Metrics.ReservoirSampling; - - /// - /// Returns the based on the CTL configuration. - /// - public class ReservoirProvider - { - /// - /// Create and returns a new instance of the based on the Benchmark configuration. - /// - /// An instance of containing the Benchmark config params. - /// An implementation of . - public static IReservoir GetReservoir(BenchmarkConfig benchmarkConfig) - { - return benchmarkConfig.ReservoirType switch - { - ReservoirTypes.Uniform => new App.Metrics.ReservoirSampling.Uniform.DefaultAlgorithmRReservoir( - sampleSize: benchmarkConfig.ReservoirSampleSize), - - ReservoirTypes.SlidingWindow => new App.Metrics.ReservoirSampling.SlidingWindow.DefaultSlidingWindowReservoir( - sampleSize: benchmarkConfig.ReservoirSampleSize), - - ReservoirTypes.ExponentialDecay => new App.Metrics.ReservoirSampling.ExponentialDecay.DefaultForwardDecayingReservoir( - sampleSize: benchmarkConfig.ReservoirSampleSize, - alpha: 0.015), - - _ => throw new ArgumentException( - message: "Invalid ReservoirType Specified."), - }; - } - - /// - /// An enum containing different reservoir types. - /// - public enum ReservoirTypes - { - Uniform, - SlidingWindow, - ExponentialDecay - } - } -} From 92adc384855933a56c09525385331d6b3e3d5194 Mon Sep 17 00:00:00 2001 From: Mikhail Lipin Date: Sat, 17 Jun 2023 01:43:56 +0300 Subject: [PATCH 06/10] Moving back to OpenTelemetry. --- .../Tools/Benchmark/CosmosBenchmark.csproj | 2 + .../Tools/Benchmark/Fx/IExecutionStrategy.cs | 3 +- .../Tools/Benchmark/Fx/IExecutor.cs | 4 +- .../Tools/Benchmark/Fx/IMetricsCollector.cs | 2 +- .../Fx/InsertOperationMetricsCollector.cs | 5 +- .../Tools/Benchmark/Fx/MetricsCollector.cs | 148 +++--------------- .../Benchmark/Fx/MetricsCollectorProvider.cs | 34 ++-- .../Benchmark/Fx/ParallelExecutionStrategy.cs | 11 +- .../Fx/QueryOperationMetricsCollector.cs | 6 +- .../Fx/ReadOperationMetricsCollector.cs | 8 +- .../Benchmark/Fx/SerialOperationExecutor.cs | 8 +- .../Tools/Benchmark/MetricCollectionWindow.cs | 14 +- .../Tools/Benchmark/Program.cs | 31 +++- 13 files changed, 96 insertions(+), 180 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj index 847453fa7f..94b99d2501 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj @@ -17,6 +17,7 @@ + @@ -24,6 +25,7 @@ + diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs index d49674ecca..fddbd0808f 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs @@ -8,6 +8,7 @@ namespace CosmosBenchmark using System.Threading.Tasks; using Microsoft.ApplicationInsights; using Microsoft.Extensions.Logging; + using OpenTelemetry.Metrics; internal interface IExecutionStrategy { @@ -23,6 +24,6 @@ public Task ExecuteAsync( int serialExecutorIterationCount, double warmupFraction, ILogger logger, - TelemetryClient telemetryClient); + MeterProvider meterProvider); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs index efb84782b9..a820306e3d 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs @@ -6,8 +6,8 @@ namespace CosmosBenchmark { using System; using System.Threading.Tasks; - using Microsoft.ApplicationInsights; using Microsoft.Extensions.Logging; + using OpenTelemetry.Metrics; internal interface IExecutor { @@ -22,6 +22,6 @@ public Task ExecuteAsync( Action completionCallback, BenchmarkConfig benchmarkConfig, ILogger logger, - TelemetryClient telemetryClient); + MeterProvider meterProvider); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs index 2fec49f210..faba3d7870 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs @@ -6,7 +6,7 @@ namespace CosmosBenchmark { public interface IMetricsCollector { - void RecordLatency(double milliseconds); + void RecordLatencyAndRps(double milliseconds); void CollectMetricsOnSuccess(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs index dbb4cee2cd..79d472aaf4 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs @@ -5,14 +5,15 @@ namespace CosmosBenchmark { using Microsoft.ApplicationInsights; + using System.Diagnostics.Metrics; internal class InsertOperationMetricsCollector : MetricsCollector { - public InsertOperationMetricsCollector(TelemetryClient telemetryClient) : base(telemetryClient) + public InsertOperationMetricsCollector(Meter meter) : base(meter) { } - protected override string AverageRpsMetricName => "InsertOperationAverageRps"; + protected override string RpsMetricName => "InsertOperationRps"; protected override string LatencyInMsMetricName => "InsertOperationLatencyInMs"; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs index 7de637600b..dfbf1acce7 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs @@ -4,161 +4,53 @@ namespace CosmosBenchmark { - using System; using System.Collections.Generic; - using System.Linq; - using Microsoft.ApplicationInsights; - using Microsoft.ApplicationInsights.DataContracts; + using System.Diagnostics.Metrics; internal abstract class MetricsCollector : IMetricsCollector { - private readonly List operationLatenciesInMs = new List(); + private readonly Histogram operationLatencyHistogram; - protected readonly TelemetryClient telemetryClient; + private readonly List latencies; - protected int successCounter; + private readonly Histogram rpsMetricNameHistogram; - protected int failureCounter; + private readonly Counter successOperationCounter; - public MetricsCollector(TelemetryClient telemetryClient) + private readonly Counter failureOperationCounter; + + public MetricsCollector(Meter meter) { - this.telemetryClient = telemetryClient; - this.successCounter = 0; - this.failureCounter = 0; + this.latencies = new List(); + this.rpsMetricNameHistogram = meter.CreateHistogram(this.RpsMetricName); + this.operationLatencyHistogram = meter.CreateHistogram(this.LatencyInMsMetricName); + this.successOperationCounter = meter.CreateCounter(this.SuccessOperationMetricName); + this.failureOperationCounter = meter.CreateCounter(this.FailureOperationMetricName); } public void CollectMetricsOnSuccess() { - MetricTelemetry metricTelemetry = new MetricTelemetry(this.SuccessOperationMetricName, ++this.successCounter); - this.telemetryClient.TrackMetric(metricTelemetry); + this.successOperationCounter.Add(1); } public void CollectMetricsOnFailure() { - MetricTelemetry metricTelemetry = new MetricTelemetry(this.FailureOperationMetricName, ++this.failureCounter); - this.telemetryClient.TrackMetric(metricTelemetry); + this.failureOperationCounter.Add(1); } - public void RecordLatency(double milliseconds) + public void RecordLatencyAndRps(double milliseconds) { - this.operationLatenciesInMs.Add(milliseconds); - this.TelemetryPercentiles(); + double rps = 1000 / milliseconds; + this.rpsMetricNameHistogram.Record(rps); + this.operationLatencyHistogram.Record(milliseconds); } - protected abstract string AverageRpsMetricName { get; } + protected abstract string RpsMetricName { get; } protected abstract string LatencyInMsMetricName { get; } protected abstract string FailureOperationMetricName { get; } protected abstract string SuccessOperationMetricName { get; } - - internal double GetLatencyPercentile(int percentile) - { - return MathNet.Numerics.Statistics.Statistics.Percentile(this.operationLatenciesInMs, percentile); - } - - internal double GetLatencyQuantile(double quantile) - { - return MathNet.Numerics.Statistics.Statistics.Quantile(this.operationLatenciesInMs, quantile); - } - - private void TelemetryPercentiles() - { - if (this.operationLatenciesInMs.Count > 10) - { - double top10PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.1 * this.operationLatenciesInMs.Count)).Average(), 0); - MetricTelemetry metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top10Percent", top10PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top20PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.2 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top20Percent", top20PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top30PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.3 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top30Percent", top30PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top40PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.4 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top40Percent", top40PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top50PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.5 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top50Percent", top50PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top60PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.6 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top60Percent", top60PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top70PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.7 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top70Percent", top70PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top80PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.8 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top80Percent", top80PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top90PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.9 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top90Percent", top90PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top95PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.95 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top95Percent", top95PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top99PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.99 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top99Percent", top99PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top999PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.999 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top99.9Percent", top999PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top9999PercentAverageRps = Math.Round(this.operationLatenciesInMs.Take((int)(0.9999 * this.operationLatenciesInMs.Count)).Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName + "-Top99.99Percent", top9999PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double averageRps = Math.Round(this.operationLatenciesInMs.Average(), 0); - metricTelemetry = new MetricTelemetry(this.AverageRpsMetricName, top9999PercentAverageRps); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top50PercentLatencyInMs = this.GetLatencyPercentile(50); - metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P50", top50PercentLatencyInMs); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top75PercentLatencyInMs = this.GetLatencyPercentile(75); - metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P75", top75PercentLatencyInMs); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top90PercentLatencyInMs = this.GetLatencyPercentile(90); - metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P90", top90PercentLatencyInMs); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top95PercentLatencyInMs = this.GetLatencyPercentile(95); - metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P95", top95PercentLatencyInMs); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top98PercentLatencyInMs = this.GetLatencyPercentile(98); - metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P98", top95PercentLatencyInMs); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top99PercentLatencyInMs = this.GetLatencyPercentile(99); - metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P99", top99PercentLatencyInMs); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top999PercentLatencyInMs = this.GetLatencyQuantile(0.999); - metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-Q999", top999PercentLatencyInMs); - this.telemetryClient.TrackMetric(metricTelemetry); - - double top9999PercentLatencyInMs = this.GetLatencyQuantile(0.9999); - metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-Q9999", top9999PercentLatencyInMs); - this.telemetryClient.TrackMetric(metricTelemetry); - - double maxLatencyInMs = this.GetLatencyPercentile(100); - metricTelemetry = new MetricTelemetry(this.LatencyInMsMetricName + "-P100", maxLatencyInMs); - this.telemetryClient.TrackMetric(metricTelemetry); - } - } } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs index 0f78d0348c..889de4de72 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs @@ -5,7 +5,8 @@ namespace CosmosBenchmark { using System; - using Microsoft.ApplicationInsights; + using System.Diagnostics.Metrics; + using OpenTelemetry.Metrics; internal static class MetricsCollectorProvider { @@ -13,23 +14,26 @@ internal static class MetricsCollectorProvider private static readonly object metricCollectionWindowLock = new object(); - private static MetricCollectionWindow CurrentMetricCollectionWindow + private static readonly Meter insertOperationMeter = new("CosmosBenchmarkInsertOperationMeter"); + + private static readonly Meter queryOperationMeter = new("CosmosBenchmarkQueryOperationMeter"); + + private static readonly Meter readOperationMeter = new("CosmosBenchmarkReadOperationMeter"); + + private static MetricCollectionWindow GetCurrentMetricCollectionWindow(MeterProvider meterProvider) { - get + if (CheckMetricCollectionInvalid(metricCollectionWindow)) { - if (CheckMetricCollectionInvalid(metricCollectionWindow)) + lock (metricCollectionWindowLock) { - lock (metricCollectionWindowLock) + if (CheckMetricCollectionInvalid(metricCollectionWindow)) { - if (CheckMetricCollectionInvalid(metricCollectionWindow)) - { - metricCollectionWindow = new MetricCollectionWindow(); - } + metricCollectionWindow = new MetricCollectionWindow(); } } - - return metricCollectionWindow; } + + return metricCollectionWindow; } private static bool CheckMetricCollectionInvalid(MetricCollectionWindow metricCollectionWindow) @@ -37,20 +41,20 @@ private static bool CheckMetricCollectionInvalid(MetricCollectionWindow metricCo return metricCollectionWindow is null || DateTime.Now > metricCollectionWindow.DateTimeCreated.AddSeconds(5); } - public static IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation, TelemetryClient telemetryClient) + public static IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation, MeterProvider meterProvider) { Type benchmarkOperationType = benchmarkOperation.GetType(); if (typeof(InsertBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) { - return CurrentMetricCollectionWindow.GetInsertOperationMetricsCollector(telemetryClient); + return GetCurrentMetricCollectionWindow(meterProvider).GetInsertOperationMetricsCollector(insertOperationMeter); } else if (typeof(QueryBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) { - return CurrentMetricCollectionWindow.GetQueryOperationMetricsCollector(telemetryClient); + return GetCurrentMetricCollectionWindow(meterProvider).GetQueryOperationMetricsCollector(queryOperationMeter); } else if (typeof(ReadBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) { - return CurrentMetricCollectionWindow.GetReadOperationMetricsCollector(telemetryClient); + return GetCurrentMetricCollectionWindow(meterProvider).GetReadOperationMetricsCollector(readOperationMeter); } else { diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs index 86b791ad24..a88e365db9 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs @@ -13,6 +13,7 @@ namespace CosmosBenchmark using Microsoft.ApplicationInsights; using Microsoft.Extensions.Logging; using Newtonsoft.Json; + using OpenTelemetry.Metrics; internal class ParallelExecutionStrategy : IExecutionStrategy { @@ -32,7 +33,7 @@ public async Task ExecuteAsync( int serialExecutorIterationCount, double warmupFraction, ILogger logger, - TelemetryClient telemetryClient) + MeterProvider meterProvider) { IExecutor warmupExecutor = new SerialOperationExecutor( executorId: "Warmup", @@ -44,7 +45,7 @@ await warmupExecutor.ExecuteAsync( completionCallback: () => { }, benchmarkConfig, logger, - telemetryClient); + meterProvider); IExecutor[] executors = new IExecutor[serialExecutorConcurrency]; for (int i = 0; i < serialExecutorConcurrency; i++) @@ -64,21 +65,21 @@ await warmupExecutor.ExecuteAsync( completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount), benchmarkConfig, logger, - telemetryClient); + meterProvider); } return await this.LogOutputStats( benchmarkConfig, executors, logger, - telemetryClient); + meterProvider); } private async Task LogOutputStats( BenchmarkConfig benchmarkConfig, IExecutor[] executors, ILogger logger, - TelemetryClient telemetryClient) + MeterProvider meterProvider) { const int outputLoopDelayInSeconds = 1; IList perLoopCounters = new List(); diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs index 7512b3c77a..c84b2d9385 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs @@ -4,15 +4,15 @@ namespace CosmosBenchmark { - using Microsoft.ApplicationInsights; + using System.Diagnostics.Metrics; internal class QueryOperationMetricsCollector : MetricsCollector { - public QueryOperationMetricsCollector(TelemetryClient telemetryClient) : base(telemetryClient) + public QueryOperationMetricsCollector(Meter meter) : base(meter) { } - protected override string AverageRpsMetricName => "QueryOperationAverageRps"; + protected override string RpsMetricName => "QueryOperationRps"; protected override string LatencyInMsMetricName => "QueryOperationLatencyInMs"; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs index 3109b88d50..f798c22566 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs @@ -4,20 +4,20 @@ namespace CosmosBenchmark { - using Microsoft.ApplicationInsights; + using System.Diagnostics.Metrics; internal class ReadOperationMetricsCollector : MetricsCollector { - public ReadOperationMetricsCollector(TelemetryClient telemetryClient) : base(telemetryClient) + public ReadOperationMetricsCollector(Meter meter) : base(meter) { } - protected override string AverageRpsMetricName => "ReadOperationAverageRps"; + protected override string RpsMetricName => "ReadOperationRps"; protected override string LatencyInMsMetricName => "ReadOperationLatencyInMs"; protected override string FailureOperationMetricName => "ReadOperationFailure"; - protected override string SuccessOperationMetricName => "ReadOperationFailure"; + protected override string SuccessOperationMetricName => "ReadOperationSuccess"; } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs index 2bca6291e6..326e214f35 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs @@ -7,9 +7,9 @@ namespace CosmosBenchmark using System; using System.Diagnostics; using System.Threading.Tasks; - using Microsoft.ApplicationInsights; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; + using OpenTelemetry.Metrics; internal class SerialOperationExecutor : IExecutor { @@ -41,7 +41,7 @@ public async Task ExecuteAsync( Action completionCallback, BenchmarkConfig benchmarkConfig, ILogger logger, - TelemetryClient telemetryClient) + MeterProvider meterProvider) { logger.LogInformation($"Executor {this.executorId} started"); @@ -52,7 +52,7 @@ public async Task ExecuteAsync( int currentIterationCount = 0; do { - IMetricsCollector metricsCollector = MetricsCollectorProvider.GetMetricsCollector(this.operation, telemetryClient); + IMetricsCollector metricsCollector = MetricsCollectorProvider.GetMetricsCollector(this.operation, meterProvider); OperationResult? operationResult = null; @@ -61,7 +61,7 @@ public async Task ExecuteAsync( using (IDisposable telemetrySpan = TelemetrySpan.StartNew( () => operationResult.Value, disableTelemetry: isWarmup, - metricsCollector.RecordLatency)) + metricsCollector.RecordLatencyAndRps)) { try { diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs index 4bdf46efba..9b2a3ac13a 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/MetricCollectionWindow.cs @@ -1,7 +1,7 @@ namespace CosmosBenchmark { using System; - using Microsoft.ApplicationInsights; + using System.Diagnostics.Metrics; internal class MetricCollectionWindow { @@ -21,39 +21,39 @@ public MetricCollectionWindow() this.DateTimeCreated = DateTime.Now; } - public InsertOperationMetricsCollector GetInsertOperationMetricsCollector(TelemetryClient telemetryClient) + public InsertOperationMetricsCollector GetInsertOperationMetricsCollector(Meter meter) { if (this.insertOperationMetricsCollector is null) { lock (this.insertOperationMetricsCollectorLock) { - this.insertOperationMetricsCollector ??= new InsertOperationMetricsCollector(telemetryClient); + this.insertOperationMetricsCollector ??= new InsertOperationMetricsCollector(meter); } } return this.insertOperationMetricsCollector; } - public QueryOperationMetricsCollector GetQueryOperationMetricsCollector(TelemetryClient telemetryClient) + public QueryOperationMetricsCollector GetQueryOperationMetricsCollector(Meter meter) { if (this.queryOperationMetricsCollector is null) { lock (this.queryOperationMetricsCollectorLock) { - this.queryOperationMetricsCollector ??= new QueryOperationMetricsCollector(telemetryClient); + this.queryOperationMetricsCollector ??= new QueryOperationMetricsCollector(meter); } } return this.queryOperationMetricsCollector; } - public ReadOperationMetricsCollector GetReadOperationMetricsCollector(TelemetryClient telemetryClient) + public ReadOperationMetricsCollector GetReadOperationMetricsCollector(Meter meter) { if (this.readOperationMetricsCollector is null) { lock (this.readOperationMetricsCollectorLock) { - this.readOperationMetricsCollector ??= new ReadOperationMetricsCollector(telemetryClient); + this.readOperationMetricsCollector ??= new ReadOperationMetricsCollector(meter); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs index c324f81bfe..6a28432d40 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Program.cs @@ -14,11 +14,12 @@ namespace CosmosBenchmark using System.Reflection; using System.Threading; using System.Threading.Tasks; + using Azure.Monitor.OpenTelemetry.Exporter; using Microsoft.Azure.Cosmos; using Microsoft.Extensions.Logging; - using Microsoft.ApplicationInsights; using Newtonsoft.Json.Linq; - using Microsoft.ApplicationInsights.Extensibility; + using OpenTelemetry; + using OpenTelemetry.Metrics; /// /// This sample demonstrates how to achieve high performance writes using Azure Comsos DB. @@ -36,6 +37,18 @@ public static async Task Main(string[] args) BenchmarkConfig config = BenchmarkConfig.From(args); await Program.AddAzureInfoToRunSummary(); + OpenTelemetry.Trace.TracerProviderBuilder tracerProviderBuilder = Sdk.CreateTracerProviderBuilder() + .AddAzureMonitorTraceExporter(); + + MeterProvider meterProvider = Sdk.CreateMeterProviderBuilder() + .AddAzureMonitorMetricExporter(configure: new Action((options) => Configure(options, config))) + .AddMeter("CosmosBenchmarkInsertOperationMeter") + .AddMeter("CosmosBenchmarkQueryOperationMeter") + .AddMeter("CosmosBenchmarkReadOperationMeter") + .AddConsoleExporter() + .AddAzureMonitorMetricExporter() + .Build(); + ThreadPool.SetMinThreads(config.MinThreadPoolSize, config.MinThreadPoolSize); if (config.EnableLatencyPercentiles) @@ -49,12 +62,9 @@ public static async Task Main(string[] args) using ILoggerFactory loggerFactory = LoggerFactory.Create(builder => builder.AddConsole()); ILogger logger = loggerFactory.CreateLogger(); - TelemetryConfiguration telemetryConfiguration = new(config.AppInsightsInstrumentationKey); - TelemetryClient telemetryClient = new TelemetryClient(telemetryConfiguration); - Program program = new Program(); - RunSummary runSummary = await program.ExecuteAsync(config, logger, telemetryClient); + RunSummary runSummary = await program.ExecuteAsync(config, logger, meterProvider); } finally { @@ -90,11 +100,16 @@ private static async Task AddAzureInfoToRunSummary() } } + private static void Configure(AzureMonitorExporterOptions options, BenchmarkConfig config) + { + options.ConnectionString = $"InstrumentationKey={config.AppInsightsInstrumentationKey}"; + } + /// /// Executing benchmarks for V2/V3 cosmosdb SDK. /// /// a Task object. - private async Task ExecuteAsync(BenchmarkConfig config, ILogger logger, TelemetryClient telemetryClient) + private async Task ExecuteAsync(BenchmarkConfig config, ILogger logger, MeterProvider meterProvider) { // V3 SDK client initialization using (CosmosClient cosmosClient = config.CreateCosmosClient(config.Key)) @@ -146,7 +161,7 @@ private async Task ExecuteAsync(BenchmarkConfig config, ILogger logg } IExecutionStrategy execution = IExecutionStrategy.StartNew(benchmarkOperationFactory); - runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, logger, telemetryClient); + runSummary = await execution.ExecuteAsync(config, taskCount, opsPerTask, 0.01, logger, meterProvider); } if (config.CleanupOnFinish) From 270aeced7d77c08622ca996b50ae20e62236de76 Mon Sep 17 00:00:00 2001 From: Mikhail Lipin Date: Mon, 19 Jun 2023 15:20:18 +0300 Subject: [PATCH 07/10] Adding gauges. --- .../Fx/InsertOperationMetricsCollector.cs | 4 +++ .../Tools/Benchmark/Fx/MetricsCollector.cs | 36 ++++++++++++++----- .../Benchmark/Fx/MetricsCollectorProvider.cs | 3 +- .../Fx/QueryOperationMetricsCollector.cs | 4 +++ .../Fx/ReadOperationMetricsCollector.cs | 4 +++ 5 files changed, 41 insertions(+), 10 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs index 79d472aaf4..060a85780d 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs @@ -13,6 +13,10 @@ public InsertOperationMetricsCollector(Meter meter) : base(meter) { } + protected override string RpsHistogramName => "InsertOperationRpsHistogram"; + + protected override string LatencyInMsHistogramName => "InsertOperationLatencyInMsHistogram"; + protected override string RpsMetricName => "InsertOperationRps"; protected override string LatencyInMsMetricName => "InsertOperationLatencyInMs"; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs index dfbf1acce7..8925bff3a2 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs @@ -4,14 +4,13 @@ namespace CosmosBenchmark { - using System.Collections.Generic; using System.Diagnostics.Metrics; internal abstract class MetricsCollector : IMetricsCollector { - private readonly Histogram operationLatencyHistogram; + private readonly Meter meter; - private readonly List latencies; + private readonly Histogram operationLatencyHistogram; private readonly Histogram rpsMetricNameHistogram; @@ -19,13 +18,27 @@ internal abstract class MetricsCollector : IMetricsCollector private readonly Counter failureOperationCounter; + private readonly ObservableGauge latencyInMsMetricNameGauge; + + private readonly ObservableGauge rpsNameGauge; + + private double latencyInMs; + + private double rps; + public MetricsCollector(Meter meter) { - this.latencies = new List(); - this.rpsMetricNameHistogram = meter.CreateHistogram(this.RpsMetricName); - this.operationLatencyHistogram = meter.CreateHistogram(this.LatencyInMsMetricName); + this.meter = meter; + this.rpsMetricNameHistogram = meter.CreateHistogram(this.RpsHistogramName); + this.operationLatencyHistogram = meter.CreateHistogram(this.LatencyInMsHistogramName); this.successOperationCounter = meter.CreateCounter(this.SuccessOperationMetricName); this.failureOperationCounter = meter.CreateCounter(this.FailureOperationMetricName); + + this.latencyInMsMetricNameGauge = this.meter.CreateObservableGauge(this.LatencyInMsMetricName, + () => new Measurement(this.latencyInMs)); + + this.rpsNameGauge = this.meter.CreateObservableGauge(this.LatencyInMsMetricName, + () => new Measurement(this.rps)); } public void CollectMetricsOnSuccess() @@ -40,11 +53,16 @@ public void CollectMetricsOnFailure() public void RecordLatencyAndRps(double milliseconds) { - double rps = 1000 / milliseconds; - this.rpsMetricNameHistogram.Record(rps); - this.operationLatencyHistogram.Record(milliseconds); + this.rps = 1000 / milliseconds; + this.latencyInMs = milliseconds; + this.rpsMetricNameHistogram.Record(this.rps); + this.operationLatencyHistogram.Record(this.latencyInMs); } + protected abstract string RpsHistogramName { get; } + + protected abstract string LatencyInMsHistogramName { get; } + protected abstract string RpsMetricName { get; } protected abstract string LatencyInMsMetricName { get; } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs index 889de4de72..42de2561ca 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs @@ -18,7 +18,7 @@ internal static class MetricsCollectorProvider private static readonly Meter queryOperationMeter = new("CosmosBenchmarkQueryOperationMeter"); - private static readonly Meter readOperationMeter = new("CosmosBenchmarkReadOperationMeter"); + private static readonly Meter readOperationMeter = new ("CosmosBenchmarkReadOperationMeter"); private static MetricCollectionWindow GetCurrentMetricCollectionWindow(MeterProvider meterProvider) { @@ -28,6 +28,7 @@ private static MetricCollectionWindow GetCurrentMetricCollectionWindow(MeterProv { if (CheckMetricCollectionInvalid(metricCollectionWindow)) { + // Reset Meter provider and meters. metricCollectionWindow = new MetricCollectionWindow(); } } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs index c84b2d9385..c9a247d530 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs @@ -12,6 +12,10 @@ public QueryOperationMetricsCollector(Meter meter) : base(meter) { } + protected override string RpsHistogramName => "QueryOperationRpsHistogram"; + + protected override string LatencyInMsHistogramName => "QueryOperationLatencyInMsHistogram"; + protected override string RpsMetricName => "QueryOperationRps"; protected override string LatencyInMsMetricName => "QueryOperationLatencyInMs"; diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs index f798c22566..2ba49cc438 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadOperationMetricsCollector.cs @@ -12,6 +12,10 @@ public ReadOperationMetricsCollector(Meter meter) : base(meter) { } + protected override string RpsHistogramName => "ReadOperationRpsHistogram"; + + protected override string LatencyInMsHistogramName => "ReadOperationLatencyInMsHistogram"; + protected override string RpsMetricName => "ReadOperationRps"; protected override string LatencyInMsMetricName => "ReadOperationLatencyInMs"; From 33a2597e6d3a333d65f1a0c876b74bb2336e19d4 Mon Sep 17 00:00:00 2001 From: Mikhail Lipin Date: Mon, 19 Jun 2023 18:41:41 +0300 Subject: [PATCH 08/10] Adding configurable flush interval. --- .../Tools/Benchmark/BenchmarkConfig.cs | 2 +- .../Benchmark/Fx/MetricsCollectorProvider.cs | 19 ++++++++++--------- .../Benchmark/Fx/SerialOperationExecutor.cs | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs index 7f22090503..71e2c5eef5 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs @@ -124,7 +124,7 @@ public class BenchmarkConfig public string ResultsContainer { get; set; } = "runsummary"; [Option(Required = false, HelpText = "Metrics reporting interval in seconds")] - public int MetricsReportingIntervalInSec { get; set; } + public int MetricsReportingIntervalInSec { get; set; } = 5; [Option(Required = false, HelpText = "Application Insights instrumentation key")] public string AppInsightsInstrumentationKey { get; set; } diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs index 42de2561ca..104551f240 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs @@ -20,16 +20,17 @@ internal static class MetricsCollectorProvider private static readonly Meter readOperationMeter = new ("CosmosBenchmarkReadOperationMeter"); - private static MetricCollectionWindow GetCurrentMetricCollectionWindow(MeterProvider meterProvider) + private static MetricCollectionWindow GetCurrentMetricCollectionWindow(MeterProvider meterProvider, BenchmarkConfig config) { - if (CheckMetricCollectionInvalid(metricCollectionWindow)) + if (CheckMetricCollectionInvalid(metricCollectionWindow, config)) { lock (metricCollectionWindowLock) { - if (CheckMetricCollectionInvalid(metricCollectionWindow)) + if (CheckMetricCollectionInvalid(metricCollectionWindow, config)) { // Reset Meter provider and meters. metricCollectionWindow = new MetricCollectionWindow(); + meterProvider.ForceFlush(); } } } @@ -37,25 +38,25 @@ private static MetricCollectionWindow GetCurrentMetricCollectionWindow(MeterProv return metricCollectionWindow; } - private static bool CheckMetricCollectionInvalid(MetricCollectionWindow metricCollectionWindow) + private static bool CheckMetricCollectionInvalid(MetricCollectionWindow metricCollectionWindow, BenchmarkConfig config) { - return metricCollectionWindow is null || DateTime.Now > metricCollectionWindow.DateTimeCreated.AddSeconds(5); + return metricCollectionWindow is null || DateTime.Now > metricCollectionWindow.DateTimeCreated.AddSeconds(config.MetricsReportingIntervalInSec); } - public static IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation, MeterProvider meterProvider) + public static IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation, MeterProvider meterProvider, BenchmarkConfig config) { Type benchmarkOperationType = benchmarkOperation.GetType(); if (typeof(InsertBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) { - return GetCurrentMetricCollectionWindow(meterProvider).GetInsertOperationMetricsCollector(insertOperationMeter); + return GetCurrentMetricCollectionWindow(meterProvider, config).GetInsertOperationMetricsCollector(insertOperationMeter); } else if (typeof(QueryBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) { - return GetCurrentMetricCollectionWindow(meterProvider).GetQueryOperationMetricsCollector(queryOperationMeter); + return GetCurrentMetricCollectionWindow(meterProvider, config).GetQueryOperationMetricsCollector(queryOperationMeter); } else if (typeof(ReadBenchmarkOperation).IsAssignableFrom(benchmarkOperationType)) { - return GetCurrentMetricCollectionWindow(meterProvider).GetReadOperationMetricsCollector(readOperationMeter); + return GetCurrentMetricCollectionWindow(meterProvider, config).GetReadOperationMetricsCollector(readOperationMeter); } else { diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs index 326e214f35..99428d154d 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/SerialOperationExecutor.cs @@ -52,7 +52,7 @@ public async Task ExecuteAsync( int currentIterationCount = 0; do { - IMetricsCollector metricsCollector = MetricsCollectorProvider.GetMetricsCollector(this.operation, meterProvider); + IMetricsCollector metricsCollector = MetricsCollectorProvider.GetMetricsCollector(this.operation, meterProvider, benchmarkConfig); OperationResult? operationResult = null; From 7888b8e59580d3eca52397fda8351fd2dd272c87 Mon Sep 17 00:00:00 2001 From: Mikhail Lipin Date: Thu, 22 Jun 2023 17:14:20 +0300 Subject: [PATCH 09/10] Adding ARM template for Dashboard. --- .../ARMTemplate/benchmarkTemplate.json | 186 +++++++++++++++++- 1 file changed, 185 insertions(+), 1 deletion(-) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json index 13bb3070da..0bdabc5165 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json @@ -88,9 +88,19 @@ "metadata": { "description": "Location for all resources." } + }, + "dashboardName": { + "type": "string", + "defaultValue": "Cosmos Benchmark Statistics" + }, + "appInsightsResoureName": { + "type": "string", + "defaultValue": "CosmosDBBenchmarkTestAppInsights" } }, - "variables": {}, + "variables": { + "appInsightsResourceIds": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/microsoft.insights/components/', parameters('appInsightsResoureName'))]" + }, "resources": [ { "name": "[parameters('containerGroupName')]", @@ -149,6 +159,180 @@ } ] } + }, + { + "name": "CosmosDBBenchmarkDashboard", + "type": "Microsoft.Portal/dashboards", + "location": "[resourceGroup().location]", + "apiVersion": "2015-08-01-preview", + "tags": { + "hidden-title": "[parameters('dashboardName')]" + }, + + "properties": { + "lenses": { + "0": { + "order": 0, + "parts": { + "0": { + "position": { + "x": 0, + "y": 0, + "colSpan": 11, + "rowSpan": 5 + }, + "metadata": { + "inputs": [ + { + "name": "resourceTypeMode", + "isOptional": true + }, + { + "name": "ComponentId", + "isOptional": true + }, + { + "name": "Scope", + "value": { + "resourceIds": [ + "[variables('appInsightsResourceIds')]" + ] + }, + "isOptional": true + }, + { + "name": "PartId", + "value": "104528fc-0216-49c6-bf70-fffe9d37f93d", + "isOptional": true + }, + { + "name": "Version", + "value": "2.0", + "isOptional": true + }, + { + "name": "TimeRange", + "isOptional": true + }, + { + "name": "DashboardId", + "isOptional": true + }, + { + "name": "DraftRequestParameters", + "isOptional": true + }, + { + "name": "Query", + "value": "customMetrics\n| where name == \"ReadOperationLatencyInMs\" and timestamp > ago(1d)\n| summarize\n percentile(value, 50),\n percentile(value, 75),\n percentile(value, 90),\n percentile(value, 95),\n percentile(value, 99),\n percentile(value, 99.9),\n percentile(value, 99.99)\n by ts = bin(timestamp, 5s)\n| render timechart \n\n", + "isOptional": true + }, + { + "name": "ControlType", + "value": "FrameControlChart", + "isOptional": true + }, + { + "name": "SpecificChart", + "value": "Line", + "isOptional": true + }, + { + "name": "PartTitle", + "value": "Analytics", + "isOptional": true + }, + { + "name": "PartSubTitle", + "value": "CosmosDBBenchmarkTestAppInsights", + "isOptional": true + }, + { + "name": "Dimensions", + "value": { + "xAxis": { + "name": "ts", + "type": "datetime" + }, + "yAxis": [ + { + "name": "percentile_value_50", + "type": "real" + }, + { + "name": "percentile_value_75", + "type": "real" + }, + { + "name": "percentile_value_90", + "type": "real" + }, + { + "name": "percentile_value_95", + "type": "real" + } + ], + "splitBy": [], + "aggregation": "Sum" + }, + "isOptional": true + }, + { + "name": "LegendOptions", + "value": { + "isEnabled": true, + "position": "Bottom" + }, + "isOptional": true + }, + { + "name": "IsQueryContainTimeRange", + "value": true, + "isOptional": true + } + ], + "type": "Extension/Microsoft_OperationsManagementSuite_Workspace/PartType/LogsDashboardPart", + "settings": {} + } + } + } + } + }, + "metadata": { + "model": { + "timeRange": { + "value": { + "relative": { + "duration": 24, + "timeUnit": 1 + } + }, + "type": "MsPortalFx.Composition.Configuration.ValueTypes.TimeRange" + }, + "filterLocale": { + "value": "en-us" + }, + "filters": { + "value": { + "MsPortalFx_TimeRange": { + "model": { + "format": "utc", + "granularity": "auto", + "relative": "24h" + }, + "displayCache": { + "name": "UTC Time", + "value": "Past 24 hours" + }, + "filteredPartIds": [ + "StartboardPart-LogsDashboardPart-4f9d44ab-efbe-4310-8809-63ae942a3091" + ] + } + } + } + } + } + } } ] } \ No newline at end of file From c55f4c215e86b27c4d24489edc44e2b5934e9238 Mon Sep 17 00:00:00 2001 From: Mikhail Lipin Date: Thu, 22 Jun 2023 17:42:50 +0300 Subject: [PATCH 10/10] Adding Application Insights resource. --- .../ARMTemplate/benchmarkTemplate.json | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json index 0bdabc5165..ac163dbad9 100644 --- a/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json +++ b/Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json @@ -45,7 +45,7 @@ "metadata": { "description": "Name for the container group" }, - "defaultValue": "CosmosDBBenchmark" + "defaultValue": "CosmosDBBenchmark" }, "containerName": { "type": "string", @@ -89,17 +89,20 @@ "description": "Location for all resources." } }, - "dashboardName": { + "applicationInsightsName": { "type": "string", - "defaultValue": "Cosmos Benchmark Statistics" + "defaultValue": "CosmosDBBenchmarkTestAppInsights", + "metadata": { + "description": "Specifies Application Insights Account Nmae" + } }, - "appInsightsResoureName": { + "dashboardName": { "type": "string", - "defaultValue": "CosmosDBBenchmarkTestAppInsights" + "defaultValue": "Cosmos Benchmark Statistics" } }, "variables": { - "appInsightsResourceIds": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/microsoft.insights/components/', parameters('appInsightsResoureName'))]" + "appInsightsResourceIds": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/microsoft.insights/components/', parameters('applicationInsightsName'))]" }, "resources": [ { @@ -160,6 +163,16 @@ ] } }, + { + "name": "[parameters('applicationInsightsName')]", + "type": "Microsoft.Insights/components", + "apiVersion": "2015-05-01", + "location": "[resourceGroup().location]", + "properties": { + "ApplicationId": "[parameters('applicationInsightsName')]", + "Application_Type": "web" + } + }, { "name": "CosmosDBBenchmarkDashboard", "type": "Microsoft.Portal/dashboards",