Draft: 3892: Adding metrics to Azure Benchmark tool

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs

@@ -11,6 +11,7 @@ namespace CosmosBenchmark
     using CommandLine;
     using Microsoft.Azure.Documents.Client;
     using Newtonsoft.Json;
+    using static CosmosBenchmark.ReservoirProvider;
 
     public class BenchmarkConfig
     {
@@ -123,6 +124,21 @@ public class BenchmarkConfig
         [Option(Required = false, HelpText = "Container to publish results to")]
         public string ResultsContainer { get; set; } = "runsummary";
 
+        [Option(Required = false, HelpText = "Metrics reporting interval in seconds")]
+        public int MetricsReportingIntervalInSec { get; set; }
+
+        [Option(Required = false, HelpText = "Application Insights instrumentation key")]
+        public string AppInsightsInstrumentationKey { get; set; }
+
+        [Option(Required = false, HelpText = "Defines the reservoir type. Valid values are: Uniform, SlidingWindow and ExponentialDecay. The default value is SlidingWindow.")]
+        public ReservoirTypes ReservoirType { get; set; } = ReservoirTypes.SlidingWindow;
+
+        [Option(Required = false, HelpText = "The reservoir sample size.")]
+        public int ReservoirSampleSize { get; set; } = 1028;
+
+        [Option(Required = false, HelpText = "Logging context name. The default value is \"CosmosDBBenchmarkLoggingContext\"")]
+        public string LoggingContextIdentifier { get; set; } = "CosmosDBBenchmarkLoggingContext";
+
         internal int GetTaskCount(int containerThroughput)
         {
             int taskCount = this.DegreeOfParallelism;
@@ -270,4 +286,4 @@ private static void HandleParseError(IEnumerable<Error> errors)
             Environment.Exit(errors.Count());
         }
     }
-}
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfigProvider.cs

@@ -0,0 +1,16 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using App.Metrics;
+    using App.Metrics.Counter;
+    using App.Metrics.Timer;
+
+    // TODO: Temporary solution. Remove after adding DI.
+    public class BenchmarkConfigProvider
+    {
+        public static BenchmarkConfig CurrentBenchmarkConfig { get; set; }
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj

@@ -17,9 +17,17 @@
     </None>
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Include="App.Metrics" Version="4.3.0" />
+    <PackageReference Include="App.Metrics.Abstractions" Version="4.3.0" />
+    <PackageReference Include="App.Metrics.Reporting.ApplicationInsights" Version="3.0.0" />
+    <PackageReference Include="App.Metrics.Reporting.Console" Version="4.3.0" />
     <PackageReference Include="CommandLineParser" Version="2.8.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
+    <PackageReference Include="Microsoft.ApplicationInsights" Version="2.21.0" />
     <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="*" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(ProjectRef)' != 'True' ">

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/BenchmarkOperation.cs

@@ -0,0 +1,15 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using System.Threading.Tasks;
+
+    internal abstract class BenchmarkOperation : IBenchmarkOperation
+    {
+        public abstract Task<OperationResult> ExecuteOnceAsync();
+
+        public abstract Task PrepareAsync();
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs

@@ -6,6 +6,8 @@ namespace CosmosBenchmark
 {
     using System;
     using System.Threading.Tasks;
+    using App.Metrics;
+    using Microsoft.Extensions.Logging;
 
     internal interface IExecutionStrategy
     {
@@ -19,7 +21,8 @@ public Task<RunSummary> ExecuteAsync(
             BenchmarkConfig benchmarkConfig,
             int serialExecutorConcurrency,
             int serialExecutorIterationCount,
-            double warmupFraction);
-
+            double warmupFraction,
+            ILogger logger,
+            IMetrics metrics);
     }
 }

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs

@@ -6,6 +6,8 @@ namespace CosmosBenchmark
 {
     using System;
     using System.Threading.Tasks;
+    using App.Metrics;
+    using Microsoft.Extensions.Logging;
 
     internal interface IExecutor
     {
@@ -17,6 +19,9 @@ public Task ExecuteAsync(
                 int iterationCount,
                 bool isWarmup,
                 bool traceFailures,
-                Action completionCallback);
+                Action completionCallback,
+                ILogger logger,
+                IMetrics metrics,
+                BenchmarkConfig benchmarkConfig);
     }
 }

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs

@@ -0,0 +1,17 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using App.Metrics.Timer;
+
+    public interface IMetricsCollector
+    {
+        TimerContext GetTimer();
+
+        void CollectMetricsOnSuccess();
+
+        void CollectMetricsOnFailure();
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs

@@ -0,0 +1,13 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using App.Metrics;
+    using App.Metrics.Timer;
+
+    internal abstract class InsertBenchmarkOperation : BenchmarkOperation
+    {
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs

@@ -0,0 +1,31 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using App.Metrics.Timer;
+    using App.Metrics;
+
+    internal class InsertOperationMetricsCollector : MetricsCollector
+    {
+        public InsertOperationMetricsCollector(MetricsContext metricsContext, IMetrics metrics) : base(metricsContext, metrics)
+        {
+        }
+
+        public override TimerContext GetTimer()
+        {
+            return this.metrics.Measure.Timer.Time(this.metricsContext.InsertLatencyTimer);
+        }
+
+        public override void CollectMetricsOnSuccess()
+        {
+            this.metrics.Measure.Counter.Increment(this.metricsContext.WriteSuccessMeter);
+        }
+
+        public override void CollectMetricsOnFailure()
+        {
+            this.metrics.Measure.Counter.Increment(this.metricsContext.WriteFailureMeter);
+        }
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollector.cs

@@ -0,0 +1,28 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using App.Metrics.Timer;
+    using App.Metrics;
+
+    internal abstract class MetricsCollector : IMetricsCollector
+    {
+        protected MetricsContext metricsContext;
+
+        protected IMetrics metrics;
+
+        public MetricsCollector(MetricsContext metricsContext, IMetrics metrics)
+        {
+            this.metricsContext = metricsContext;
+            this.metrics = metrics;
+        }
+
+        public abstract TimerContext GetTimer();
+
+        public abstract void CollectMetricsOnFailure();
+
+        public abstract void CollectMetricsOnSuccess();
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/MetricsCollectorProvider.cs

@@ -0,0 +1,33 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using System;
+    using App.Metrics;
+
+    internal static class MetricsCollectorProvider
+    {
+        public static IMetricsCollector GetMetricsCollector(IBenchmarkOperation benchmarkOperation, MetricsContext metricsContext, IMetrics metrics)
+        {
+            Type benchmarkOperationType = benchmarkOperation.GetType();
+            if (typeof(InsertBenchmarkOperation).IsAssignableFrom(benchmarkOperationType))
+            {
+                return new InsertOperationMetricsCollector(metricsContext, metrics);
+            }
+            else if (typeof(QueryBenchmarkOperation).IsAssignableFrom(benchmarkOperationType))
+            {
+                return new QueryOperationMetricsCollector(metricsContext, metrics);
+            }
+            else if (typeof(ReadBenchmarkOperation).IsAssignableFrom(benchmarkOperationType))
+            {
+                return new ReadOperationMetricsCollector(metricsContext, metrics);
+            }
+            else
+            {
+                throw new NotSupportedException($"The type {nameof(benchmarkOperationType)} is not supported for collecting metrics.");
+            }
+        }
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ParallelExecutionStrategy.cs

@@ -10,6 +10,8 @@ namespace CosmosBenchmark
     using System.Linq;
     using System.Threading;
     using System.Threading.Tasks;
+    using App.Metrics;
+    using Microsoft.Extensions.Logging;
     using Newtonsoft.Json;
 
     internal class ParallelExecutionStrategy : IExecutionStrategy
@@ -28,7 +30,9 @@ public async Task<RunSummary> ExecuteAsync(
             BenchmarkConfig benchmarkConfig,
             int serialExecutorConcurrency,
             int serialExecutorIterationCount,
-            double warmupFraction)
+            double warmupFraction,
+            ILogger logger,
+            IMetrics metrics)
         {
             IExecutor warmupExecutor = new SerialOperationExecutor(
                         executorId: "Warmup",
@@ -37,7 +41,10 @@ await warmupExecutor.ExecuteAsync(
                     (int)(serialExecutorIterationCount * warmupFraction),
                     isWarmup: true,
                     traceFailures: benchmarkConfig.TraceFailures,
-                    completionCallback: () => { });
+                    completionCallback: () => { },
+                    logger,
+                    metrics,
+                    benchmarkConfig);
 
             IExecutor[] executors = new IExecutor[serialExecutorConcurrency];
             for (int i = 0; i < serialExecutorConcurrency; i++)
@@ -54,17 +61,22 @@ await warmupExecutor.ExecuteAsync(
                         iterationCount: serialExecutorIterationCount,
                         isWarmup: false,
                         traceFailures: benchmarkConfig.TraceFailures,
-                        completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount));
+                        completionCallback: () => Interlocked.Decrement(ref this.pendingExecutorCount),
+                        logger,
+                        metrics,
+                        benchmarkConfig);
             }
 
             return await this.LogOutputStats(
                 benchmarkConfig, 
-                executors);
+                executors,
+                metrics);
         }
 
         private async Task<RunSummary> LogOutputStats(
             BenchmarkConfig benchmarkConfig,
-            IExecutor[] executors)
+            IExecutor[] executors,
+            IMetrics metrics)
         {
             const int outputLoopDelayInSeconds = 1;
             IList<int> perLoopCounters = new List<int>();
@@ -135,6 +147,8 @@ private async Task<RunSummary> LogOutputStats(
                     runSummary.Top80PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.8 * summaryCounters.Length)).Average(), 0);
                     runSummary.Top90PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.9 * summaryCounters.Length)).Average(), 0);
                     runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.95 * summaryCounters.Length)).Average(), 0);
+                    runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.999 * summaryCounters.Length)).Average(), 0);
+                    runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.9999 * summaryCounters.Length)).Average(), 0);
                     runSummary.Top99PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.99 * summaryCounters.Length)).Average(), 0);
                     runSummary.AverageRps = Math.Round(summaryCounters.Average(), 0);
 
@@ -144,6 +158,8 @@ private async Task<RunSummary> LogOutputStats(
                     runSummary.Top95PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(95);
                     runSummary.Top98PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(98);
                     runSummary.Top99PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(99);
+                    runSummary.Top999PercentLatencyInMs = TelemetrySpan.GetLatencyQuantile(0.999);
+                    runSummary.Top9999PercentLatencyInMs = TelemetrySpan.GetLatencyQuantile(0.9999);
                     runSummary.MaxLatencyInMs = TelemetrySpan.GetLatencyPercentile(100);
 
                     string summary = JsonConvert.SerializeObject(runSummary);
@@ -160,4 +176,4 @@ private async Task<RunSummary> LogOutputStats(
             }
         }
     }
-}
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryBenchmarkOperation.cs

@@ -0,0 +1,10 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    internal abstract class QueryBenchmarkOperation : BenchmarkOperation
+    {
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/QueryOperationMetricsCollector.cs

@@ -0,0 +1,31 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using App.Metrics;
+    using App.Metrics.Timer;
+
+    internal class QueryOperationMetricsCollector : MetricsCollector
+    {
+        public QueryOperationMetricsCollector(MetricsContext metricsContext, IMetrics metrics) : base(metricsContext, metrics)
+        {
+        }
+
+        public override TimerContext GetTimer()
+        {
+            return this.metrics.Measure.Timer.Time(this.metricsContext.QueryLatencyTimer);
+        }
+
+        public override void CollectMetricsOnSuccess()
+        {
+            this.metrics.Measure.Counter.Increment(this.metricsContext.QuerySuccessMeter);
+        }
+
+        public override void CollectMetricsOnFailure()
+        {
+            this.metrics.Measure.Counter.Increment(this.metricsContext.QueryFailureMeter);
+        }
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/ReadBenchmarkOperation.cs

@@ -0,0 +1,10 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    internal abstract class ReadBenchmarkOperation : BenchmarkOperation
+    {
+    }
+}