Draft: 3892: Adding metrics to Azure Benchmark tool

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/ARMTemplate/benchmarkTemplate.json

@@ -45,7 +45,7 @@
       "metadata": {
         "description": "Name for the container group"
       },
-      "defaultValue":  "CosmosDBBenchmark"
+      "defaultValue": "CosmosDBBenchmark"
     },
     "containerName": {
       "type": "string",
@@ -88,9 +88,22 @@
       "metadata": {
         "description": "Location for all resources."
       }
+    },
+    "applicationInsightsName": {
+      "type": "string",
+      "defaultValue": "CosmosDBBenchmarkTestAppInsights",
+      "metadata": {
+        "description": "Specifies Application Insights Account Nmae"
+      }
+    },
+    "dashboardName": {
+      "type": "string",
+      "defaultValue": "Cosmos Benchmark Statistics"
     }
   },
-  "variables": {},
+  "variables": {
+    "appInsightsResourceIds": "[concat('/subscriptions/', subscription().subscriptionId, '/resourceGroups/', resourceGroup().name, '/providers/microsoft.insights/components/', parameters('applicationInsightsName'))]"
+  },
   "resources": [
     {
       "name": "[parameters('containerGroupName')]",
@@ -149,6 +162,190 @@
           }
         ]
       }
+    },
+    {
+      "name": "[parameters('applicationInsightsName')]",
+      "type": "Microsoft.Insights/components",
+      "apiVersion": "2015-05-01",
+      "location": "[resourceGroup().location]",
+      "properties": {
+        "ApplicationId": "[parameters('applicationInsightsName')]",
+        "Application_Type": "web"
+      }
+    },
+    {
+      "name": "CosmosDBBenchmarkDashboard",
+      "type": "Microsoft.Portal/dashboards",
+      "location": "[resourceGroup().location]",
+      "apiVersion": "2015-08-01-preview",
+      "tags": {
+        "hidden-title": "[parameters('dashboardName')]"
+      },
+
+      "properties": {
+        "lenses": {
+          "0": {
+            "order": 0,
+            "parts": {
+              "0": {
+                "position": {
+                  "x": 0,
+                  "y": 0,
+                  "colSpan": 11,
+                  "rowSpan": 5
+                },
+                "metadata": {
+                  "inputs": [
+                    {
+                      "name": "resourceTypeMode",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "ComponentId",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "Scope",
+                      "value": {
+                        "resourceIds": [
+                          "[variables('appInsightsResourceIds')]"
+                        ]
+                      },
+                      "isOptional": true
+                    },
+                    {
+                      "name": "PartId",
+                      "value": "104528fc-0216-49c6-bf70-fffe9d37f93d",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "Version",
+                      "value": "2.0",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "TimeRange",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "DashboardId",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "DraftRequestParameters",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "Query",
+                      "value": "customMetrics\n| where name == \"ReadOperationLatencyInMs\" and timestamp > ago(1d)\n| summarize\n    percentile(value, 50),\n    percentile(value, 75),\n    percentile(value, 90),\n    percentile(value, 95),\n    percentile(value, 99),\n    percentile(value, 99.9),\n    percentile(value, 99.99)\n    by ts = bin(timestamp, 5s)\n| render timechart \n\n",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "ControlType",
+                      "value": "FrameControlChart",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "SpecificChart",
+                      "value": "Line",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "PartTitle",
+                      "value": "Analytics",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "PartSubTitle",
+                      "value": "CosmosDBBenchmarkTestAppInsights",
+                      "isOptional": true
+                    },
+                    {
+                      "name": "Dimensions",
+                      "value": {
+                        "xAxis": {
+                          "name": "ts",
+                          "type": "datetime"
+                        },
+                        "yAxis": [
+                          {
+                            "name": "percentile_value_50",
+                            "type": "real"
+                          },
+                          {
+                            "name": "percentile_value_75",
+                            "type": "real"
+                          },
+                          {
+                            "name": "percentile_value_90",
+                            "type": "real"
+                          },
+                          {
+                            "name": "percentile_value_95",
+                            "type": "real"
+                          }
+                        ],
+                        "splitBy": [],
+                        "aggregation": "Sum"
+                      },
+                      "isOptional": true
+                    },
+                    {
+                      "name": "LegendOptions",
+                      "value": {
+                        "isEnabled": true,
+                        "position": "Bottom"
+                      },
+                      "isOptional": true
+                    },
+                    {
+                      "name": "IsQueryContainTimeRange",
+                      "value": true,
+                      "isOptional": true
+                    }
+                  ],
+                  "type": "Extension/Microsoft_OperationsManagementSuite_Workspace/PartType/LogsDashboardPart",
+                  "settings": {}
+                }
+              }
+            }
+          }
+        },
+        "metadata": {
+          "model": {
+            "timeRange": {
+              "value": {
+                "relative": {
+                  "duration": 24,
+                  "timeUnit": 1
+                }
+              },
+              "type": "MsPortalFx.Composition.Configuration.ValueTypes.TimeRange"
+            },
+            "filterLocale": {
+              "value": "en-us"
+            },
+            "filters": {
+              "value": {
+                "MsPortalFx_TimeRange": {
+                  "model": {
+                    "format": "utc",
+                    "granularity": "auto",
+                    "relative": "24h"
+                  },
+                  "displayCache": {
+                    "name": "UTC Time",
+                    "value": "Past 24 hours"
+                  },
+                  "filteredPartIds": [
+                    "StartboardPart-LogsDashboardPart-4f9d44ab-efbe-4310-8809-63ae942a3091"
+                  ]
+                }
+              }
+            }
+          }
+        }
+      }
     }
   ]
 }

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/BenchmarkConfig.cs

@@ -123,6 +123,18 @@ public class BenchmarkConfig
         [Option(Required = false, HelpText = "Container to publish results to")]
         public string ResultsContainer { get; set; } = "runsummary";
 
+        [Option(Required = false, HelpText = "Metrics reporting interval in seconds")]
+        public int MetricsReportingIntervalInSec { get; set; } = 5;
+
+        [Option(Required = false, HelpText = "Application Insights instrumentation key")]
+        public string AppInsightsInstrumentationKey { get; set; }
+
+        [Option(Required = false, HelpText = "The reservoir sample size.")]
+        public int ReservoirSampleSize { get; set; } = 1028;
+
+        [Option(Required = false, HelpText = "Logging context name. The default value is \"CosmosDBBenchmarkLoggingContext\"")]
+        public string LoggingContextIdentifier { get; set; } = "CosmosDBBenchmarkLoggingContext";
+
         internal int GetTaskCount(int containerThroughput)
         {
             int taskCount = this.DegreeOfParallelism;
@@ -270,4 +282,4 @@ private static void HandleParseError(IEnumerable<Error> errors)
             Environment.Exit(errors.Count());
         }
     }
-}
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/CosmosBenchmark.csproj

@@ -17,9 +17,15 @@
     </None>
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.0.0-beta.12" />
     <PackageReference Include="CommandLineParser" Version="2.8.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
+    <PackageReference Include="Microsoft.ApplicationInsights" Version="2.21.0" />
     <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="*" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
+    <PackageReference Include="OpenTelemetry.Exporter.Console" Version="1.5.0" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(ProjectRef)' != 'True' ">

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/BenchmarkOperation.cs

@@ -0,0 +1,15 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using System.Threading.Tasks;
+
+    internal abstract class BenchmarkOperation : IBenchmarkOperation
+    {
+        public abstract Task<OperationResult> ExecuteOnceAsync();
+
+        public abstract Task PrepareAsync();
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutionStrategy.cs

@@ -6,6 +6,9 @@ namespace CosmosBenchmark
 {
     using System;
     using System.Threading.Tasks;
+    using Microsoft.ApplicationInsights;
+    using Microsoft.Extensions.Logging;
+    using OpenTelemetry.Metrics;
 
     internal interface IExecutionStrategy
     {
@@ -19,7 +22,8 @@ public Task<RunSummary> ExecuteAsync(
             BenchmarkConfig benchmarkConfig,
             int serialExecutorConcurrency,
             int serialExecutorIterationCount,
-            double warmupFraction);
-
+            double warmupFraction,
+            ILogger logger,
+            MeterProvider meterProvider);
     }
 }

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IExecutor.cs

@@ -6,6 +6,8 @@ namespace CosmosBenchmark
 {
     using System;
     using System.Threading.Tasks;
+    using Microsoft.Extensions.Logging;
+    using OpenTelemetry.Metrics;
 
     internal interface IExecutor
     {
@@ -17,6 +19,9 @@ public Task ExecuteAsync(
                 int iterationCount,
                 bool isWarmup,
                 bool traceFailures,
-                Action completionCallback);
+                Action completionCallback,
+                BenchmarkConfig benchmarkConfig,
+                ILogger logger,
+                MeterProvider meterProvider);
     }
 }

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/IMetricsCollector.cs

@@ -0,0 +1,15 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    public interface IMetricsCollector
+    {
+        void RecordLatencyAndRps(double milliseconds);
+
+        void CollectMetricsOnSuccess();
+
+        void CollectMetricsOnFailure();
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertBenchmarkOperation.cs

@@ -0,0 +1,10 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    internal abstract class InsertBenchmarkOperation : BenchmarkOperation
+    {
+    }
+}

Microsoft.Azure.Cosmos.Samples/Tools/Benchmark/Fx/InsertOperationMetricsCollector.cs

@@ -0,0 +1,28 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace CosmosBenchmark
+{
+    using Microsoft.ApplicationInsights;
+    using System.Diagnostics.Metrics;
+
+    internal class InsertOperationMetricsCollector : MetricsCollector
+    {
+        public InsertOperationMetricsCollector(Meter meter) : base(meter)
+        {
+        }
+
+        protected override string RpsHistogramName => "InsertOperationRpsHistogram";
+
+        protected override string LatencyInMsHistogramName => "InsertOperationLatencyInMsHistogram";
+
+        protected override string RpsMetricName => "InsertOperationRps";
+
+        protected override string LatencyInMsMetricName => "InsertOperationLatencyInMs";
+
+        protected override string FailureOperationMetricName => "InsertOperationFailure";
+
+        protected override string SuccessOperationMetricName => "InsertOperationSuccess";
+    }
+}