DOTNET-5968 Add extra information to /metrics endpoint (SUP-5887)

src/Contrast.K8s.AgentOperator/Controllers/MetricsController.cs

@@ -1,17 +1,24 @@
 // Contrast Security, Inc licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Contrast.K8s.AgentOperator.Core.Telemetry;
 using Contrast.K8s.AgentOperator.Core.Telemetry.Services.Metrics;
 using Microsoft.AspNetCore.Mvc;
+using NLog;
 
 namespace Contrast.K8s.AgentOperator.Controllers;
 
 [ApiController, Route("api/v1/metrics")]
 public class MetricsController : Controller
 {
+    private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
+
     private readonly StatusReportGenerator _reportGenerator;
     private readonly DefaultTagsFactory _defaultTagsFactory;
 
@@ -25,14 +32,105 @@ public MetricsController(StatusReportGenerator reportGenerator, DefaultTagsFacto
     public async Task<IActionResult> GetMetrics(CancellationToken cancellationToken = default)
     {
         var report = await _reportGenerator.Generate(cancellationToken);
-        return Ok(report.Values);
+
+        // Combine the Values and Tags from the report into a single output
+        var output = new Dictionary<string, object>();
+
+        foreach (var value in report.Values.OrderBy(r => r.Key))
+        {
+            output.Add(value.Key, value.Value);
+        }
+
+        foreach (var stat in GenerateProcessingStatistics())
+        {
+            output.Add(stat.Key, stat.Value);
+        }
+
+        foreach (var tag in report.ExtraTags.OrderBy(r => r.Key))
+        {
+            output.Add(tag.Key, tag.Value);
+        }
+
+        return Ok(output);
     }
 
-
     [HttpGet("tags")]
     public async Task<IActionResult> GetTags(CancellationToken cancellationToken = default)
     {
         var tags = await _defaultTagsFactory.GetDefaultTags();
         return Ok(tags);
     }
+
+    private IDictionary<string, object> GenerateProcessingStatistics()
+    {
+        var statistics = new Dictionary<string, object>();
+
+        try
+        {
+            var currentProcess = Process.GetCurrentProcess();
+
+            // Do them in groups because they can all throw InvalidOperationException/NotSupportedException
+            // Makes the code messier, but it means we'll still get as many as we can if some don't work
+            try
+            {
+                statistics.Add("Process.WorkingSet64", currentProcess.WorkingSet64);
+                statistics.Add("Process.MinWorkingSet", currentProcess.MinWorkingSet.ToInt64());
+                statistics.Add("Process.MaxWorkingSet", currentProcess.MaxWorkingSet.ToInt64());
+                statistics.Add("Process.PeakWorkingSet64", currentProcess.PeakWorkingSet64);
+            }
+            catch (Exception ex)
+            {
+                Logger.Warn(ex);
+            }
+
+            try
+            {
+                statistics.Add("Process.PrivateMemorySize64", currentProcess.PrivateMemorySize64);
+                statistics.Add("Process.VirtualMemorySize64", currentProcess.VirtualMemorySize64);
+                statistics.Add("Process.PeakVirtualMemorySize64", currentProcess.PeakVirtualMemorySize64);
+            }
+            catch (Exception ex)
+            {
+                Logger.Warn(ex);
+            }
+
+            try
+            {
+                statistics.Add("Process.PagedMemorySize64", currentProcess.PagedMemorySize64);
+                statistics.Add("Process.PeakPagedMemorySize64", currentProcess.PeakPagedMemorySize64);
+                statistics.Add("Process.NonpagedSystemMemorySize64", currentProcess.NonpagedSystemMemorySize64);
+            }
+            catch (Exception ex)
+            {
+                Logger.Warn(ex);
+            }
+
+            try
+            {
+                statistics.Add("Process.TotalProcessorTime", currentProcess.TotalProcessorTime);
+                statistics.Add("Process.UserProcessorTime", currentProcess.UserProcessorTime);
+                statistics.Add("Process.PrivilegedProcessorTime", currentProcess.PrivilegedProcessorTime);
+            }
+            catch (Exception ex)
+            {
+                Logger.Warn(ex);
+            }
+
+            try
+            {
+                statistics.Add("Process.Thread", currentProcess.Threads.Count);
+                statistics.Add("Process.Modules", currentProcess.Modules.Count);
+            }
+            catch (Exception ex)
+            {
+                Logger.Warn(ex);
+            }
+        }
+        catch (Exception ex)
+        {
+            Logger.Warn(ex);
+        }
+
+        return statistics;
+    }
 }

src/Contrast.K8s.AgentOperator/Core/Telemetry/DefaultTagsFactory.cs

@@ -6,6 +6,7 @@
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using Contrast.K8s.AgentOperator.Core.Leading;
 using Contrast.K8s.AgentOperator.Core.Telemetry.Getters;
 using Contrast.K8s.AgentOperator.Options;
 
@@ -18,28 +19,37 @@ public class DefaultTagsFactory
     private readonly TelemetryOptions _telemetryOptions;
     private readonly MachineIdGetter _machineIdGetter;
     private readonly K8sClusterGetter _cluster;
+    private readonly ILeaderElectionState _leaderElectionState;
 
     public DefaultTagsFactory(IsPublicTelemetryBuildGetter isPublicTelemetryBuildGetter,
                               TelemetryState telemetryState,
                               TelemetryOptions telemetryOptions,
                               MachineIdGetter machineIdGetter,
-                              K8sClusterGetter cluster)
+                              K8sClusterGetter cluster,
+                              ILeaderElectionState leaderElectionState)
     {
         _isPublicTelemetryBuildGetter = isPublicTelemetryBuildGetter;
         _telemetryState = telemetryState;
         _telemetryOptions = telemetryOptions;
         _machineIdGetter = machineIdGetter;
         _cluster = cluster;
+        _leaderElectionState = leaderElectionState;
     }
 
     public async Task<IReadOnlyDictionary<string, string>> GetDefaultTags()
     {
         var defaultTags = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
         {
             { "Operator.IsPublicBuild", _isPublicTelemetryBuildGetter.IsPublicBuild().ToString() },
-            { "Operator.Version", _telemetryState.OperatorVersion }
+            { "Operator.Version", _telemetryState.OperatorVersion },
+            { "Operator.IsLeader", _leaderElectionState.IsLeader().ToString() }
         };
 
+        if (!string.IsNullOrWhiteSpace(_telemetryOptions.InstallSource))
+        {
+            defaultTags.Add("Operator.InstallSource", _telemetryOptions.InstallSource);
+        }
+
         if (await _cluster.GetClusterInfo() is { } clusterInfo)
         {
             defaultTags.Add("Cluster.BuildDate", clusterInfo.BuildDate);
@@ -53,11 +63,6 @@ public async Task<IReadOnlyDictionary<string, string>> GetDefaultTags()
             defaultTags.Add("Cluster.Platform", clusterInfo.Platform);
         }
 
-        if (!string.IsNullOrWhiteSpace(_telemetryOptions.InstallSource))
-        {
-            defaultTags.Add("Operator.InstallSource", _telemetryOptions.InstallSource);
-        }
-
         return defaultTags;
     }
 

src/Contrast.K8s.AgentOperator/Core/Telemetry/Services/Metrics/StatusReportGenerator.cs

@@ -6,6 +6,7 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Contrast.K8s.AgentOperator.Core.Leading;
 using Contrast.K8s.AgentOperator.Core.State;
 using Contrast.K8s.AgentOperator.Core.State.Resources;
 using Contrast.K8s.AgentOperator.Core.Telemetry.Counters;
@@ -17,12 +18,18 @@ public class StatusReportGenerator
 {
     private readonly TelemetryState _telemetryState;
     private readonly IStateContainer _clusterState;
+    private readonly ILeaderElectionState _leaderElectionState;
     private readonly PerformanceCounterContainer _performanceCounterContainer;
 
-    public StatusReportGenerator(TelemetryState telemetryState, IStateContainer clusterState, PerformanceCounterContainer performanceCounterContainer)
+    public StatusReportGenerator(
+        TelemetryState telemetryState,
+        IStateContainer clusterState,
+        ILeaderElectionState leaderElectionState,
+        PerformanceCounterContainer performanceCounterContainer)
     {
         _telemetryState = telemetryState;
         _clusterState = clusterState;
+        _leaderElectionState = leaderElectionState;
         _performanceCounterContainer = performanceCounterContainer;
     }
 
@@ -52,7 +59,8 @@ public async Task<TelemetryMeasurement> Generate(CancellationToken cancellationT
 
         return new TelemetryMeasurement("status-report")
         {
-            Values = values
+            Values = values,
+            ExtraTags = GetExtraTags()
         };
     }
 
@@ -119,4 +127,12 @@ private async Task<Dictionary<string, decimal>> GetPerformanceStatistics()
 
         return metrics;
     }
+
+    private Dictionary<string, string> GetExtraTags()
+    {
+        return new Dictionary<string, string>
+        {
+            { "IsLeader", _leaderElectionState.IsLeader().ToString() }
+        };
+    }
 }