From ac149836b01cfa0a4237b4d3c589f7a6f51a8113 Mon Sep 17 00:00:00 2001 From: Matt Warren Date: Thu, 14 Nov 2024 13:52:53 +0000 Subject: [PATCH 1/3] Expose "IsLeader" in Telemetry metrics and /metrics endpoint --- .../Controllers/MetricsController.cs | 18 +++++++++++++++-- .../Core/Telemetry/DefaultTagsFactory.cs | 17 ++++++++++------ .../Services/Metrics/StatusReportGenerator.cs | 20 +++++++++++++++++-- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/src/Contrast.K8s.AgentOperator/Controllers/MetricsController.cs b/src/Contrast.K8s.AgentOperator/Controllers/MetricsController.cs index a8dc3a42..abf164f7 100644 --- a/src/Contrast.K8s.AgentOperator/Controllers/MetricsController.cs +++ b/src/Contrast.K8s.AgentOperator/Controllers/MetricsController.cs @@ -1,6 +1,7 @@ // Contrast Security, Inc licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Contrast.K8s.AgentOperator.Core.Telemetry; @@ -25,10 +26,23 @@ public MetricsController(StatusReportGenerator reportGenerator, DefaultTagsFacto public async Task GetMetrics(CancellationToken cancellationToken = default) { var report = await _reportGenerator.Generate(cancellationToken); - return Ok(report.Values); + + // Combine the Values and Tags from the report into a single output + var output = new Dictionary(); + + foreach (var value in report.Values) + { + output.Add(value.Key, value.Value); + } + + foreach (var tag in report.ExtraTags) + { + output.Add(tag.Key, tag.Value); + } + + return Ok(output); } - [HttpGet("tags")] public async Task GetTags(CancellationToken cancellationToken = default) { diff --git a/src/Contrast.K8s.AgentOperator/Core/Telemetry/DefaultTagsFactory.cs b/src/Contrast.K8s.AgentOperator/Core/Telemetry/DefaultTagsFactory.cs index ee8cefee..fdc97f87 100644 --- a/src/Contrast.K8s.AgentOperator/Core/Telemetry/DefaultTagsFactory.cs +++ b/src/Contrast.K8s.AgentOperator/Core/Telemetry/DefaultTagsFactory.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Contrast.K8s.AgentOperator.Core.Leading; using Contrast.K8s.AgentOperator.Core.Telemetry.Getters; using Contrast.K8s.AgentOperator.Options; @@ -18,28 +19,37 @@ public class DefaultTagsFactory private readonly TelemetryOptions _telemetryOptions; private readonly MachineIdGetter _machineIdGetter; private readonly K8sClusterGetter _cluster; + private readonly ILeaderElectionState _leaderElectionState; public DefaultTagsFactory(IsPublicTelemetryBuildGetter isPublicTelemetryBuildGetter, TelemetryState telemetryState, TelemetryOptions telemetryOptions, MachineIdGetter machineIdGetter, - K8sClusterGetter cluster) + K8sClusterGetter cluster, + ILeaderElectionState leaderElectionState) { _isPublicTelemetryBuildGetter = isPublicTelemetryBuildGetter; _telemetryState = telemetryState; _telemetryOptions = telemetryOptions; _machineIdGetter = machineIdGetter; _cluster = cluster; + _leaderElectionState = leaderElectionState; } public async Task> GetDefaultTags() { var defaultTags = new Dictionary(StringComparer.OrdinalIgnoreCase) { + { "IsLeader", _leaderElectionState.IsLeader().ToString() }, { "Operator.IsPublicBuild", _isPublicTelemetryBuildGetter.IsPublicBuild().ToString() }, { "Operator.Version", _telemetryState.OperatorVersion } }; + if (!string.IsNullOrWhiteSpace(_telemetryOptions.InstallSource)) + { + defaultTags.Add("Operator.InstallSource", _telemetryOptions.InstallSource); + } + if (await _cluster.GetClusterInfo() is { } clusterInfo) { defaultTags.Add("Cluster.BuildDate", clusterInfo.BuildDate); @@ -53,11 +63,6 @@ public async Task> GetDefaultTags() defaultTags.Add("Cluster.Platform", clusterInfo.Platform); } - if (!string.IsNullOrWhiteSpace(_telemetryOptions.InstallSource)) - { - defaultTags.Add("Operator.InstallSource", _telemetryOptions.InstallSource); - } - return defaultTags; } diff --git a/src/Contrast.K8s.AgentOperator/Core/Telemetry/Services/Metrics/StatusReportGenerator.cs b/src/Contrast.K8s.AgentOperator/Core/Telemetry/Services/Metrics/StatusReportGenerator.cs index 364ed51f..528daca7 100644 --- a/src/Contrast.K8s.AgentOperator/Core/Telemetry/Services/Metrics/StatusReportGenerator.cs +++ b/src/Contrast.K8s.AgentOperator/Core/Telemetry/Services/Metrics/StatusReportGenerator.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Contrast.K8s.AgentOperator.Core.Leading; using Contrast.K8s.AgentOperator.Core.State; using Contrast.K8s.AgentOperator.Core.State.Resources; using Contrast.K8s.AgentOperator.Core.Telemetry.Counters; @@ -17,12 +18,18 @@ public class StatusReportGenerator { private readonly TelemetryState _telemetryState; private readonly IStateContainer _clusterState; + private readonly ILeaderElectionState _leaderElectionState; private readonly PerformanceCounterContainer _performanceCounterContainer; - public StatusReportGenerator(TelemetryState telemetryState, IStateContainer clusterState, PerformanceCounterContainer performanceCounterContainer) + public StatusReportGenerator( + TelemetryState telemetryState, + IStateContainer clusterState, + ILeaderElectionState leaderElectionState, + PerformanceCounterContainer performanceCounterContainer) { _telemetryState = telemetryState; _clusterState = clusterState; + _leaderElectionState = leaderElectionState; _performanceCounterContainer = performanceCounterContainer; } @@ -52,7 +59,8 @@ public async Task Generate(CancellationToken cancellationT return new TelemetryMeasurement("status-report") { - Values = values + Values = values, + ExtraTags = GetExtraTags() }; } @@ -119,4 +127,12 @@ private async Task> GetPerformanceStatistics() return metrics; } + + private Dictionary GetExtraTags() + { + return new Dictionary + { + { "IsLeader", _leaderElectionState.IsLeader().ToString() } + }; + } } From 97a9c7580c5239066f3e299e24acf3d1ba2097a1 Mon Sep 17 00:00:00 2001 From: Matt Warren Date: Fri, 15 Nov 2024 12:33:41 +0000 Subject: [PATCH 2/3] Change metrics tag name from "IsLeader" -> "Operator.IsLeader" --- .../Core/Telemetry/DefaultTagsFactory.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Contrast.K8s.AgentOperator/Core/Telemetry/DefaultTagsFactory.cs b/src/Contrast.K8s.AgentOperator/Core/Telemetry/DefaultTagsFactory.cs index fdc97f87..ce870d83 100644 --- a/src/Contrast.K8s.AgentOperator/Core/Telemetry/DefaultTagsFactory.cs +++ b/src/Contrast.K8s.AgentOperator/Core/Telemetry/DefaultTagsFactory.cs @@ -40,9 +40,9 @@ public async Task> GetDefaultTags() { var defaultTags = new Dictionary(StringComparer.OrdinalIgnoreCase) { - { "IsLeader", _leaderElectionState.IsLeader().ToString() }, { "Operator.IsPublicBuild", _isPublicTelemetryBuildGetter.IsPublicBuild().ToString() }, - { "Operator.Version", _telemetryState.OperatorVersion } + { "Operator.Version", _telemetryState.OperatorVersion }, + { "Operator.IsLeader", _leaderElectionState.IsLeader().ToString() } }; if (!string.IsNullOrWhiteSpace(_telemetryOptions.InstallSource)) From 47dab05e15787416f62a2c339ee18e165486113e Mon Sep 17 00:00:00 2001 From: Matt Warren Date: Fri, 15 Nov 2024 12:35:04 +0000 Subject: [PATCH 3/3] Add details statistics about the Process to /metrics endpoint --- .../Controllers/MetricsController.cs | 88 ++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/src/Contrast.K8s.AgentOperator/Controllers/MetricsController.cs b/src/Contrast.K8s.AgentOperator/Controllers/MetricsController.cs index abf164f7..83bb9348 100644 --- a/src/Contrast.K8s.AgentOperator/Controllers/MetricsController.cs +++ b/src/Contrast.K8s.AgentOperator/Controllers/MetricsController.cs @@ -1,18 +1,24 @@ // Contrast Security, Inc licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Contrast.K8s.AgentOperator.Core.Telemetry; using Contrast.K8s.AgentOperator.Core.Telemetry.Services.Metrics; using Microsoft.AspNetCore.Mvc; +using NLog; namespace Contrast.K8s.AgentOperator.Controllers; [ApiController, Route("api/v1/metrics")] public class MetricsController : Controller { + private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); + private readonly StatusReportGenerator _reportGenerator; private readonly DefaultTagsFactory _defaultTagsFactory; @@ -30,12 +36,17 @@ public async Task GetMetrics(CancellationToken cancellationToken // Combine the Values and Tags from the report into a single output var output = new Dictionary(); - foreach (var value in report.Values) + foreach (var value in report.Values.OrderBy(r => r.Key)) { output.Add(value.Key, value.Value); } - foreach (var tag in report.ExtraTags) + foreach (var stat in GenerateProcessingStatistics()) + { + output.Add(stat.Key, stat.Value); + } + + foreach (var tag in report.ExtraTags.OrderBy(r => r.Key)) { output.Add(tag.Key, tag.Value); } @@ -49,4 +60,77 @@ public async Task GetTags(CancellationToken cancellationToken = d var tags = await _defaultTagsFactory.GetDefaultTags(); return Ok(tags); } + + private IDictionary GenerateProcessingStatistics() + { + var statistics = new Dictionary(); + + try + { + var currentProcess = Process.GetCurrentProcess(); + + // Do them in groups because they can all throw InvalidOperationException/NotSupportedException + // Makes the code messier, but it means we'll still get as many as we can if some don't work + try + { + statistics.Add("Process.WorkingSet64", currentProcess.WorkingSet64); + statistics.Add("Process.MinWorkingSet", currentProcess.MinWorkingSet.ToInt64()); + statistics.Add("Process.MaxWorkingSet", currentProcess.MaxWorkingSet.ToInt64()); + statistics.Add("Process.PeakWorkingSet64", currentProcess.PeakWorkingSet64); + } + catch (Exception ex) + { + Logger.Warn(ex); + } + + try + { + statistics.Add("Process.PrivateMemorySize64", currentProcess.PrivateMemorySize64); + statistics.Add("Process.VirtualMemorySize64", currentProcess.VirtualMemorySize64); + statistics.Add("Process.PeakVirtualMemorySize64", currentProcess.PeakVirtualMemorySize64); + } + catch (Exception ex) + { + Logger.Warn(ex); + } + + try + { + statistics.Add("Process.PagedMemorySize64", currentProcess.PagedMemorySize64); + statistics.Add("Process.PeakPagedMemorySize64", currentProcess.PeakPagedMemorySize64); + statistics.Add("Process.NonpagedSystemMemorySize64", currentProcess.NonpagedSystemMemorySize64); + } + catch (Exception ex) + { + Logger.Warn(ex); + } + + try + { + statistics.Add("Process.TotalProcessorTime", currentProcess.TotalProcessorTime); + statistics.Add("Process.UserProcessorTime", currentProcess.UserProcessorTime); + statistics.Add("Process.PrivilegedProcessorTime", currentProcess.PrivilegedProcessorTime); + } + catch (Exception ex) + { + Logger.Warn(ex); + } + + try + { + statistics.Add("Process.Thread", currentProcess.Threads.Count); + statistics.Add("Process.Modules", currentProcess.Modules.Count); + } + catch (Exception ex) + { + Logger.Warn(ex); + } + } + catch (Exception ex) + { + Logger.Warn(ex); + } + + return statistics; + } }