diff --git a/k8s/elastic-monitor.yaml b/k8s/elastic-monitor.yaml index d7054cb787..a5d95cdb5f 100644 --- a/k8s/elastic-monitor.yaml +++ b/k8s/elastic-monitor.yaml @@ -115,12 +115,12 @@ metadata: name: elastic-monitor namespace: elastic-system annotations: - kubernetes.io/ingress.class: "nginx" cert-manager.io/cluster-issuer: letsencrypt-prod nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" nginx.ingress.kubernetes.io/proxy-ssl-verify: "off" nginx.ingress.kubernetes.io/ssl-redirect: "true" spec: + ingressClassName: nginx tls: - hosts: - kibana.exceptionless.io diff --git a/k8s/ex-prod-values.yaml b/k8s/ex-prod-values.yaml index fe1a19e0a9..aadc468cc1 100644 --- a/k8s/ex-prod-values.yaml +++ b/k8s/ex-prod-values.yaml @@ -37,9 +37,6 @@ config: EX_TestEmailAddress: "test@exceptionless.io" EX_EnableArchive: "false" EX_Serilog__MinimumLevel__Default: "Warning" - EX_Apm__Endpoint: http://apm.elastic-system.svc:8200 + EX_OTEL_EXPORTER_OTLP_ENDPOINT: http://apm.elastic-system.svc:8200 EX_Apm__EnableLogs: "true" - EX_Apm__EnableMetrics: "true" - EX_Apm__EnableTracing: "true" EX_Apm__FullDetails: "true" - EX_Apm__Insecure: "true" diff --git a/src/Exceptionless.Job/Exceptionless.Job.csproj b/src/Exceptionless.Job/Exceptionless.Job.csproj index e4c3e34404..ebe2c855f1 100644 --- a/src/Exceptionless.Job/Exceptionless.Job.csproj +++ b/src/Exceptionless.Job/Exceptionless.Job.csproj @@ -20,6 +20,7 @@ + diff --git a/src/Exceptionless.Job/Program.cs b/src/Exceptionless.Job/Program.cs index b34451884e..c024d3f389 100644 --- a/src/Exceptionless.Job/Program.cs +++ b/src/Exceptionless.Job/Program.cs @@ -100,8 +100,7 @@ public static IHostBuilder CreateHostBuilder(string[] args) if (!String.IsNullOrEmpty(options.ExceptionlessApiKey) && !String.IsNullOrEmpty(options.ExceptionlessServerUrl)) app.UseExceptionless(ExceptionlessClient.Default); - if (apmConfig.EnableMetrics) - app.UseOpenTelemetryPrometheusScrapingEndpoint(); + app.UseOpenTelemetryPrometheusScrapingEndpoint(); app.UseHealthChecks("/health", new HealthCheckOptions { diff --git a/src/Exceptionless.Job/appsettings.Development.yml b/src/Exceptionless.Job/appsettings.Development.yml index e40525e6ee..063d606a2a 100644 --- a/src/Exceptionless.Job/appsettings.Development.yml +++ b/src/Exceptionless.Job/appsettings.Development.yml @@ -16,11 +16,7 @@ Serilog: Default: Debug Apm: - #Endpoint: http://host.docker.internal:8200 - Insecure: true EnableLogs: false - EnableTracing: false - EnableMetrics: true FullDetails: true Debug: false Console: false diff --git a/src/Exceptionless.Job/appsettings.Staging.yml b/src/Exceptionless.Job/appsettings.Staging.yml index 8de325be7b..c26bf33eec 100644 --- a/src/Exceptionless.Job/appsettings.Staging.yml +++ b/src/Exceptionless.Job/appsettings.Staging.yml @@ -20,7 +20,7 @@ Serilog: Default: Warning WriteTo: - Name: Console - Args: + Args: theme: "Serilog.Sinks.SystemConsole.Themes.ConsoleTheme::None, Serilog.Sinks.Console" Apm: diff --git a/src/Exceptionless.Job/appsettings.yml b/src/Exceptionless.Job/appsettings.yml index 4bf58cab87..af8a4d786c 100644 --- a/src/Exceptionless.Job/appsettings.yml +++ b/src/Exceptionless.Job/appsettings.yml @@ -17,7 +17,7 @@ Serilog: Foundatio: Information WriteTo: - Name: Console - Args: + Args: theme: "Serilog.Sinks.SystemConsole.Themes.AnsiConsoleTheme::Literate, Serilog.Sinks.Console" Enrich: - FromLogContext @@ -27,8 +27,5 @@ Serilog: Apm: ServiceName: exceptionless EnableLogs: true - EnableTracing: true - EnableMetrics: true - SampleRate: 1.0 FullDetails: true - Debug: false \ No newline at end of file + Debug: false diff --git a/src/Exceptionless.Web/ApmExtensions.cs b/src/Exceptionless.Web/ApmExtensions.cs index 0df71ab72d..aa7b52b7ef 100644 --- a/src/Exceptionless.Web/ApmExtensions.cs +++ b/src/Exceptionless.Web/ApmExtensions.cs @@ -8,7 +8,6 @@ using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; -using Serilog; namespace OpenTelemetry; @@ -16,24 +15,10 @@ public static partial class ApmExtensions { public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) { - // check if everything is disabled - if (!config.IsEnabled) - { - Log.Information("APM is disabled"); - return builder; - } - - string apiKey = config.ApiKey; - if (!String.IsNullOrEmpty(apiKey) && apiKey.Length > 6) - apiKey = String.Concat(apiKey.AsSpan(0, 6), "***"); - - Log.Information("Configuring APM: Endpoint={Endpoint} ApiKey={ApiKey} EnableTracing={EnableTracing} EnableLogs={EnableLogs} FullDetails={FullDetails} EnableRedis={EnableRedis} SampleRate={SampleRate}", - config.Endpoint, apiKey, config.EnableTracing, config.EnableLogs, config.FullDetails, config.EnableRedis, config.SampleRate); - var attributes = new Dictionary() { { "service.namespace", config.ServiceNamespace }, - { "service.environment", config.ServiceEnvironment } + { "deployment.environment", config.DeploymentEnvironment } }; if (!String.IsNullOrEmpty(config.ServiceVersion)) @@ -45,90 +30,66 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) services.AddSingleton(config); services.AddHostedService(sp => new SelfDiagnosticsLoggingHostedService(sp.GetRequiredService(), config.Debug ? EventLevel.Verbose : null)); - if (config.EnableTracing) - services.AddOpenTelemetry().WithTracing(b => - { - b.SetResourceBuilder(resourceBuilder); + services.AddOpenTelemetry().WithTracing(b => + { + b.AddProcessor(); + b.SetResourceBuilder(resourceBuilder); - b.AddAspNetCoreInstrumentation(o => + b.AddAspNetCoreInstrumentation(o => + { + o.Filter = context => { - o.Filter = context => - { - return !context.Request.Headers.UserAgent.ToString().Contains("HealthChecker"); - }; - }); + if (context.Request.Path.StartsWithSegments("/api/v2/push", StringComparison.OrdinalIgnoreCase)) + return false; - b.AddElasticsearchClientInstrumentation(c => - { - c.SuppressDownstreamInstrumentation = true; - c.ParseAndFormatRequest = config.FullDetails; - c.Enrich = (activity, source, data) => - { - // truncate statements - if (activity.GetTagItem("db.statement") is string dbStatement && dbStatement.Length > 10000) - { - dbStatement = _stackIdListShortener.Replace(dbStatement, "$1...]"); - if (dbStatement.Length > 10000) - dbStatement = dbStatement.Substring(0, 10000); - - activity.SetTag("db.statement", dbStatement); - } - - // 404s should not be error - int? httpStatus = activity.GetTagItem("http.status_code") as int?; - if (httpStatus.HasValue && httpStatus.Value == 404) - activity.SetStatus(Status.Unset); - }; - }); + if (context.Request.Headers.UserAgent.ToString().Contains("HealthChecker")) + return false; - b.AddHttpClientInstrumentation(); - b.AddSource("Exceptionless", "Foundatio"); + return true; + }; + }); - if (config.EnableRedis) - b.AddRedisInstrumentation(c => + b.AddElasticsearchClientInstrumentation(c => + { + c.SuppressDownstreamInstrumentation = true; + c.ParseAndFormatRequest = config.FullDetails; + c.Enrich = (activity, source, data) => + { + // truncate statements + if (activity.GetTagItem("db.statement") is string dbStatement && dbStatement.Length > 10000) { - c.EnrichActivityWithTimingEvents = false; - c.SetVerboseDatabaseStatements = config.FullDetails; - }); + dbStatement = _stackIdListShortener.Replace(dbStatement, "$1...]"); + if (dbStatement.Length > 10000) + dbStatement = dbStatement.Substring(0, 10000); - b.SetSampler(new TraceIdRatioBasedSampler(config.SampleRate)); + activity.SetTag("db.statement", dbStatement); + } - if (config.Console) - b.AddConsoleExporter(); + // 404s should not be error + int? httpStatus = activity.GetTagItem("http.status_code") as int?; + if (httpStatus.HasValue && httpStatus.Value == 404) + activity.SetStatus(Status.Unset); + }; + }); - if (!String.IsNullOrEmpty(config.Endpoint)) + b.AddHttpClientInstrumentation(); + b.AddSource("Exceptionless", "Foundatio"); + + if (config.EnableRedis) + b.AddRedisInstrumentation(c => { - if (config.MinDurationMs > 0) - { - // filter out insignificant activities - b.AddFilteredOtlpExporter(c => - { - if (config.Insecure || !String.IsNullOrEmpty(config.SslThumbprint)) - c.Protocol = OtlpExportProtocol.HttpProtobuf; - - if (!String.IsNullOrEmpty(config.Endpoint)) - c.Endpoint = new Uri(config.Endpoint); - if (!String.IsNullOrEmpty(config.ApiKey)) - c.Headers = $"api-key={config.ApiKey}"; - - c.Filter = a => a.Duration > TimeSpan.FromMilliseconds(config.MinDurationMs) || a.GetTagItem("db.system") is not null; - }); - } - else - { - b.AddOtlpExporter(c => - { - if (config.Insecure || !String.IsNullOrEmpty(config.SslThumbprint)) - c.Protocol = OtlpExportProtocol.HttpProtobuf; - - if (!String.IsNullOrEmpty(config.Endpoint)) - c.Endpoint = new Uri(config.Endpoint); - if (!String.IsNullOrEmpty(config.ApiKey)) - c.Headers = $"api-key={config.ApiKey}"; - }); - } - } + c.EnrichActivityWithTimingEvents = false; + c.SetVerboseDatabaseStatements = config.FullDetails; + }); + + if (config.Console) + b.AddConsoleExporter(); + + b.AddFilteredOtlpExporter(c => + { + c.Filter = a => a.Duration > TimeSpan.FromMilliseconds(config.MinDurationMs) || a.GetTagItem("db.system") is not null; }); + }); services.AddOpenTelemetry().WithMetrics(b => { @@ -138,6 +99,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) b.AddAspNetCoreInstrumentation(); b.AddMeter("Exceptionless", "Foundatio"); b.AddRuntimeInstrumentation(); + b.AddProcessInstrumentation(); if (config.Console) b.AddConsoleExporter((_, metricReaderOptions) => @@ -148,21 +110,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) }); b.AddPrometheusExporter(); - - if (!String.IsNullOrEmpty(config.Endpoint)) - b.AddOtlpExporter((c, o) => - { - if (config.Insecure || !String.IsNullOrEmpty(config.SslThumbprint)) - c.Protocol = OtlpExportProtocol.HttpProtobuf; - - // needed for newrelic compatibility until they support cumulative - o.TemporalityPreference = MetricReaderTemporalityPreference.Delta; - - if (!String.IsNullOrEmpty(config.Endpoint)) - c.Endpoint = new Uri(config.Endpoint); - if (!String.IsNullOrEmpty(config.ApiKey)) - c.Headers = $"api-key={config.ApiKey}"; - }); + b.AddOtlpExporter(); }); }); @@ -180,19 +128,7 @@ public static IHostBuilder AddApm(this IHostBuilder builder, ApmConfig config) if (config.Console) o.AddConsoleExporter(); - if (!String.IsNullOrEmpty(config.Endpoint)) - { - o.AddOtlpExporter(c => - { - if (config.Insecure || !String.IsNullOrEmpty(config.SslThumbprint)) - c.Protocol = OtlpExportProtocol.HttpProtobuf; - - if (!String.IsNullOrEmpty(config.Endpoint)) - c.Endpoint = new Uri(config.Endpoint); - if (!String.IsNullOrEmpty(config.ApiKey)) - c.Headers = $"api-key={config.ApiKey}"; - }); - } + o.AddOtlpExporter(); }); }); } @@ -219,28 +155,19 @@ public ApmConfig(IConfigurationRoot config, string processName, string? serviceV if (ServiceName.StartsWith('-')) ServiceName = ServiceName.Substring(1); - ServiceEnvironment = _apmConfig.GetValue("ServiceEnvironment", "") ?? throw new InvalidOperationException(); + DeploymentEnvironment = _apmConfig.GetValue("ServiceEnvironment", "dev") ?? throw new InvalidOperationException(); ServiceNamespace = _apmConfig.GetValue("ServiceNamespace", ServiceName) ?? throw new InvalidOperationException(); ServiceVersion = serviceVersion; EnableRedis = enableRedis; } - public bool IsEnabled => EnableLogs || EnableMetrics || EnableTracing; - public bool EnableLogs => _apmConfig.GetValue("EnableLogs", false); - public bool EnableMetrics => _apmConfig.GetValue("EnableMetrics", true); - public bool EnableTracing => _apmConfig.GetValue("EnableTracing", _apmConfig.GetValue("Enabled", false)); - public bool Insecure => _apmConfig.GetValue("Insecure", false); - public string SslThumbprint => _apmConfig.GetValue("SslThumbprint", String.Empty) ?? throw new InvalidOperationException(); public string ServiceName { get; } - public string ServiceEnvironment { get; } + public string DeploymentEnvironment { get; } public string ServiceNamespace { get; } public string? ServiceVersion { get; } - public string Endpoint => _apmConfig.GetValue("Endpoint", String.Empty) ?? throw new InvalidOperationException(); - public string ApiKey => _apmConfig.GetValue("ApiKey", String.Empty) ?? throw new InvalidOperationException(); public bool FullDetails => _apmConfig.GetValue("FullDetails", false); - public double SampleRate => _apmConfig.GetValue("SampleRate", 1.0); - public int MinDurationMs => _apmConfig.GetValue("MinDurationMs", -1); + public int MinDurationMs => _apmConfig.GetValue("MinDurationMs", -1); public bool EnableRedis { get; } public bool Debug => _apmConfig.GetValue("Debug", false); public bool Console => _apmConfig.GetValue("Console", false); @@ -329,3 +256,131 @@ public class FilteredOtlpExporterOptions : OtlpExporterOptions { public Func? Filter { get; set; } } + +public class ElasticCompatibilityProcessor : BaseProcessor +{ + private readonly AsyncLocal _currentTransactionId = new(); + public const string TransactionIdTagName = "transaction.id"; + + public override void OnEnd(Activity activity) + { + if (activity.Parent == null) + _currentTransactionId.Value = activity.SpanId; + + if (_currentTransactionId.Value.HasValue) + activity.SetTag(TransactionIdTagName, _currentTransactionId.Value.Value.ToString()); + + if (activity.Kind == ActivityKind.Server) + { + string? httpScheme = null; + string? httpTarget = null; + string? urlScheme = null; + string? urlPath = null; + string? urlQuery = null; + string? netHostName = null; + int? netHostPort = null; + string? serverAddress = null; + int? serverPort = null; + + foreach (var tag in activity.TagObjects) + { + if (tag.Key == TraceSemanticConventions.HttpScheme) + httpScheme = ProcessStringAttribute(tag); + + if (tag.Key == TraceSemanticConventions.HttpTarget) + httpTarget = ProcessStringAttribute(tag); + + if (tag.Key == TraceSemanticConventions.UrlScheme) + urlScheme = ProcessStringAttribute(tag); + + if (tag.Key == TraceSemanticConventions.UrlPath) + urlPath = ProcessStringAttribute(tag); + + if (tag.Key == TraceSemanticConventions.UrlQuery) + urlQuery = ProcessStringAttribute(tag); + + if (tag.Key == TraceSemanticConventions.NetHostName) + netHostName = ProcessStringAttribute(tag); + + if (tag.Key == TraceSemanticConventions.ServerAddress) + serverAddress = ProcessStringAttribute(tag); + + if (tag.Key == TraceSemanticConventions.NetHostPort) + netHostPort = ProcessIntAttribute(tag); + + if (tag.Key == TraceSemanticConventions.ServerPort) + serverPort = ProcessIntAttribute(tag); + } + + // Set the older semantic convention attributes + if (httpScheme is null && urlScheme is not null) + SetStringAttribute(TraceSemanticConventions.HttpScheme, urlScheme); + + if (httpTarget is null && urlPath is not null) + { + var target = urlPath; + + if (urlQuery is not null) + target += $"?{urlQuery}"; + + SetStringAttribute(TraceSemanticConventions.HttpTarget, target); + } + + if (netHostName is null && serverAddress is not null) + SetStringAttribute(TraceSemanticConventions.NetHostName, serverAddress); + + if (netHostPort is null && serverPort is not null) + SetIntAttribute(TraceSemanticConventions.NetHostPort, serverPort.Value); + } + + string? ProcessStringAttribute(KeyValuePair tag) + { + if (tag.Value is string value) + { + return value; + } + + return null; + } + + int? ProcessIntAttribute(KeyValuePair tag) + { + if (tag.Value is int value) + { + return value; + } + + return null; + } + + void SetStringAttribute(string attributeName, string value) + { + activity.SetTag(attributeName, value); + } + + void SetIntAttribute(string attributeName, int value) + { + activity.SetTag(attributeName, value); + } + } +} + +internal static class TraceSemanticConventions +{ + // HTTP + public const string HttpScheme = "http.scheme"; + public const string HttpTarget = "http.target"; + + // NET + public const string NetHostName = "net.host.name"; + public const string NetHostPort = "net.host.port"; + + // SERVER + public const string ServerAddress = "server.address"; + public const string ServerPort = "server.port"; + + // URL + public const string UrlPath = "url.path"; + public const string UrlQuery = "url.query"; + public const string UrlScheme = "url.scheme"; +} diff --git a/src/Exceptionless.Web/Exceptionless.Web.csproj b/src/Exceptionless.Web/Exceptionless.Web.csproj index 505f5664e7..ee0dfa690e 100644 --- a/src/Exceptionless.Web/Exceptionless.Web.csproj +++ b/src/Exceptionless.Web/Exceptionless.Web.csproj @@ -38,6 +38,7 @@ + diff --git a/src/Exceptionless.Web/Properties/launchSettings.json b/src/Exceptionless.Web/Properties/launchSettings.json index 7e0d45f5ed..c660ee9846 100644 --- a/src/Exceptionless.Web/Properties/launchSettings.json +++ b/src/Exceptionless.Web/Properties/launchSettings.json @@ -6,7 +6,8 @@ "launchUrl": "http://localhost:5200/next", "environmentVariables": { "EX_AppMode": "Development", - "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy" + "ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "Microsoft.AspNetCore.SpaProxy", + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://localhost:8200" }, "dotnetRunMessages": true, "applicationUrl": "http://localhost:5200" diff --git a/src/Exceptionless.Web/Startup.cs b/src/Exceptionless.Web/Startup.cs index 1fa08bd3e1..5db3b5b756 100644 --- a/src/Exceptionless.Web/Startup.cs +++ b/src/Exceptionless.Web/Startup.cs @@ -183,9 +183,7 @@ public void Configure(IApplicationBuilder app) app.UseStatusCodePages(); app.UseMiddleware(); - var apmConfig = app.ApplicationServices.GetRequiredService(); - if (apmConfig.EnableMetrics) - app.UseOpenTelemetryPrometheusScrapingEndpoint(); + app.UseOpenTelemetryPrometheusScrapingEndpoint(); app.UseHealthChecks("/health", new HealthCheckOptions { diff --git a/src/Exceptionless.Web/appsettings.Development.yml b/src/Exceptionless.Web/appsettings.Development.yml index a42bc7fb07..421ea240b4 100644 --- a/src/Exceptionless.Web/appsettings.Development.yml +++ b/src/Exceptionless.Web/appsettings.Development.yml @@ -24,12 +24,7 @@ Serilog: Default: Debug Apm: - #Endpoint: http://localhost:4317 - Insecure: true - #SslThumbprint: CB16E1B3DFE42DF751F93A8575942DA89E10BC98 - EnableLogs: false - EnableTracing: false - EnableMetrics: true + EnableLogs: true FullDetails: true - Debug: false + Debug: true Console: false diff --git a/src/Exceptionless.Web/appsettings.Production.yml b/src/Exceptionless.Web/appsettings.Production.yml index e58fba62ed..a6bbd82962 100644 --- a/src/Exceptionless.Web/appsettings.Production.yml +++ b/src/Exceptionless.Web/appsettings.Production.yml @@ -24,7 +24,7 @@ Serilog: Default: Warning WriteTo: - Name: Console - Args: + Args: theme: "Serilog.Sinks.SystemConsole.Themes.ConsoleTheme::None, Serilog.Sinks.Console" Apm: diff --git a/src/Exceptionless.Web/appsettings.Staging.yml b/src/Exceptionless.Web/appsettings.Staging.yml index 0ccea244e6..3743bb48f2 100644 --- a/src/Exceptionless.Web/appsettings.Staging.yml +++ b/src/Exceptionless.Web/appsettings.Staging.yml @@ -20,7 +20,7 @@ Serilog: Default: Warning WriteTo: - Name: Console - Args: + Args: theme: "Serilog.Sinks.SystemConsole.Themes.ConsoleTheme::None, Serilog.Sinks.Console" Apm: diff --git a/src/Exceptionless.Web/appsettings.yml b/src/Exceptionless.Web/appsettings.yml index c3fa292291..c72fbad0a7 100644 --- a/src/Exceptionless.Web/appsettings.yml +++ b/src/Exceptionless.Web/appsettings.yml @@ -31,8 +31,5 @@ Serilog: Apm: ServiceName: exceptionless EnableLogs: true - EnableTracing: true - EnableMetrics: true - SampleRate: 1.0 FullDetails: true Debug: false