diff --git a/src/Serilog.Sinks.Loki.Example/LogLabelProvider.cs b/src/Serilog.Sinks.Loki.Example/LogLabelProvider.cs index c4b4d4b..d8cdac6 100644 --- a/src/Serilog.Sinks.Loki.Example/LogLabelProvider.cs +++ b/src/Serilog.Sinks.Loki.Example/LogLabelProvider.cs @@ -16,6 +16,7 @@ public IList GetLabels() public IList PropertiesAsLabels { get; set; } = new List { + "level", // Since 3.0.0, you need to explicitly add level if you want it! "MyLabelPropertyName" }; public IList PropertiesToAppend { get; set; } = new List diff --git a/src/Serilog.Sinks.Loki/Labels/DefaultLogLabelProvider.cs b/src/Serilog.Sinks.Loki/Labels/DefaultLogLabelProvider.cs index 9a97df4..d421060 100644 --- a/src/Serilog.Sinks.Loki/Labels/DefaultLogLabelProvider.cs +++ b/src/Serilog.Sinks.Loki/Labels/DefaultLogLabelProvider.cs @@ -3,7 +3,7 @@ namespace Serilog.Sinks.Loki.Labels { - class DefaultLogLabelProvider : ILogLabelProvider + public class DefaultLogLabelProvider : ILogLabelProvider { public DefaultLogLabelProvider() : this(null) { @@ -15,7 +15,7 @@ public DefaultLogLabelProvider(IEnumerable labels, LokiFormatterStrategy formatterStrategy = LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestAppended) { this.Labels = labels?.ToList() ?? new List(); - this.PropertiesAsLabels = propertiesAsLabels?.ToList() ?? new List(); + this.PropertiesAsLabels = propertiesAsLabels?.ToList() ?? new List {"level"}; this.PropertiesToAppend = propertiesToAppend?.ToList() ?? new List(); this.FormatterStrategy = formatterStrategy; } diff --git a/src/Serilog.Sinks.Loki/LokiBatchFormatter.cs b/src/Serilog.Sinks.Loki/LokiBatchFormatter.cs index c6922a8..0a5f214 100644 --- a/src/Serilog.Sinks.Loki/LokiBatchFormatter.cs +++ b/src/Serilog.Sinks.Loki/LokiBatchFormatter.cs @@ -32,26 +32,6 @@ public LokiBatchFormatter(IList globalLabels) this.LogLabelProvider = new DefaultLogLabelProvider(globalLabels); } - // This avoids additional quoting as described in https://github.com/serilog/serilog/issues/936 - private static void RenderMessage(TextWriter tw, LogEvent logEvent) - { - bool IsString(LogEventPropertyValue pv) - { - return pv is ScalarValue sv && sv.Value is string; - } - - foreach(var t in logEvent.MessageTemplate.Tokens) - { - if (t is PropertyToken pt && - logEvent.Properties.TryGetValue(pt.PropertyName, out var propVal) && - IsString(propVal)) - tw.Write(((ScalarValue)propVal).Value); - else - t.Render(logEvent.Properties, tw); - } - tw.Write('\n'); - } - public void Format(IEnumerable logEvents, ITextFormatter formatter, TextWriter output) { if (logEvents == null) @@ -63,47 +43,34 @@ public void Format(IEnumerable logEvents, ITextFormatter formatter, Te if (!logs.Any()) return; - var content = new LokiContent(); + var streamsDictionary = new Dictionary(); foreach (LogEvent logEvent in logs) { - var stream = new LokiContentStream(); - content.Streams.Add(stream); + var labels = new List(); - stream.Labels.Add(new LokiLabel("level", GetLevel(logEvent.Level))); foreach (LokiLabel globalLabel in this.LogLabelProvider.GetLabels()) - stream.Labels.Add(new LokiLabel(globalLabel.Key, globalLabel.Value)); + labels.Add(new LokiLabel(globalLabel.Key, globalLabel.Value)); var time = logEvent.Timestamp.ToString("o"); var sb = new StringBuilder(); using (var tw = new StringWriter(sb)) { - RenderMessage(tw, logEvent); + formatter.Format(logEvent, tw); } - if (logEvent.Exception != null) - // AggregateException adds a Environment.Newline to the end of ToString(), so we trim it off - sb.AppendLine(logEvent.Exception.ToString().TrimEnd()); + HandleProperty("level", GetLevel(logEvent.Level), labels, sb); foreach (KeyValuePair property in logEvent.Properties) { - // Some enrichers pass strings with quotes surrounding the values inside the string, - // which results in redundant quotes after serialization and a "bad request" response. - // To avoid this, remove all quotes from the value. - // We also remove any \r\n newlines and replace with \n new lines to prevent "bad request" responses - // We also remove backslashes and replace with forward slashes, Loki doesn't like those either - var propertyValue = property.Value.ToString().Replace("\r\n", "\n"); - - switch (DetermineHandleActionForProperty(property.Key)) - { - case HandleAction.Discard: - continue; - case HandleAction.SendAsLabel: - propertyValue = propertyValue.Replace("\"", "").Replace("\\", "/"); - stream.Labels.Add(new LokiLabel(property.Key, propertyValue)); - break; - case HandleAction.AppendToMessage: - sb.Append($" {property.Key}={propertyValue}"); - break; - } + HandleProperty(property.Key, property.Value.ToString(), labels, sb); + } + + // Order the labels so they always get the same chunk in loki + labels = labels.OrderBy(l => l.Key).ToList(); + var key = string.Join(",", labels.Select(l => $"{l.Key}={l.Value}")); + if (!streamsDictionary.TryGetValue(key, out var stream)) + { + streamsDictionary.Add(key, stream = new LokiContentStream()); + stream.Labels.AddRange(labels); } // Loki doesn't like \r\n for new line, and we can't guarantee the message doesn't have any @@ -111,8 +78,36 @@ public void Format(IEnumerable logEvents, ITextFormatter formatter, Te stream.Entries.Add(new LokiEntry(time, sb.ToString().Replace("\r\n", "\n"))); } - if (content.Streams.Count > 0) + if (streamsDictionary.Count > 0) + { + var content = new LokiContent + { + Streams = streamsDictionary.Values.ToList() + }; output.Write(content.Serialize()); + } + } + + private void HandleProperty(string name, string value, ICollection labels, StringBuilder sb) + { + // Some enrichers pass strings with quotes surrounding the values inside the string, + // which results in redundant quotes after serialization and a "bad request" response. + // To avoid this, remove all quotes from the value. + // We also remove any \r\n newlines and replace with \n new lines to prevent "bad request" responses + // We also remove backslashes and replace with forward slashes, Loki doesn't like those either + value = value.Replace("\r\n", "\n"); + + switch (DetermineHandleActionForProperty(name)) + { + case HandleAction.Discard: return; + case HandleAction.SendAsLabel: + value = value.Replace("\"", "").Replace("\\", "/"); + labels.Add(new LokiLabel(name, value)); + break; + case HandleAction.AppendToMessage: + sb.Append($" {name}={value}"); + break; + } } public void Format(IEnumerable logEvents, TextWriter output) diff --git a/src/Serilog.Sinks.Loki/LokiSinkConfiguration.cs b/src/Serilog.Sinks.Loki/LokiSinkConfiguration.cs index 03f1b50..0a1b2dc 100644 --- a/src/Serilog.Sinks.Loki/LokiSinkConfiguration.cs +++ b/src/Serilog.Sinks.Loki/LokiSinkConfiguration.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System; using Serilog.Sinks.Http; using Serilog.Sinks.Loki.Labels; @@ -6,10 +6,14 @@ namespace Serilog.Sinks.Loki { public class LokiSinkConfiguration { + internal const string DefaultTemplate = "{Message}{NewLine}{Exception}"; + public string LokiUrl { get; set; } public string LokiUsername { get; set; } public string LokiPassword { get; set; } public ILogLabelProvider LogLabelProvider { get; set; } public IHttpClient HttpClient { get; set; } + public string OutputTemplate { get; set; } = DefaultTemplate; + public IFormatProvider FormatProvider { get; set; } } } diff --git a/src/Serilog.Sinks.Loki/LokiSinkExtensions.cs b/src/Serilog.Sinks.Loki/LokiSinkExtensions.cs index 6615547..df1ab02 100644 --- a/src/Serilog.Sinks.Loki/LokiSinkExtensions.cs +++ b/src/Serilog.Sinks.Loki/LokiSinkExtensions.cs @@ -1,5 +1,6 @@ using System; using Serilog.Configuration; +using Serilog.Formatting.Display; using Serilog.Sinks.Http; using Serilog.Sinks.Loki.Labels; @@ -13,8 +14,8 @@ public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConf public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, string serverUrl, string username, string password) => sinkConfiguration.LokiHttp(new BasicAuthCredentials(serverUrl, username, password)); - public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, LokiCredentials credentials, ILogLabelProvider labelProvider = null, LokiHttpClient httpClient = null) - => LokiHttpImpl(sinkConfiguration, credentials, labelProvider, httpClient); + public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, LokiCredentials credentials, ILogLabelProvider labelProvider = null, LokiHttpClient httpClient = null, string outputTemplate = LokiSinkConfiguration.DefaultTemplate, IFormatProvider formatProvider = null) + => LokiHttpImpl(sinkConfiguration, credentials, labelProvider, httpClient, outputTemplate, formatProvider); public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, Func configFactory) => LokiHttpImpl(sinkConfiguration, configFactory()); @@ -25,10 +26,16 @@ private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration ser ? (LokiCredentials)new NoAuthCredentials(lokiConfig.LokiUrl) : new BasicAuthCredentials(lokiConfig.LokiUrl, lokiConfig.LokiUsername, lokiConfig.LokiPassword); - return LokiHttpImpl(serilogConfig, credentials, lokiConfig.LogLabelProvider, lokiConfig.HttpClient); + return LokiHttpImpl(serilogConfig, credentials, lokiConfig.LogLabelProvider, lokiConfig.HttpClient, lokiConfig.OutputTemplate, lokiConfig.FormatProvider); } - private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration sinkConfiguration, LokiCredentials credentials, ILogLabelProvider logLabelProvider, IHttpClient httpClient) + private static LoggerConfiguration LokiHttpImpl( + this LoggerSinkConfiguration sinkConfiguration, + LokiCredentials credentials, + ILogLabelProvider logLabelProvider, + IHttpClient httpClient, + string outputTemplate, + IFormatProvider formatProvider) { var formatter = new LokiBatchFormatter(logLabelProvider ?? new DefaultLogLabelProvider()); var client = httpClient ?? new DefaultLokiHttpClient(); @@ -37,7 +44,10 @@ private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration sin c.SetAuthCredentials(credentials); } - return sinkConfiguration.Http(LokiRouteBuilder.BuildPostUri(credentials.Url), batchFormatter: formatter, httpClient: client); + return sinkConfiguration.Http(LokiRouteBuilder.BuildPostUri(credentials.Url), + batchFormatter: formatter, + textFormatter: new MessageTemplateTextFormatter(outputTemplate, formatProvider), + httpClient: client); } } } diff --git a/test/Serilog.Sinks.Loki.Tests/Infrastructure/TestLabelProvider.cs b/test/Serilog.Sinks.Loki.Tests/Infrastructure/TestLabelProvider.cs deleted file mode 100644 index c458850..0000000 --- a/test/Serilog.Sinks.Loki.Tests/Infrastructure/TestLabelProvider.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.Collections.Generic; -using Serilog.Sinks.Loki.Labels; - -namespace Serilog.Sinks.Loki.Tests.Infrastructure -{ - public class TestLabelProvider : ILogLabelProvider - { - public IList GetLabels() - { - return new List - { - new LokiLabel("app", "tests") - }; - } - - public IList PropertiesAsLabels { get; set; } = new List(); - public IList PropertiesToAppend { get; set; } = new List(); - public LokiFormatterStrategy FormatterStrategy { get; set; } = LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestAppended; - } -} \ No newline at end of file diff --git a/test/Serilog.Sinks.Loki.Tests/Labels/GlobalLabelsTests.cs b/test/Serilog.Sinks.Loki.Tests/Labels/GlobalLabelsTests.cs index d97491c..79be0fa 100644 --- a/test/Serilog.Sinks.Loki.Tests/Labels/GlobalLabelsTests.cs +++ b/test/Serilog.Sinks.Loki.Tests/Labels/GlobalLabelsTests.cs @@ -1,5 +1,6 @@ using System.Linq; using Newtonsoft.Json; +using Serilog.Sinks.Loki.Labels; using Serilog.Sinks.Loki.Tests.Infrastructure; using Shouldly; using Xunit; @@ -23,9 +24,10 @@ public GlobalLabelsTests(HttpClientTestFixture httpClientTestFixture) public void GlobalLabelsCanBeSet() { // Arrange + var provider = new DefaultLogLabelProvider(new[] {new LokiLabel("app", "tests")}); var log = new LoggerConfiguration() .MinimumLevel.Information() - .WriteTo.LokiHttp(_credentials, new TestLabelProvider(), _client) + .WriteTo.LokiHttp(_credentials, provider, _client) .CreateLogger(); // Act @@ -34,7 +36,7 @@ public void GlobalLabelsCanBeSet() // Assert var response = JsonConvert.DeserializeObject(_client.Content); - response.Streams.First().Labels.ShouldBe("{level=\"error\",app=\"tests\"}"); + response.Streams.First().Labels.ShouldBe("{app=\"tests\",level=\"error\"}"); } } } \ No newline at end of file diff --git a/test/Serilog.Sinks.Loki.Tests/Labels/LogLevelLabelTests.cs b/test/Serilog.Sinks.Loki.Tests/Labels/LogLevelLabelTests.cs index 67ad4f3..9f49380 100644 --- a/test/Serilog.Sinks.Loki.Tests/Labels/LogLevelLabelTests.cs +++ b/test/Serilog.Sinks.Loki.Tests/Labels/LogLevelLabelTests.cs @@ -1,5 +1,6 @@ using System.Linq; using Newtonsoft.Json; +using Serilog.Sinks.Loki.Labels; using Serilog.Sinks.Loki.Tests.Infrastructure; using Shouldly; using Xunit; @@ -17,6 +18,25 @@ public LogLevelTests() _credentials = new BasicAuthCredentials("http://test:80", "Walter", "White"); } + [Fact] + public void NoLabelIsSet() + { + // Arrange + var provider = new DefaultLogLabelProvider(new LokiLabel[0], new string[0]); // Explicitly NOT include level + var log = new LoggerConfiguration() + .MinimumLevel.Verbose() + .WriteTo.LokiHttp(_credentials, provider, _client) + .CreateLogger(); + + // Act + log.Fatal("Fatal Level"); + log.Dispose(); + + // Assert + var response = JsonConvert.DeserializeObject(_client.Content); + response.Streams.First().Labels.ShouldBe("{}"); + } + [Fact] public void VerboseLabelIsSet() {