Merge pull request #33 from Falco20019/Falco20019/enhance-32

Add suggestions from #32

src/Serilog.Sinks.Loki.Example/LogLabelProvider.cs

@@ -16,6 +16,7 @@ public IList<LokiLabel> GetLabels()
 
         public IList<string> PropertiesAsLabels { get; set; } = new List<string>
         {
+            "level", // Since 3.0.0, you need to explicitly add level if you want it!
             "MyLabelPropertyName"
         };
         public IList<string> PropertiesToAppend { get; set; } = new List<string>

src/Serilog.Sinks.Loki/Labels/DefaultLogLabelProvider.cs

@@ -3,7 +3,7 @@
 
 namespace Serilog.Sinks.Loki.Labels
 {
-    class DefaultLogLabelProvider : ILogLabelProvider
+    public class DefaultLogLabelProvider : ILogLabelProvider
     {
         public DefaultLogLabelProvider() : this(null)
         {
@@ -15,7 +15,7 @@ public DefaultLogLabelProvider(IEnumerable<LokiLabel> labels,
             LokiFormatterStrategy formatterStrategy = LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestAppended)
         {
             this.Labels = labels?.ToList() ?? new List<LokiLabel>();
-            this.PropertiesAsLabels = propertiesAsLabels?.ToList() ?? new List<string>();
+            this.PropertiesAsLabels = propertiesAsLabels?.ToList() ?? new List<string> {"level"};
             this.PropertiesToAppend = propertiesToAppend?.ToList() ?? new List<string>();
             this.FormatterStrategy = formatterStrategy;
         }

src/Serilog.Sinks.Loki/LokiBatchFormatter.cs

@@ -32,26 +32,6 @@ public LokiBatchFormatter(IList<LokiLabel> globalLabels)
             this.LogLabelProvider = new DefaultLogLabelProvider(globalLabels);
         }
 
-        // This avoids additional quoting as described in https://github.com/serilog/serilog/issues/936
-        private static void RenderMessage(TextWriter tw, LogEvent logEvent)
-        {
-            bool IsString(LogEventPropertyValue pv)
-            {
-                return pv is ScalarValue sv && sv.Value is string;
-            }
-
-            foreach(var t in logEvent.MessageTemplate.Tokens)
-            {
-                if (t is PropertyToken pt &&
-                    logEvent.Properties.TryGetValue(pt.PropertyName, out var propVal) &&
-                    IsString(propVal))
-                    tw.Write(((ScalarValue)propVal).Value);
-                else
-                    t.Render(logEvent.Properties, tw);
-            }
-            tw.Write('\n');
-        }   
-
         public void Format(IEnumerable<LogEvent> logEvents, ITextFormatter formatter, TextWriter output)
         {
             if (logEvents == null)
@@ -63,56 +43,71 @@ public void Format(IEnumerable<LogEvent> logEvents, ITextFormatter formatter, Te
             if (!logs.Any())
                 return;
 
-            var content = new LokiContent();
+            var streamsDictionary = new Dictionary<string, LokiContentStream>();
             foreach (LogEvent logEvent in logs)
             {
-                var stream = new LokiContentStream();
-                content.Streams.Add(stream);
+                var labels = new List<LokiLabel>();
 
-                stream.Labels.Add(new LokiLabel("level", GetLevel(logEvent.Level)));
                 foreach (LokiLabel globalLabel in this.LogLabelProvider.GetLabels())
-                    stream.Labels.Add(new LokiLabel(globalLabel.Key, globalLabel.Value));
+                    labels.Add(new LokiLabel(globalLabel.Key, globalLabel.Value));
 
                 var time = logEvent.Timestamp.ToString("o");
                 var sb = new StringBuilder();
                 using (var tw = new StringWriter(sb))
                 {
-                    RenderMessage(tw, logEvent);
+                    formatter.Format(logEvent, tw);
                 }
-                if (logEvent.Exception != null)
-                    // AggregateException adds a Environment.Newline to the end of ToString(), so we trim it off
-                    sb.AppendLine(logEvent.Exception.ToString().TrimEnd());
 
+                HandleProperty("level", GetLevel(logEvent.Level), labels, sb);
                 foreach (KeyValuePair<string, LogEventPropertyValue> property in logEvent.Properties)
                 {
-                    // Some enrichers pass strings with quotes surrounding the values inside the string,
-                    // which results in redundant quotes after serialization and a "bad request" response.
-                    // To avoid this, remove all quotes from the value.
-                    // We also remove any \r\n newlines and replace with \n new lines to prevent "bad request" responses
-                    // We also remove backslashes and replace with forward slashes, Loki doesn't like those either
-                    var propertyValue = property.Value.ToString().Replace("\r\n", "\n");
-
-                    switch (DetermineHandleActionForProperty(property.Key))
-                    {
-                        case HandleAction.Discard:
-                            continue;
-                        case HandleAction.SendAsLabel:
-                            propertyValue = propertyValue.Replace("\"", "").Replace("\\", "/");
-                            stream.Labels.Add(new LokiLabel(property.Key, propertyValue));
-                            break;
-                        case HandleAction.AppendToMessage:
-                            sb.Append($" {property.Key}={propertyValue}");
-                            break;
-                    }
+                    HandleProperty(property.Key, property.Value.ToString(), labels, sb);
+                }
+
+                // Order the labels so they always get the same chunk in loki
+                labels = labels.OrderBy(l => l.Key).ToList();
+                var key = string.Join(",", labels.Select(l => $"{l.Key}={l.Value}"));
+                if (!streamsDictionary.TryGetValue(key, out var stream))
+                {
+                    streamsDictionary.Add(key, stream = new LokiContentStream());
+                    stream.Labels.AddRange(labels);
                 }
 
                 // Loki doesn't like \r\n for new line, and we can't guarantee the message doesn't have any
                 // in it, so we replace \r\n with \n on the final message
                 stream.Entries.Add(new LokiEntry(time, sb.ToString().Replace("\r\n", "\n")));
             }
 
-            if (content.Streams.Count > 0)
+            if (streamsDictionary.Count > 0)
+            {
+                var content = new LokiContent
+                {
+                    Streams = streamsDictionary.Values.ToList()
+                };
                 output.Write(content.Serialize());
+            }
+        }
+
+        private void HandleProperty(string name, string value, ICollection<LokiLabel> labels, StringBuilder sb)
+        {
+            // Some enrichers pass strings with quotes surrounding the values inside the string,
+            // which results in redundant quotes after serialization and a "bad request" response.
+            // To avoid this, remove all quotes from the value.
+            // We also remove any \r\n newlines and replace with \n new lines to prevent "bad request" responses
+            // We also remove backslashes and replace with forward slashes, Loki doesn't like those either
+            value = value.Replace("\r\n", "\n");
+
+            switch (DetermineHandleActionForProperty(name))
+            {
+                case HandleAction.Discard: return;
+                case HandleAction.SendAsLabel:
+                    value = value.Replace("\"", "").Replace("\\", "/");
+                    labels.Add(new LokiLabel(name, value));
+                    break;
+                case HandleAction.AppendToMessage:
+                    sb.Append($" {name}={value}");
+                    break;
+            }
         }
 
         public void Format(IEnumerable<string> logEvents, TextWriter output)

src/Serilog.Sinks.Loki/LokiSinkConfiguration.cs

@@ -1,15 +1,19 @@
-using System.Collections.Generic;
+using System;
 using Serilog.Sinks.Http;
 using Serilog.Sinks.Loki.Labels;
 
 namespace Serilog.Sinks.Loki
 {
     public class LokiSinkConfiguration
     {
+        internal const string DefaultTemplate = "{Message}{NewLine}{Exception}";
+
         public string LokiUrl { get; set; }
         public string LokiUsername { get; set; }
         public string LokiPassword { get; set; }
         public ILogLabelProvider LogLabelProvider { get; set; }
         public IHttpClient HttpClient { get; set; }
+        public string OutputTemplate { get; set; } = DefaultTemplate;
+        public IFormatProvider FormatProvider { get; set; }
     }
 }

src/Serilog.Sinks.Loki/LokiSinkExtensions.cs

@@ -1,5 +1,6 @@
 using System;
 using Serilog.Configuration;
+using Serilog.Formatting.Display;
 using Serilog.Sinks.Http;
 using Serilog.Sinks.Loki.Labels;
 
@@ -13,8 +14,8 @@ public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConf
         public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, string serverUrl, string username, string password)
             => sinkConfiguration.LokiHttp(new BasicAuthCredentials(serverUrl, username, password));
 
-        public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, LokiCredentials credentials, ILogLabelProvider labelProvider = null, LokiHttpClient httpClient = null)
-            => LokiHttpImpl(sinkConfiguration, credentials, labelProvider, httpClient);
+        public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, LokiCredentials credentials, ILogLabelProvider labelProvider = null, LokiHttpClient httpClient = null, string outputTemplate = LokiSinkConfiguration.DefaultTemplate, IFormatProvider formatProvider = null)
+            => LokiHttpImpl(sinkConfiguration, credentials, labelProvider, httpClient, outputTemplate, formatProvider);
 
         public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, Func<LokiSinkConfiguration> configFactory)
             => LokiHttpImpl(sinkConfiguration, configFactory());
@@ -25,10 +26,16 @@ private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration ser
                 ? (LokiCredentials)new NoAuthCredentials(lokiConfig.LokiUrl)
                 : new BasicAuthCredentials(lokiConfig.LokiUrl, lokiConfig.LokiUsername, lokiConfig.LokiPassword);
 
-            return LokiHttpImpl(serilogConfig, credentials, lokiConfig.LogLabelProvider, lokiConfig.HttpClient);
+            return LokiHttpImpl(serilogConfig, credentials, lokiConfig.LogLabelProvider, lokiConfig.HttpClient, lokiConfig.OutputTemplate, lokiConfig.FormatProvider);
         }
 
-        private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration sinkConfiguration, LokiCredentials credentials, ILogLabelProvider logLabelProvider, IHttpClient httpClient)
+        private static LoggerConfiguration LokiHttpImpl(
+            this LoggerSinkConfiguration sinkConfiguration,
+            LokiCredentials credentials,
+            ILogLabelProvider logLabelProvider,
+            IHttpClient httpClient,
+            string outputTemplate,
+            IFormatProvider formatProvider)
         {
             var formatter = new LokiBatchFormatter(logLabelProvider ?? new DefaultLogLabelProvider());
             var client = httpClient ?? new DefaultLokiHttpClient();
@@ -37,7 +44,10 @@ private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration sin
                 c.SetAuthCredentials(credentials);
             }
 
-            return sinkConfiguration.Http(LokiRouteBuilder.BuildPostUri(credentials.Url), batchFormatter: formatter, httpClient: client);
+            return sinkConfiguration.Http(LokiRouteBuilder.BuildPostUri(credentials.Url),
+                batchFormatter: formatter,
+                textFormatter: new MessageTemplateTextFormatter(outputTemplate, formatProvider),
+                httpClient: client);
         }
     }
 }

test/Serilog.Sinks.Loki.Tests/Labels/GlobalLabelsTests.cs

@@ -1,5 +1,6 @@
 using System.Linq;
 using Newtonsoft.Json;
+using Serilog.Sinks.Loki.Labels;
 using Serilog.Sinks.Loki.Tests.Infrastructure;
 using Shouldly;
 using Xunit;
@@ -23,9 +24,10 @@ public GlobalLabelsTests(HttpClientTestFixture httpClientTestFixture)
         public void GlobalLabelsCanBeSet()
         {
             // Arrange
+            var provider = new DefaultLogLabelProvider(new[] {new LokiLabel("app", "tests")});
             var log = new LoggerConfiguration()
                 .MinimumLevel.Information()
-                .WriteTo.LokiHttp(_credentials, new TestLabelProvider(), _client)
+                .WriteTo.LokiHttp(_credentials, provider, _client)
                 .CreateLogger();
 
             // Act
@@ -34,7 +36,7 @@ public void GlobalLabelsCanBeSet()
 
             // Assert
             var response = JsonConvert.DeserializeObject<TestResponse>(_client.Content);
-            response.Streams.First().Labels.ShouldBe("{level=\"error\",app=\"tests\"}");
+            response.Streams.First().Labels.ShouldBe("{app=\"tests\",level=\"error\"}");
         }
     }
 }

test/Serilog.Sinks.Loki.Tests/Labels/LogLevelLabelTests.cs

@@ -1,5 +1,6 @@
 using System.Linq;
 using Newtonsoft.Json;
+using Serilog.Sinks.Loki.Labels;
 using Serilog.Sinks.Loki.Tests.Infrastructure;
 using Shouldly;
 using Xunit;
@@ -17,6 +18,25 @@ public LogLevelTests()
             _credentials = new BasicAuthCredentials("http://test:80", "Walter", "White");
         }
 
+        [Fact]
+        public void NoLabelIsSet()
+        {
+            // Arrange
+            var provider = new DefaultLogLabelProvider(new LokiLabel[0], new string[0]); // Explicitly NOT include level
+            var log = new LoggerConfiguration()
+                .MinimumLevel.Verbose()
+                .WriteTo.LokiHttp(_credentials, provider, _client)
+                .CreateLogger();
+
+            // Act
+            log.Fatal("Fatal Level");
+            log.Dispose();
+
+            // Assert
+            var response = JsonConvert.DeserializeObject<TestResponse>(_client.Content);
+            response.Streams.First().Labels.ShouldBe("{}");
+        }
+
         [Fact]
         public void VerboseLabelIsSet()
         {