Merge pull request #32 from Falco20019/Falco20019/pr15+22

Falco20019/pr15+22

src/Serilog.Sinks.Loki.Example/LogLabelProvider.cs

@@ -16,7 +16,12 @@ public IList<LokiLabel> GetLabels()
 
         public IList<string> PropertiesAsLabels { get; set; } = new List<string>
         {
-            "MyPropertyName"
+            "MyLabelPropertyName"
         };
+        public IList<string> PropertiesToAppend { get; set; } = new List<string>
+        {
+            "MyAppendPropertyName"
+        };
+        public LokiFormatterStrategy FormatterStrategy { get; set; } = LokiFormatterStrategy.SpecificPropertiesAsLabelsOrAppended;
     }
 }

src/Serilog.Sinks.Loki.Example/Program.cs

@@ -13,7 +13,7 @@ static void Main(string[] args)
             Logger log = new LoggerConfiguration()
                         .MinimumLevel.Verbose()
                         .Enrich.FromLogContext()
-                        .Enrich.WithProperty("MyPropertyName","MyPropertyValue")
+                        .Enrich.WithProperty("MyLabelPropertyName","MyPropertyValue")
                         .Enrich.WithThreadId()
                         .WriteTo.Console()
                         .WriteTo.LokiHttp(credentials, new LogLabelProvider(), new LokiExampleHttpClient())
@@ -46,6 +46,12 @@ static void Main(string[] args)
                 log.Fatal("Fatal with Property A");
             }
 
+            using (LogContext.PushProperty("MyAppendPropertyName", 1))
+            {
+                log.Warning("Warning with Property MyAppendPropertyName");
+                log.Fatal("Fatal with Property MyAppendPropertyName");
+            }
+
             log.Dispose();
         }
     }

src/Serilog.Sinks.Loki.ExampleWebApp/Models/ErrorViewModel.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace Serilog.Sinks.Loki.ExampleWebApp.Models
 {
     public class ErrorViewModel

src/Serilog.Sinks.Loki.ExampleWebApp/Startup.cs

@@ -1,12 +1,6 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.AspNetCore.Http;
-using Microsoft.AspNetCore.HttpsPolicy;
-using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;

src/Serilog.Sinks.Loki/ContextualLabels.cs

@@ -1,5 +1,3 @@
-using Serilog.Core;
-
 namespace Serilog.Sinks.Loki
 {
 /*    public static class ContextualLabels

src/Serilog.Sinks.Loki/DefaultLokiHttpClient.cs

@@ -1,8 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-
-namespace Serilog.Sinks.Loki
+namespace Serilog.Sinks.Loki
 {
   public class DefaultLokiHttpClient : LokiHttpClient
   {

src/Serilog.Sinks.Loki/Labels/DefaultLogLabelProvider.cs

@@ -1,14 +1,33 @@
 using System.Collections.Generic;
+using System.Linq;
 
 namespace Serilog.Sinks.Loki.Labels
 {
     class DefaultLogLabelProvider : ILogLabelProvider
     {
-        public IList<LokiLabel> GetLabels()
+        public DefaultLogLabelProvider() : this(null)
+        {
+        }
+
+        public DefaultLogLabelProvider(IEnumerable<LokiLabel> labels,
+            IEnumerable<string> propertiesAsLabels = null,
+            IEnumerable<string> propertiesToAppend = null,
+            LokiFormatterStrategy formatterStrategy = LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestAppended)
         {
-            return new List<LokiLabel>();
+            this.Labels = labels?.ToList() ?? new List<LokiLabel>();
+            this.PropertiesAsLabels = propertiesAsLabels?.ToList() ?? new List<string>();
+            this.PropertiesToAppend = propertiesToAppend?.ToList() ?? new List<string>();
+            this.FormatterStrategy = formatterStrategy;
         }
 
-        public IList<string> PropertiesAsLabels { get; set; } = new List<string>();
+        public IList<LokiLabel> GetLabels()
+        {
+            return this.Labels;
+        }
+
+        private IList<LokiLabel> Labels { get; }
+        public IList<string> PropertiesAsLabels { get; }
+        public IList<string> PropertiesToAppend { get; }
+        public LokiFormatterStrategy FormatterStrategy { get; }
     }
 }

src/Serilog.Sinks.Loki/Labels/ILogLabelProvider.cs

@@ -5,6 +5,9 @@ namespace Serilog.Sinks.Loki.Labels
     public interface ILogLabelProvider
     {
         IList<LokiLabel> GetLabels();
-        IList<string> PropertiesAsLabels { get; set; }
+
+        IList<string> PropertiesAsLabels { get; }
+        IList<string> PropertiesToAppend { get; }
+        LokiFormatterStrategy FormatterStrategy { get; }
     }
 }

src/Serilog.Sinks.Loki/LokiBatchFormatter.cs

@@ -4,29 +4,54 @@
 using System.Linq;
 using Serilog.Events;
 using Serilog.Formatting;
+using Serilog.Parsing;
 using Serilog.Sinks.Http;
 using Serilog.Sinks.Loki.Labels;
 
 namespace Serilog.Sinks.Loki
 {
     using System.Text;
 
-    internal class LokiBatchFormatter : IBatchFormatter 
+    internal class LokiBatchFormatter : IBatchFormatter
     {
-        private readonly IList<LokiLabel> _globalLabels;
-        private readonly IList<string> _propertiesAsLabels;
+        public ILogLabelProvider LogLabelProvider { get; }
 
         public LokiBatchFormatter()
         {
-            _globalLabels = new List<LokiLabel>();
+            this.LogLabelProvider = new DefaultLogLabelProvider();
         }
 
         public LokiBatchFormatter(ILogLabelProvider logLabelProvider)
         {
-            _globalLabels = logLabelProvider.GetLabels();
-            _propertiesAsLabels = logLabelProvider.PropertiesAsLabels;
+            this.LogLabelProvider = logLabelProvider;
         }
 
+        [Obsolete("Assign to LokiBatchFormatter.GlobalLabels instead.")]
+        public LokiBatchFormatter(IList<LokiLabel> globalLabels)
+        {
+            this.LogLabelProvider = new DefaultLogLabelProvider(globalLabels);
+        }
+
+        // This avoids additional quoting as described in https://github.com/serilog/serilog/issues/936
+        private static void RenderMessage(TextWriter tw, LogEvent logEvent)
+        {
+            bool IsString(LogEventPropertyValue pv)
+            {
+                return pv is ScalarValue sv && sv.Value is string;
+            }
+
+            foreach(var t in logEvent.MessageTemplate.Tokens)
+            {
+                if (t is PropertyToken pt &&
+                    logEvent.Properties.TryGetValue(pt.PropertyName, out var propVal) &&
+                    IsString(propVal))
+                    tw.Write(((ScalarValue)propVal).Value);
+                else
+                    t.Render(logEvent.Properties, tw);
+            }
+            tw.Write('\n');
+        }   
+
         public void Format(IEnumerable<LogEvent> logEvents, ITextFormatter formatter, TextWriter output)
         {
             if (logEvents == null)
@@ -45,13 +70,15 @@ public void Format(IEnumerable<LogEvent> logEvents, ITextFormatter formatter, Te
                 content.Streams.Add(stream);
 
                 stream.Labels.Add(new LokiLabel("level", GetLevel(logEvent.Level)));
-                foreach (LokiLabel globalLabel in _globalLabels)
+                foreach (LokiLabel globalLabel in this.LogLabelProvider.GetLabels())
                     stream.Labels.Add(new LokiLabel(globalLabel.Key, globalLabel.Value));
 
                 var time = logEvent.Timestamp.ToString("o");
-
                 var sb = new StringBuilder();
-                sb.AppendLine(logEvent.RenderMessage());
+                using (var tw = new StringWriter(sb))
+                {
+                    RenderMessage(tw, logEvent);
+                }
                 if (logEvent.Exception != null)
                     // AggregateException adds a Environment.Newline to the end of ToString(), so we trim it off
                     sb.AppendLine(logEvent.Exception.ToString().TrimEnd());
@@ -63,21 +90,25 @@ public void Format(IEnumerable<LogEvent> logEvents, ITextFormatter formatter, Te
                     // To avoid this, remove all quotes from the value.
                     // We also remove any \r\n newlines and replace with \n new lines to prevent "bad request" responses
                     // We also remove backslashes and replace with forward slashes, Loki doesn't like those either
-                    var propertyValue = property.Value.ToString().Replace("\"", "").Replace("\r\n", "\n").Replace("\\", "/");
-                    if (_propertiesAsLabels.Contains(property.Key, StringComparer.OrdinalIgnoreCase))
-                    {
-                        stream.Labels.Add(new LokiLabel(property.Key, propertyValue));
-                    }
-                    else
+                    var propertyValue = property.Value.ToString().Replace("\r\n", "\n");
+
+                    switch (DetermineHandleActionForProperty(property.Key))
                     {
-                        sb.Append($" {property.Key}={propertyValue}");
+                        case HandleAction.Discard:
+                            continue;
+                        case HandleAction.SendAsLabel:
+                            propertyValue = propertyValue.Replace("\"", "").Replace("\\", "/");
+                            stream.Labels.Add(new LokiLabel(property.Key, propertyValue));
+                            break;
+                        case HandleAction.AppendToMessage:
+                            sb.Append($" {property.Key}={propertyValue}");
+                            break;
                     }
                 }
 
                 // Loki doesn't like \r\n for new line, and we can't guarantee the message doesn't have any
                 // in it, so we replace \r\n with \n on the final message
-                // We also flip backslashes to forward slashes, Loki doesn't like those either.
-                stream.Entries.Add(new LokiEntry(time, sb.ToString().Replace("\\", "/").Replace("\r\n", "\n")));
+                stream.Entries.Add(new LokiEntry(time, sb.ToString().Replace("\r\n", "\n")));
             }
 
             if (content.Streams.Count > 0)
@@ -99,5 +130,40 @@ private static string GetLevel(LogEventLevel level)
                 default: return level.ToString().ToLower();
             }
         }
+
+        private HandleAction DetermineHandleActionForProperty(string propertyName)
+        {
+            var provider = this.LogLabelProvider;
+            switch (provider.FormatterStrategy)
+            {
+                case LokiFormatterStrategy.AllPropertiesAsLabels:
+                    return HandleAction.SendAsLabel;
+
+                case LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestDiscarded:
+                    return provider.PropertiesAsLabels.Contains(propertyName)
+                        ? HandleAction.SendAsLabel
+                        : HandleAction.Discard;
+
+                case LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestAppended:
+                    return provider.PropertiesAsLabels.Contains(propertyName)
+                        ? HandleAction.SendAsLabel
+                        : HandleAction.AppendToMessage;
+
+                //case LokiFormatterStrategy.SpecificPropertiesAsLabelsOrAppended:
+                default:
+                    return provider.PropertiesAsLabels.Contains(propertyName)
+                        ? HandleAction.SendAsLabel
+                        : provider.PropertiesToAppend.Contains(propertyName)
+                            ? HandleAction.AppendToMessage
+                            : HandleAction.Discard;
+            }
+        }
+
+        private enum HandleAction
+        {
+            Discard,
+            SendAsLabel,
+            AppendToMessage
+        }
     }
-}
+}

src/Serilog.Sinks.Loki/LokiFormatterStrategy.cs

@@ -0,0 +1,19 @@
+namespace Serilog.Sinks.Loki {
+    public enum LokiFormatterStrategy {
+        /// All Serilog Event properties will be sent as labels
+        AllPropertiesAsLabels,
+
+        /// Specific Serilog Event properties will be sent as labels.
+        /// The rest of properties will be discarder.
+        SpecificPropertiesAsLabelsAndRestDiscarded,
+
+        /// Specific Serilog Event properties will be sent as labels.
+        /// The rest of properties will be appended to the log message.
+        SpecificPropertiesAsLabelsAndRestAppended,
+
+        /// Specific Serilog Event properties will be sent as labels.
+        /// Other specific properties will be appended to the log message.
+        /// The rest of properties will be discarded
+        SpecificPropertiesAsLabelsOrAppended
+    }
+}

src/Serilog.Sinks.Loki/LokiSinkConfiguration.cs

@@ -0,0 +1,15 @@
+using System.Collections.Generic;
+using Serilog.Sinks.Http;
+using Serilog.Sinks.Loki.Labels;
+
+namespace Serilog.Sinks.Loki
+{
+    public class LokiSinkConfiguration
+    {
+        public string LokiUrl { get; set; }
+        public string LokiUsername { get; set; }
+        public string LokiPassword { get; set; }
+        public ILogLabelProvider LogLabelProvider { get; set; }
+        public IHttpClient HttpClient { get; set; }
+    }
+}

src/Serilog.Sinks.Loki/LokiSinkExtensions.cs

@@ -1,3 +1,4 @@
+using System;
 using Serilog.Configuration;
 using Serilog.Sinks.Http;
 using Serilog.Sinks.Loki.Labels;
@@ -6,13 +7,30 @@ namespace Serilog.Sinks.Loki
 {
     public static class LokiSinkExtensions
     {
+        public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, string serverUrl)
+            => sinkConfiguration.LokiHttp(new NoAuthCredentials(serverUrl));
+
+        public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, string serverUrl, string username, string password)
+            => sinkConfiguration.LokiHttp(new BasicAuthCredentials(serverUrl, username, password));
+
         public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, LokiCredentials credentials, ILogLabelProvider labelProvider = null, LokiHttpClient httpClient = null)
-            => LokiHttpImpl(sinkConfiguration, credentials, labelProvider, httpClient); 
-
+            => LokiHttpImpl(sinkConfiguration, credentials, labelProvider, httpClient);
+
+        public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, Func<LokiSinkConfiguration> configFactory)
+            => LokiHttpImpl(sinkConfiguration, configFactory());
+
+        private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration serilogConfig, LokiSinkConfiguration lokiConfig)
+        {
+            var credentials = string.IsNullOrWhiteSpace(lokiConfig.LokiUsername)
+                ? (LokiCredentials)new NoAuthCredentials(lokiConfig.LokiUrl)
+                : new BasicAuthCredentials(lokiConfig.LokiUrl, lokiConfig.LokiUsername, lokiConfig.LokiPassword);
+
+            return LokiHttpImpl(serilogConfig, credentials, lokiConfig.LogLabelProvider, lokiConfig.HttpClient);
+        }
+
         private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration sinkConfiguration, LokiCredentials credentials, ILogLabelProvider logLabelProvider, IHttpClient httpClient)
         {
             var formatter = new LokiBatchFormatter(logLabelProvider ?? new DefaultLogLabelProvider());
-
             var client = httpClient ?? new DefaultLokiHttpClient();
             if (client is LokiHttpClient c)
             {
@@ -21,11 +39,5 @@ private static LoggerConfiguration LokiHttpImpl(this LoggerSinkConfiguration sin
 
             return sinkConfiguration.Http(LokiRouteBuilder.BuildPostUri(credentials.Url), batchFormatter: formatter, httpClient: client);
         }
-
-        public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, string serverUrl)
-            => sinkConfiguration.LokiHttp(new NoAuthCredentials(serverUrl));
-
-        public static LoggerConfiguration LokiHttp(this LoggerSinkConfiguration sinkConfiguration, string serverUrl, string username, string password)
-            => sinkConfiguration.LokiHttp(new BasicAuthCredentials(serverUrl, username, password));
     }
-}
+}

test/Serilog.Sinks.Loki.Tests/Fixtures/HttpClientTestFixture.cs

@@ -1,6 +1,4 @@
 using System;
-using JustEat.HttpClientInterception;
-using Xunit;
 
 namespace Serilog.Sinks.Loki.Tests
 {

test/Serilog.Sinks.Loki.Tests/Infrastructure/TestLabelProvider.cs

@@ -14,5 +14,7 @@ public IList<LokiLabel> GetLabels()
         }
 
         public IList<string> PropertiesAsLabels { get; set; } = new List<string>();
+        public IList<string> PropertiesToAppend { get; set; } = new List<string>();
+        public LokiFormatterStrategy FormatterStrategy { get; set; } = LokiFormatterStrategy.SpecificPropertiesAsLabelsAndRestAppended;
     }
 }

test/Serilog.Sinks.Loki.Tests/Labels/LocalLabelsTests.cs

@@ -1,7 +1,4 @@
-using System.Linq;
-using Newtonsoft.Json;
 using Serilog.Sinks.Loki.Tests.Infrastructure;
-using Shouldly;
 using Xunit;
 
 namespace Serilog.Sinks.Loki.Tests.Labels