Add detection and recovery for no logs being seen for a configurable …

…number of heartbeats
MattMofDoom · Feb 27, 2022 · 7832593 · 7832593
1 parent 7260bc2
commit 7832593
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 1 deletion.
diff --git a/src/Seq.Client.EventLog/App.config b/src/Seq.Client.EventLog/App.config
@@ -14,6 +14,8 @@
     <add key="LogFolder" value="" />
     <!-- Heartbeat log entry interval in seconds, 0 disables heartbeat, default 600, maximum 86399 -->
     <add key="HeartbeatInterval" value="600" />
+    <!-- If the Event Log service doesn't see any new log activity for the below number of heartbeats, it can restart the listener to recover-->
+    <add key="HeartbeatsBeforeReset" value="2"/>
     <!-- Set IsDebug to true for additional heartbeat logging -->
     <add key="IsDebug" value="false" />
   </appSettings>

diff --git a/src/Seq.Client.EventLog/Config.cs b/src/Seq.Client.EventLog/Config.cs
@@ -15,6 +15,7 @@ static Config()
             LogToFile = GetBool(ConfigurationManager.AppSettings["LogSeqApiKey"], true);
             LogFolder = ConfigurationManager.AppSettings["LogFolder"];
             HeartbeatInterval = GetInt(ConfigurationManager.AppSettings["HeartbeatInterval"]);
+            HeartbeatsBeforeReset = GetInt(ConfigurationManager.AppSettings["HeartbeatsBeforeReset"]);
 
             //Minimum is 0 (disabled)
             if (HeartbeatInterval < 0)
@@ -23,6 +24,9 @@ static Config()
             if (HeartbeatInterval > 3600)
                 HeartbeatInterval = 3600;
 
+            if (HeartbeatsBeforeReset < 0)
+                HeartbeatsBeforeReset = 0;
+
             IsDebug = GetBool(ConfigurationManager.AppSettings["IsDebug"]);
 
             var isSuccess = true;
@@ -65,6 +69,7 @@ static Config()
         public static string LogFolder { get; }
         public static int HeartbeatInterval { get; }
         public static bool IsDebug { get; }
+        public static int HeartbeatsBeforeReset { get; }
 
         /// <summary>
         ///     Convert the supplied <see cref="object" /> to an <see cref="int" />

diff --git a/src/Seq.Client.EventLog/Seq.Client.EventLog.csproj b/src/Seq.Client.EventLog/Seq.Client.EventLog.csproj
@@ -35,7 +35,7 @@
   </PropertyGroup>
   <PropertyGroup>
     <StartupObject />
-    <Version>3.0.3</Version>
+    <Version>3.0.4</Version>
     <Authors>Connor O'Shea and contributors</Authors>
     <Copyright>Connor O'Shea</Copyright>
     <ApplicationIcon>EventLog.ico</ApplicationIcon>

diff --git a/src/Seq.Client.EventLog/ServiceManager.cs b/src/Seq.Client.EventLog/ServiceManager.cs
@@ -2,9 +2,12 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
 using System.Timers;
 using Lurgle.Logging;
 using Newtonsoft.Json;
+using Timer = System.Timers.Timer;
 
 // ReSharper disable MemberCanBePrivate.Global
 
@@ -33,6 +36,7 @@ public static class ServiceManager
         public static string JsonConfigPath { get; set; }
         public static List<EventLogListener> EventLogListeners { get; set; }
 
+        private static int ZeroLogHeartbeats { get; set; } = 0;
 
         public static void Start(bool isInteractive)
         {
@@ -145,6 +149,26 @@ private static void ServiceHeartbeat(object sender, ElapsedEventArgs e)
                 serviceCounters.Add("NonInteractiveLogons", NonInteractiveLogons);
             }
 
+            if (Config.HeartbeatsBeforeReset > 0)
+            {
+                if (EventsProcessed > LastProcessed)
+                    ZeroLogHeartbeats = 0;
+                else
+                    ZeroLogHeartbeats++;
+
+                if (ZeroLogHeartbeats > Config.HeartbeatsBeforeReset)
+                {
+                    Log.Warning().AddProperty("HeartbeatName", $"{Config.AppName} Heartbeat")
+                        .AddProperty("HeartbeatCount", ZeroLogHeartbeats)
+                        .AddProperty("NextTime", timeNow.AddMilliseconds(_isInteractive ? 60000 : Config.HeartbeatInterval * 1000))
+                        .Add("[{HeartbeatName:l} - {MachineName:l}] - Detected that new no log entries have been seen in the past {Heartbeats} heartbeats, resetting listeners ...");
+
+                    StopListeners(); 
+                    Thread.Sleep(1000);
+                    StartListeners(_isInteractive);
+                }
+            }
+
             Log.Debug()
                 .AddProperty("HeartbeatName", $"{Config.AppName} Heartbeat")
                 .AddProperty(serviceCounters)