If we can't read the encryption key, don't try and read the logs

OctopusDeploy · Nov 28, 2024 · 161b5ab · 161b5ab
1 parent e1002d0
commit 161b5ab
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 21 deletions.
diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/HelmDownloader.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/HelmDownloader.cs
@@ -9,7 +9,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup.Tooling;
 
 public class HelmDownloader : ToolDownloader
 {
-    const string LatestVersion = "v3.14.3";
+    const string LatestVersion = "v3.16.3";
     public HelmDownloader( ILogger logger)
         : base("helm", logger)
     {

diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/KindDownloader.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/KindDownloader.cs
@@ -6,7 +6,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup.Tooling
 {
     public class KindDownloader : ToolDownloader
     {
-        const string LatestKindVersion = "v0.22.0";
+        const string LatestKindVersion = "v0.25.0";
 
         public KindDownloader(ILogger logger)
             : base("kind", logger)

diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/KubeCtlDownloader.cs b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/Tooling/KubeCtlDownloader.cs
@@ -4,7 +4,7 @@ namespace Octopus.Tentacle.Kubernetes.Tests.Integration.Setup.Tooling;
 
 public class KubeCtlDownloader : ToolDownloader
 {
-    public const string LatestKubeCtlVersion = "v1.29.3";
+    public const string LatestKubeCtlVersion = "v1.30.6";
 
     public KubeCtlDownloader(ILogger logger) 
         : base("kubectl", logger)

diff --git a/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/kind-config.yaml b/source/Octopus.Tentacle.Kubernetes.Tests.Integration/Setup/kind-config.yaml
@@ -3,6 +3,6 @@ apiVersion: kind.x-k8s.io/v1alpha4
 
 nodes:
   - role: control-plane
-    image: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245
+    image: kindest/node:v1.30.6@sha256:b6d08db72079ba5ae1f4a88a09025c0a904af3b52387643c285442afb05ab994
   - role: worker
-    image: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245
+    image: kindest/node:v1.30.6@sha256:b6d08db72079ba5ae1f4a88a09025c0a904af3b52387643c285442afb05ab994
diff --git a/source/Octopus.Tentacle/Kubernetes/IKubernetesPodLogService.cs b/source/Octopus.Tentacle/Kubernetes/IKubernetesPodLogService.cs
@@ -86,16 +86,28 @@ public KubernetesPodLogService(
                 var sinceTime = scriptPodSinceTimeStore.GetPodLogsSinceTime(scriptTicket);
                 try
                 {
-                    return await GetPodLogsWithSinceTime(sinceTime);
+                    try
+                    {
+                        return await GetPodLogsWithSinceTime(sinceTime);
+                    }
+                    catch (UnexpectedPodLogLineNumberException ex)
+                    {
+                        var message = $"Unexpected Pod log line numbers found with sinceTime='{sinceTime}', loading all logs";
+                        tentacleScriptLog.Verbose(message);
+                        Log.Warn(ex, message);
+
+                        //If we somehow come across weird/missing line numbers, try load the whole Pod logs to see if that helps
+                        return await GetPodLogsWithSinceTime(null);
+                    }
                 }
-                catch (UnexpectedPodLogLineNumberException ex)
+                catch (PodLogEncryptionKeyException ex)
                 {
-                    var message = $"Unexpected Pod log line numbers found with sinceTime='{sinceTime}', loading all logs";
+                    //if we can't read the pod log encryption key for a while
+                    var message = $"Failed to read pod log encryption key. No new pod logs will be read.";
                     tentacleScriptLog.Verbose(message);
                     Log.Warn(ex, message);
 
-                    //If we somehow come across weird/missing line numbers, try load the whole Pod logs to see if that helps
-                    return await GetPodLogsWithSinceTime(null);
+                    return (new List<ProcessOutput>(), lastLogSequence, null);
                 }
             }
 
@@ -120,8 +132,7 @@ async Task<IEnumerable<ProcessOutput>> GetPodEvents(ScriptTicket scriptTicket, s
             {
                 return Array.Empty<ProcessOutput>();
             }
-
-
+
             var sinceTime = scriptPodSinceTimeStore.GetPodEventsSinceTime(scriptTicket);
 
             var allEvents = await eventService.FetchAllEventsAsync(KubernetesConfig.Namespace, podName, cancellationToken);
@@ -219,6 +230,5 @@ public static class EventExtensions
         public static bool IsPullingReason(this Corev1Event @event) => @event.Reason.Equals("Pulling", StringComparison.OrdinalIgnoreCase);
         public static bool IsPulledReason(this Corev1Event @event) => @event.Reason.Equals("Pulled", StringComparison.OrdinalIgnoreCase);
         public static bool IsWarning(this Corev1Event @event) => @event.Type.Equals("Warning", StringComparison.OrdinalIgnoreCase);
-
     }
 }
diff --git a/source/Octopus.Tentacle/Kubernetes/IScriptPodLogEncryptionKeyProvider.cs b/source/Octopus.Tentacle/Kubernetes/IScriptPodLogEncryptionKeyProvider.cs
@@ -15,7 +15,7 @@ public interface IScriptPodLogEncryptionKeyProvider
 
     public class ScriptPodLogEncryptionKeyProvider : IScriptPodLogEncryptionKeyProvider
     {
-        const string Filename = "keyfile"; 
+        const string Filename = "keyfile";
         readonly IScriptWorkspaceFactory scriptWorkspaceFactory;
 
         readonly ConcurrentDictionary<ScriptTicket, byte[]> encryptionKeyCache = new();
@@ -29,13 +29,13 @@ public void WriteEncryptionKeyfileToWorkspace(ScriptTicket scriptTicket)
         {
             if (encryptionKeyCache.ContainsKey(scriptTicket))
             {
-                throw new InvalidOperationException($"An encryption key already exists for script {scriptTicket.TaskId}");
+                throw new PodLogEncryptionKeyException($"An encryption key already exists for script {scriptTicket.TaskId}");
             }
 
             var encryptionKeyBytes = GenerateEncryptionKeyBytes();
             if (!encryptionKeyCache.TryAdd(scriptTicket, encryptionKeyBytes))
             {
-                throw new InvalidOperationException($"Failed to store encryption key in memory cache for script {scriptTicket.TaskId}");
+                throw new PodLogEncryptionKeyException($"Failed to store encryption key in memory cache for script {scriptTicket.TaskId}");
             }
 
             try
@@ -46,7 +46,7 @@ public void WriteEncryptionKeyfileToWorkspace(ScriptTicket scriptTicket)
             }
             catch (Exception e)
             {
-                throw new InvalidOperationException($"Failed to write encryption key to workspace for script {scriptTicket.TaskId}", e);
+                throw new PodLogEncryptionKeyException($"Failed to write encryption key to workspace for script {scriptTicket.TaskId}", e);
             }
         }
 
@@ -56,24 +56,24 @@ public byte[] GetEncryptionKey(ScriptTicket scriptTicket)
             {
                 return keyBytes;
             }
-            
+
             //read from file
             var workspace = scriptWorkspaceFactory.GetWorkspace(scriptTicket);
             var fileContents = workspace.TryReadFile(Filename);
             if (fileContents == null)
             {
-                throw new InvalidOperationException($"Failed to load encryption key from workspace for script {scriptTicket.TaskId}");
+                throw new PodLogEncryptionKeyException($"Failed to load encryption key from workspace for script {scriptTicket.TaskId}");
             }
 
             if (string.IsNullOrWhiteSpace(fileContents))
             {
-                throw new InvalidOperationException($"Encryption key loaded from workspace for script {scriptTicket.TaskId} is empty or whitespace");
+                throw new PodLogEncryptionKeyException($"Encryption key loaded from workspace for script {scriptTicket.TaskId} is empty or whitespace");
             }
 
             var encryptionKeyBytes = Convert.FromBase64String(fileContents);
             if (!encryptionKeyCache.TryAdd(scriptTicket, encryptionKeyBytes))
             {
-                throw new InvalidOperationException($"Failed to store encryption key in memory cache for script {scriptTicket.TaskId}");
+                throw new PodLogEncryptionKeyException($"Failed to store encryption key in memory cache for script {scriptTicket.TaskId}");
             }
 
             return encryptionKeyBytes;
@@ -100,4 +100,15 @@ static byte[] GenerateEncryptionKeyBytes()
 #endif
         }
     }
+
+    public class PodLogEncryptionKeyException : Exception
+    {
+        public PodLogEncryptionKeyException(string message) : base(message)
+        {
+        }
+
+        public PodLogEncryptionKeyException(string message, Exception innerException) : base(message, innerException)
+        {
+        }
+    }
 }