RavenDB-22487 : flaky test adjustments - increase timeout when waitin…

…g for 2nd batch to complete, add more debug info to error message

test/SlowTests/Server/Documents/ETL/Olap/FailoverTests.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Orders;
@@ -87,7 +88,7 @@ await session.StoreAsync(new Order
     });
 ";
 
-            var connectionStringName = $"{store.Database} to S3";
+            var connectionStringName = $"{store.Database} to local machine";
             var configName = "olap-s3";
             var transformationName = "MonthlyOrders";
             var path = NewDataPath(forceCreateDir: true);
@@ -119,8 +120,10 @@ await session.StoreAsync(new Order
                     }
                 });
 
-            var timeout = TimeSpan.FromSeconds(65);
-            Assert.True(await etlDone.WaitAsync(timeout), await GetPerformanceStats(mentorNode, dbName, timeout));
+            var timeout = TimeSpan.FromSeconds(30);
+            string originalPerformanceStats = null;
+
+            Assert.True(await etlDone.WaitAsync(timeout), originalPerformanceStats = await GetPerformanceStats(mentorNode, dbName, timeout));
 
             var files = Directory.GetFiles(path, "*.*", SearchOption.AllDirectories);
             Assert.Equal(1, files.Length);
@@ -151,7 +154,8 @@ await session.StoreAsync(new Order
                 await session.SaveChangesAsync();
             }
 
-            Assert.True(await etlDone.WaitAsync(timeout), await GetPerformanceStats(newResponsibleNode, dbName, timeout));
+            timeout = TimeSpan.FromSeconds(90);
+            Assert.True(await etlDone.WaitAsync(timeout), await AddDebugInfoToErrorMessage(newResponsibleNode, dbName, originalPerformanceStats, timeout));
 
             files = Directory.GetFiles(path, "*.*", SearchOption.AllDirectories);
             Assert.True(files.Length == 2, $"Expected 2 output files but got {files.Length}. " +
@@ -187,9 +191,32 @@ internal static AddEtlOperationResult AddOlapEtl(IDocumentStore src, OlapEtlConf
 
         private static async Task<string> GetPerformanceStats(RavenServer server, string database, TimeSpan timeout)
         {
+            var now = DateTime.UtcNow.ToString("o");
             var documentDatabase = await GetDatabase(server, database);
             var performanceStats = S3Tests.GetPerformanceStats(documentDatabase);
-            return $"olap etl to local machine did not finish in {timeout.TotalSeconds} seconds. stats : {performanceStats}";
+
+            var sb = new StringBuilder()
+                .Append("time: ").AppendLine(now)
+                .Append("responsible node:  ").AppendLine(server.ServerStore.NodeTag)
+                .AppendLine("ETL performance stats:")
+                .AppendLine(performanceStats);
+
+            return sb.ToString();
+        }
+
+        private static async Task<string> AddDebugInfoToErrorMessage(RavenServer responsibleNode, string database, string originalStats, TimeSpan timeout)
+        {
+            var newStats = await GetPerformanceStats(responsibleNode, database, timeout);
+
+            var sb = new StringBuilder()
+                .AppendLine($"OLAP ETL to local machine did not finish in {timeout.TotalSeconds} seconds")
+                .AppendLine("Debug info from the original responsible node:")
+                .AppendLine(originalStats)
+                .AppendLine()
+                .AppendLine("Debug info from the new responsible node:")
+                .AppendLine(newStats);
+
+            return sb.ToString();
         }
 
     }