Add ux for retry in dotnet test (#48075)

Author: nohwnd

src/Cli/dotnet/Commands/Test/LocalizableStrings.resx

@@ -534,4 +534,8 @@ dotnet.config is a name don't translate.</comment>
   <data name="CmdNumberName" xml:space="preserve">
     <value>NUMBER</value>
   </data>
+  <data name="Try" xml:space="preserve">
+    <value>try {0}</value>
+    <comment>number or tries of the current test assembly when test assembly is being retried. {0} is number that starts at 1</comment>
+  </data>
 </root>

src/Cli/dotnet/Commands/Test/Terminal/AnsiTerminalTestProgressFrame.cs

@@ -29,6 +29,7 @@ public void AppendTestWorkerProgress(TestProgressState progress, RenderedProgres
         int passed = progress.PassedTests;
         int failed = progress.FailedTests;
         int skipped = progress.SkippedTests;
+        int retried = progress.RetriedFailedTests;
         int charsTaken = 0;
 
         terminal.Append('[');
@@ -62,6 +63,20 @@ public void AppendTestWorkerProgress(TestProgressState progress, RenderedProgres
         terminal.Append(skippedText);
         charsTaken += skippedText.Length;
         terminal.ResetColor();
+
+        if (retried > 0)
+        {
+            terminal.Append('/');
+            charsTaken++;
+            terminal.SetColor(TerminalColor.Gray);
+            terminal.Append('r');
+            charsTaken++;
+            string retriedText = retried.ToString(CultureInfo.CurrentCulture);
+            terminal.Append(retriedText);
+            charsTaken += retriedText.Length;
+            terminal.ResetColor();
+        }
+
         terminal.Append(']');
         charsTaken++;
 

src/Cli/dotnet/Commands/Test/Terminal/NonAnsiTerminal.cs

@@ -177,6 +177,7 @@ public void RenderProgress(TestProgressState?[] progress)
             int passed = p.PassedTests;
             int failed = p.FailedTests;
             int skipped = p.SkippedTests;
+            int retried = p.RetriedFailedTests;
 
             // Use just ascii here, so we don't put too many restrictions on fonts needing to
             // properly show unicode, or logs being saved in particular encoding.
@@ -199,6 +200,15 @@ public void RenderProgress(TestProgressState?[] progress)
             Append('?');
             Append(skipped.ToString(CultureInfo.CurrentCulture));
             ResetColor();
+
+            if (retried > 0)
+            {
+                SetColor(TerminalColor.Gray);
+                Append('r');
+                Append(retried.ToString(CultureInfo.CurrentCulture));
+                ResetColor();
+            }
+
             Append(']');
 
             Append(' ');

src/Cli/dotnet/Commands/Test/Terminal/TerminalTestReporter.cs

@@ -5,6 +5,7 @@
 using System.CommandLine.Help;
 using System.Globalization;
 using System.Text.RegularExpressions;
+using Microsoft.CodeAnalysis;
 using Microsoft.DotNet.Cli.Commands.Test;
 using Microsoft.DotNet.Cli.Commands.Test.Terminal;
 using FlatException = Microsoft.DotNet.Cli.Commands.Test.Terminal.FlatException;
@@ -51,6 +52,7 @@ internal event EventHandler OnProgressStopUpdate
     private readonly uint? _originalConsoleMode;
     private bool _isDiscovery;
     private bool _isHelp;
+    private bool _isRetry;
     private DateTimeOffset? _testExecutionStartTime;
 
     private DateTimeOffset? _testExecutionEndTime;
@@ -152,28 +154,52 @@ public TerminalTestReporter(IConsole console, TerminalTestReporterOptions option
         _terminalWithProgress = terminalWithProgress;
     }
 
-    public void TestExecutionStarted(DateTimeOffset testStartTime, int workerCount, bool isDiscovery, bool isHelp)
+    public void TestExecutionStarted(DateTimeOffset testStartTime, int workerCount, bool isDiscovery, bool isHelp, bool isRetry)
     {
         _isDiscovery = isDiscovery;
         _isHelp = isHelp;
+        _isRetry = isRetry;
         _testExecutionStartTime = testStartTime;
         _terminalWithProgress.StartShowingProgress(workerCount);
     }
 
-    public void AssemblyRunStarted(string assembly, string? targetFramework, string? architecture, string? executionId)
+    public void AssemblyRunStarted(string assembly, string? targetFramework, string? architecture, string? executionId, string? instanceId)
     {
+        var assemblyRun = GetOrAddAssemblyRun(assembly, targetFramework, architecture, executionId);
+        assemblyRun.Tries.Add(instanceId);
+
+        if (_isRetry)
+        {
+            // When we are retrying the new assembly run should ignore all previously failed tests and
+            // clear all errors. We restarted the run and will retry all failed tests.
+            //
+            // In case of folded dynamic data tests we do not know how many tests we will get in each run,
+            // if more or less, or the same amount as before,and we also will rerun tests that passed previously
+            // because we are unable to run just a single test from that dynamic data source.
+            // This will cause the total number of tests to differ between runs, and there is nothing we can do about it.
+            assemblyRun.TotalTests -= assemblyRun.FailedTests;
+            assemblyRun.RetriedFailedTests += assemblyRun.FailedTests;
+            assemblyRun.FailedTests = 0;
+            assemblyRun.ClearAllMessages();
+        }
+
         if (_options.ShowAssembly && _options.ShowAssemblyStartAndComplete)
         {
             _terminalWithProgress.WriteToTerminal(terminal =>
             {
+                if (_isRetry)
+                {
+                    terminal.SetColor(TerminalColor.DarkGray);
+                    terminal.Append($"({string.Format(LocalizableStrings.Try, assemblyRun.Tries.Count)}) ");
+                    terminal.ResetColor();
+                }
+
                 terminal.Append(_isDiscovery ? LocalizableStrings.DiscoveringTestsFrom : LocalizableStrings.RunningTestsFrom);
                 terminal.Append(' ');
                 AppendAssemblyLinkTargetFrameworkAndArchitecture(terminal, assembly, targetFramework, architecture);
                 terminal.AppendLine();
             });
         }
-
-        GetOrAddAssemblyRun(assembly, targetFramework, architecture, executionId);
     }
 
     private TestProgressState GetOrAddAssemblyRun(string assembly, string? targetFramework, string? architecture, string? executionId)
@@ -306,6 +332,7 @@ private void AppendTestRunSummary(ITerminal terminal)
         int failed = _assemblies.Values.Sum(t => t.FailedTests);
         int passed = _assemblies.Values.Sum(t => t.PassedTests);
         int skipped = _assemblies.Values.Sum(t => t.SkippedTests);
+        int retried = _assemblies.Values.Sum(t => t.RetriedFailedTests);
         int error = _assemblies.Values.Sum(t => !t.Success && (t.TotalTests == 0 || t.FailedTests == 0) ? 1 : 0);
         TimeSpan runDuration = _testExecutionStartTime != null && _testExecutionEndTime != null ? (_testExecutionEndTime - _testExecutionStartTime).Value : TimeSpan.Zero;
 
@@ -316,6 +343,7 @@ private void AppendTestRunSummary(ITerminal terminal)
 
         string errorText = $"{SingleIndentation}error: {error}";
         string totalText = $"{SingleIndentation}total: {total}";
+        string retriedText = $" (+{retried} retried)";
         string failedText = $"{SingleIndentation}failed: {failed}";
         string passedText = $"{SingleIndentation}succeeded: {passed}";
         string skippedText = $"{SingleIndentation}skipped: {skipped}";
@@ -330,7 +358,15 @@ private void AppendTestRunSummary(ITerminal terminal)
         }
 
         terminal.ResetColor();
-        terminal.AppendLine(totalText);
+        terminal.Append(totalText);
+        if (retried > 0)
+        {
+            terminal.SetColor(TerminalColor.DarkGray);
+            terminal.Append(retriedText);
+            terminal.ResetColor();
+        }
+        terminal.AppendLine();
+
         if (colorizeFailed)
         {
             terminal.SetColor(TerminalColor.Red);
@@ -405,44 +441,12 @@ private static void AppendAssemblyResult(ITerminal terminal, bool succeeded, int
         }
     }
 
-    internal void TestCompleted(
-       string assembly,
-       string? targetFramework,
-       string? architecture,
-       string? executionId,
-       string testNodeUid,
-       string displayName,
-       TestOutcome outcome,
-       TimeSpan duration,
-       string? errorMessage,
-       Exception? exception,
-       string? expected,
-       string? actual,
-       string? standardOutput,
-       string? errorOutput)
-    {
-        FlatException[] flatExceptions = ExceptionFlattener.Flatten(errorMessage, exception);
-        TestCompleted(
-            assembly,
-            targetFramework,
-            architecture,
-            executionId,
-            testNodeUid,
-            displayName,
-            outcome,
-            duration,
-            flatExceptions,
-            expected,
-            actual,
-            standardOutput,
-            errorOutput);
-    }
-
     internal void TestCompleted(
         string assembly,
         string? targetFramework,
         string? architecture,
         string? executionId,
+        string instanceId,
         string testNodeUid,
         string displayName,
         TestOutcome outcome,
@@ -454,6 +458,7 @@ internal void TestCompleted(
         string? errorOutput)
     {
         TestProgressState asm = _assemblies[$"{assembly}|{targetFramework}|{architecture}|{executionId}"];
+        var attempt = asm.Tries.Count;
 
         if (_options.ShowActiveTests)
         {
@@ -479,12 +484,21 @@ internal void TestCompleted(
                 break;
         }
 
+        if (_isRetry && asm.Tries.Count > 1 && outcome == TestOutcome.Passed)
+        {
+            // This is a retry of a test, and the test succeeded, so these tests are potentially flaky.
+            // Tests that come from dynamic data sources and previously succeeded will also run on the second attempt,
+            // and most likely will succeed as well, so we will get them here, even though they are probably not flaky.
+            asm.FlakyTests.Add(testNodeUid);
+            
+        }
         _terminalWithProgress.UpdateWorker(asm.SlotIndex);
         if (outcome != TestOutcome.Passed || GetShowPassedTests())
         {
             _terminalWithProgress.WriteToTerminal(terminal => RenderTestCompleted(
                 terminal,
                 assembly,
+                attempt,
                 targetFramework,
                 architecture,
                 displayName,
@@ -507,6 +521,7 @@ private bool GetShowPassedTests()
     internal /* for testing */ void RenderTestCompleted(
         ITerminal terminal,
         string assembly,
+        int attempt,
         string? targetFramework,
         string? architecture,
         string displayName,
@@ -541,6 +556,11 @@ private bool GetShowPassedTests()
 
         terminal.SetColor(color);
         terminal.Append(outcomeText);
+        if (_isRetry)
+        {
+            terminal.SetColor(TerminalColor.DarkGray);
+            terminal.Append($" ({string.Format(LocalizableStrings.Try, attempt)})");
+        }
         terminal.ResetColor();
         terminal.Append(' ');
         terminal.Append(displayName);
@@ -1033,6 +1053,7 @@ public void TestInProgress(
         string? targetFramework,
         string? architecture,
         string testNodeUid,
+        string instanceId,
         string displayName,
         string? executionId)
     {

src/Cli/dotnet/Commands/Test/Terminal/TestProgressState.cs

@@ -27,6 +27,8 @@ internal sealed class TestProgressState(long id, string assembly, string? target
 
     public int TotalTests { get; internal set; }
 
+    public int RetriedFailedTests { get; internal set; }
+
     public TestNodeResultsState? TestNodeResultsState { get; internal set; }
 
     public int SlotIndex { get; internal set; }
@@ -39,9 +41,17 @@ internal sealed class TestProgressState(long id, string assembly, string? target
     public int? ExitCode { get; internal set; }
     public bool Success { get; internal set; }
 
+    public List<string> Tries { get; } = [];
+    public HashSet<string> FlakyTests { get; } = [];
+
     internal void AddError(string text)
         => Messages.Add(new ErrorMessage(text));
 
     internal void AddWarning(string text)
         => Messages.Add(new WarningMessage(text));
+
+    internal void ClearAllMessages()
+    {
+        Messages.Clear();
+    }
 }

src/Cli/dotnet/Commands/Test/TestApplicationEventHandlers.cs

@@ -9,18 +9,19 @@ namespace Microsoft.DotNet.Cli.Commands.Test;
 
 internal sealed class TestApplicationsEventHandlers(TerminalTestReporter output) : IDisposable
 {
-    private readonly ConcurrentDictionary<TestApplication, (string ModulePath, string TargetFramework, string Architecture, string ExecutionId)> _executions = new();
+    private readonly ConcurrentDictionary<TestApplication, (string ModulePath, string TargetFramework, string Architecture, string ExecutionId, string InstanceId)> _executions = new();
     private readonly TerminalTestReporter _output = output;
 
     public void OnHandshakeReceived(object sender, HandshakeArgs args)
     {
         var testApplication = (TestApplication)sender;
         var executionId = args.Handshake.Properties[HandshakeMessagePropertyNames.ExecutionId];
+        var instanceId = args.Handshake.Properties[HandshakeMessagePropertyNames.InstanceId];
         var arch = args.Handshake.Properties[HandshakeMessagePropertyNames.Architecture]?.ToLower();
         var tfm = TargetFrameworkParser.GetShortTargetFramework(args.Handshake.Properties[HandshakeMessagePropertyNames.Framework]);
-        (string ModulePath, string TargetFramework, string Architecture, string ExecutionId) appInfo = new(testApplication.Module.RunProperties.RunCommand, tfm, arch, executionId);
+        (string ModulePath, string TargetFramework, string Architecture, string ExecutionId, string InstanceId) appInfo = new(testApplication.Module.RunProperties.RunCommand, tfm, arch, executionId, instanceId);
         _executions[testApplication] = appInfo;
-        _output.AssemblyRunStarted(appInfo.ModulePath, appInfo.TargetFramework, appInfo.Architecture, appInfo.ExecutionId);
+        _output.AssemblyRunStarted(appInfo.ModulePath, appInfo.TargetFramework, appInfo.Architecture, appInfo.ExecutionId, appInfo.InstanceId);
 
         LogHandshake(args);
     }
@@ -63,6 +64,7 @@ public void OnTestResultsReceived(object sender, TestResultEventArgs args)
         foreach (var testResult in args.SuccessfulTestResults)
         {
             _output.TestCompleted(appInfo.ModulePath, appInfo.TargetFramework, appInfo.Architecture, appInfo.ExecutionId,
+                args.InstanceId,
                 testResult.Uid,
                 testResult.DisplayName,
                 ToOutcome(testResult.State),
@@ -76,7 +78,7 @@ public void OnTestResultsReceived(object sender, TestResultEventArgs args)
 
         foreach (var testResult in args.FailedTestResults)
         {
-            _output.TestCompleted(appInfo.ModulePath, appInfo.TargetFramework, appInfo.Architecture, appInfo.ExecutionId,
+            _output.TestCompleted(appInfo.ModulePath, appInfo.TargetFramework, appInfo.Architecture, appInfo.ExecutionId, args.InstanceId,
                 testResult.Uid,
                 testResult.DisplayName,
                 ToOutcome(testResult.State),

src/Cli/dotnet/Commands/Test/TestingPlatformCommand.cs

@@ -19,6 +19,7 @@ internal partial class TestingPlatformCommand : CliCommand, ICustomHelp
 
     private byte _cancelled;
     private bool _isDiscovery;
+    private bool _isRetry;
 
     public TestingPlatformCommand(string name, string description = null) : base(name, description)
     {
@@ -88,6 +89,9 @@ private void PrepareEnvironment(ParseResult parseResult, out TestOptions testOpt
         testOptions = GetTestOptions(parseResult, filterModeEnabled, isHelp: ContainsHelpOption(arguments));
 
         _isDiscovery = ContainsListTestsOption(arguments);
+
+        // This is ugly, and we need to replace it by passing out some info from testing platform to inform us that some process level retry plugin is active.
+        _isRetry = arguments.Contains("--retry-failed-tests");
     }
 
     private void InitializeActionQueue(int degreeOfParallelism, TestOptions testOptions, bool isHelp)
@@ -126,7 +130,7 @@ private void InitializeOutput(int degreeOfParallelism, ParseResult parseResult,
             ShowAssemblyStartAndComplete = true,
         });
 
-        _output.TestExecutionStarted(DateTimeOffset.Now, degreeOfParallelism, _isDiscovery, isHelp);
+        _output.TestExecutionStarted(DateTimeOffset.Now, degreeOfParallelism, _isDiscovery, isHelp, _isRetry);
     }
 
     private void InitializeHelpActionQueue(int degreeOfParallelism, TestOptions testOptions)