dotnet test for MTP: Support TestTfmsInParallel

src/Cli/dotnet/Commands/Test/CliConstants.cs

@@ -95,4 +95,6 @@ internal static class ProjectProperties
     internal const string RunArguments = "RunArguments";
     internal const string RunWorkingDirectory = "RunWorkingDirectory";
     internal const string AppDesignerFolder = "AppDesignerFolder";
+    internal const string TestTfmsInParallel = "TestTfmsInParallel";
+    internal const string BuildInParallel = "BuildInParallel";
 }

src/Cli/dotnet/Commands/Test/MSBuildHandler.cs

@@ -2,20 +2,19 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Collections.Concurrent;
-using System.Diagnostics;
 using Microsoft.DotNet.Cli.Commands.Test.Terminal;
 using Microsoft.DotNet.Cli.Utils;
 using Microsoft.Testing.Platform.OutputDevice.Terminal;
 
 namespace Microsoft.DotNet.Cli.Commands.Test;
 
-internal sealed class MSBuildHandler(BuildOptions buildOptions, TestApplicationActionQueue actionQueue, TerminalTestReporter output) : IDisposable
+internal sealed class MSBuildHandler(BuildOptions buildOptions, TestApplicationActionQueue actionQueue, TerminalTestReporter output)
 {
     private readonly BuildOptions _buildOptions = buildOptions;
     private readonly TestApplicationActionQueue _actionQueue = actionQueue;
     private readonly TerminalTestReporter _output = output;
 
-    private readonly ConcurrentBag<TestApplication> _testApplications = [];
+    private readonly ConcurrentBag<ParallelizableTestModuleGroupWithSequentialInnerModules> _testApplications = [];
     private bool _areTestingPlatformApplications = true;
 
     public bool RunMSBuild()
@@ -64,7 +63,7 @@ private int RunBuild(string directoryPath)
             return ExitCode.GenericFailure;
         }
 
-        (IEnumerable<TestModule> projects, bool restored) = GetProjectsProperties(projectOrSolutionFilePath, isSolution);
+        (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> projects, bool restored) = GetProjectsProperties(projectOrSolutionFilePath, isSolution);
 
         InitializeTestApplications(projects);
 
@@ -73,17 +72,17 @@ private int RunBuild(string directoryPath)
 
     private int RunBuild(string filePath, bool isSolution)
     {
-        (IEnumerable<TestModule> projects, bool restored) = GetProjectsProperties(filePath, isSolution);
+        (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> projects, bool restored) = GetProjectsProperties(filePath, isSolution);
 
         InitializeTestApplications(projects);
 
         return restored ? ExitCode.Success : ExitCode.GenericFailure;
     }
 
-    private void InitializeTestApplications(IEnumerable<TestModule> modules)
+    private void InitializeTestApplications(IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> moduleGroups)
     {
         // If one test app has IsTestingPlatformApplication set to false (VSTest and not MTP), then we will not run any of the test apps
-        IEnumerable<TestModule> vsTestTestProjects = modules.Where(module => !module.IsTestingPlatformApplication);
+        IEnumerable<TestModule> vsTestTestProjects = moduleGroups.SelectMany(group => group.GetVSTestAndNotMTPModules());
 
         if (vsTestTestProjects.Any())
         {
@@ -100,16 +99,9 @@ private void InitializeTestApplications(IEnumerable<TestModule> modules)
             return;
         }
 
-        foreach (TestModule module in modules)
+        foreach (ParallelizableTestModuleGroupWithSequentialInnerModules moduleGroup in moduleGroups)
         {
-            if (!module.IsTestProject && !module.IsTestingPlatformApplication)
-            {
-                // This should never happen. We should only ever create TestModule if it's a test project.
-                throw new UnreachableException($"This program location is thought to be unreachable. Class='{nameof(MSBuildHandler)}' Method='{nameof(InitializeTestApplications)}'");
-            }
-
-            var testApp = new TestApplication(module, _buildOptions);
-            _testApplications.Add(testApp);
+            _testApplications.Add(moduleGroup);
         }
     }
 
@@ -127,9 +119,9 @@ public bool EnqueueTestApplications()
         return true;
     }
 
-    private (IEnumerable<TestModule> Projects, bool Restored) GetProjectsProperties(string solutionOrProjectFilePath, bool isSolution)
+    private (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> Projects, bool Restored) GetProjectsProperties(string solutionOrProjectFilePath, bool isSolution)
     {
-        (IEnumerable<TestModule> projects, bool isBuiltOrRestored) = isSolution ?
+        (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> projects, bool isBuiltOrRestored) = isSolution ?
             MSBuildUtility.GetProjectsFromSolution(solutionOrProjectFilePath, _buildOptions) :
             MSBuildUtility.GetProjectsFromProject(solutionOrProjectFilePath, _buildOptions);
 
@@ -138,7 +130,7 @@ public bool EnqueueTestApplications()
         return (projects, isBuiltOrRestored);
     }
 
-    private static void LogProjectProperties(IEnumerable<TestModule> modules)
+    private static void LogProjectProperties(IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> moduleGroups)
     {
         if (!Logger.TraceEnabled)
         {
@@ -147,26 +139,21 @@ private static void LogProjectProperties(IEnumerable<TestModule> modules)
 
         var logMessageBuilder = new StringBuilder();
 
-        foreach (var module in modules)
+        foreach (var moduleGroup in moduleGroups)
         {
-            logMessageBuilder.AppendLine($"{ProjectProperties.ProjectFullPath}: {module.ProjectFullPath}");
-            logMessageBuilder.AppendLine($"{ProjectProperties.IsTestProject}: {module.IsTestProject}");
-            logMessageBuilder.AppendLine($"{ProjectProperties.IsTestingPlatformApplication}: {module.IsTestingPlatformApplication}");
-            logMessageBuilder.AppendLine($"{ProjectProperties.TargetFramework}: {module.TargetFramework}");
-            logMessageBuilder.AppendLine($"{ProjectProperties.RunCommand}: {module.RunProperties.RunCommand}");
-            logMessageBuilder.AppendLine($"{ProjectProperties.RunArguments}: {module.RunProperties.RunArguments}");
-            logMessageBuilder.AppendLine($"{ProjectProperties.RunWorkingDirectory}: {module.RunProperties.RunWorkingDirectory}");
-            logMessageBuilder.AppendLine();
+            foreach (var module in moduleGroup)
+            {
+                logMessageBuilder.AppendLine($"{ProjectProperties.ProjectFullPath}: {module.ProjectFullPath}");
+                logMessageBuilder.AppendLine($"{ProjectProperties.IsTestProject}: {module.IsTestProject}");
+                logMessageBuilder.AppendLine($"{ProjectProperties.IsTestingPlatformApplication}: {module.IsTestingPlatformApplication}");
+                logMessageBuilder.AppendLine($"{ProjectProperties.TargetFramework}: {module.TargetFramework}");
+                logMessageBuilder.AppendLine($"{ProjectProperties.RunCommand}: {module.RunProperties.RunCommand}");
+                logMessageBuilder.AppendLine($"{ProjectProperties.RunArguments}: {module.RunProperties.RunArguments}");
+                logMessageBuilder.AppendLine($"{ProjectProperties.RunWorkingDirectory}: {module.RunProperties.RunWorkingDirectory}");
+                logMessageBuilder.AppendLine();
+            }
         }
 
         Logger.LogTrace(() => logMessageBuilder.ToString());
     }
-
-    public void Dispose()
-    {
-        foreach (var testApplication in _testApplications)
-        {
-            testApplication.Dispose();
-        }
-    }
 }

src/Cli/dotnet/Commands/Test/MSBuildUtility.cs

@@ -17,15 +17,15 @@ internal static class MSBuildUtility
 {
     private const string dotnetTestVerb = "dotnet-test";
 
-    public static (IEnumerable<TestModule> Projects, bool IsBuiltOrRestored) GetProjectsFromSolution(string solutionFilePath, BuildOptions buildOptions)
+    public static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> Projects, bool IsBuiltOrRestored) GetProjectsFromSolution(string solutionFilePath, BuildOptions buildOptions)
     {
         SolutionModel solutionModel = SlnFileFactory.CreateFromFileOrDirectory(solutionFilePath, includeSolutionFilterFiles: true, includeSolutionXmlFiles: true);
 
         bool isBuiltOrRestored = BuildOrRestoreProjectOrSolution(solutionFilePath, buildOptions);
 
         if (!isBuiltOrRestored)
         {
-            return (Array.Empty<TestModule>(), isBuiltOrRestored);
+            return (Array.Empty<ParallelizableTestModuleGroupWithSequentialInnerModules>(), isBuiltOrRestored);
         }
 
         string rootDirectory = solutionFilePath.HasExtension(".slnf") ?
@@ -35,25 +35,25 @@ public static (IEnumerable<TestModule> Projects, bool IsBuiltOrRestored) GetProj
         FacadeLogger? logger = LoggerUtility.DetermineBinlogger([.. buildOptions.MSBuildArgs], dotnetTestVerb);
         var collection = new ProjectCollection(globalProperties: CommonRunHelpers.GetGlobalPropertiesFromArgs([.. buildOptions.MSBuildArgs]), loggers: logger is null ? null : [logger], toolsetDefinitionLocations: ToolsetDefinitionLocations.Default);
 
-        ConcurrentBag<TestModule> projects = GetProjectsProperties(collection, solutionModel.SolutionProjects.Select(p => Path.Combine(rootDirectory, p.FilePath)), buildOptions);
+        ConcurrentBag<ParallelizableTestModuleGroupWithSequentialInnerModules> projects = GetProjectsProperties(collection, solutionModel.SolutionProjects.Select(p => Path.Combine(rootDirectory, p.FilePath)), buildOptions);
         logger?.ReallyShutdown();
 
         return (projects, isBuiltOrRestored);
     }
 
-    public static (IEnumerable<TestModule> Projects, bool IsBuiltOrRestored) GetProjectsFromProject(string projectFilePath, BuildOptions buildOptions)
+    public static (IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> Projects, bool IsBuiltOrRestored) GetProjectsFromProject(string projectFilePath, BuildOptions buildOptions)
     {
         bool isBuiltOrRestored = BuildOrRestoreProjectOrSolution(projectFilePath, buildOptions);
 
         if (!isBuiltOrRestored)
         {
-            return (Array.Empty<TestModule>(), isBuiltOrRestored);
+            return (Array.Empty<ParallelizableTestModuleGroupWithSequentialInnerModules>(), isBuiltOrRestored);
         }
 
         FacadeLogger? logger = LoggerUtility.DetermineBinlogger([.. buildOptions.MSBuildArgs], dotnetTestVerb);
         var collection = new ProjectCollection(globalProperties: CommonRunHelpers.GetGlobalPropertiesFromArgs([.. buildOptions.MSBuildArgs]), logger is null ? null : [logger], toolsetDefinitionLocations: ToolsetDefinitionLocations.Default);
 
-        IEnumerable<TestModule> projects = SolutionAndProjectUtility.GetProjectProperties(projectFilePath, collection, buildOptions.NoLaunchProfile);
+        IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> projects = SolutionAndProjectUtility.GetProjectProperties(projectFilePath, collection, buildOptions.NoLaunchProfile);
         logger?.ReallyShutdown();
 
         return (projects, isBuiltOrRestored);
@@ -117,16 +117,16 @@ private static bool BuildOrRestoreProjectOrSolution(string filePath, BuildOption
         return result == (int)BuildResultCode.Success;
     }
 
-    private static ConcurrentBag<TestModule> GetProjectsProperties(ProjectCollection projectCollection, IEnumerable<string> projects, BuildOptions buildOptions)
+    private static ConcurrentBag<ParallelizableTestModuleGroupWithSequentialInnerModules> GetProjectsProperties(ProjectCollection projectCollection, IEnumerable<string> projects, BuildOptions buildOptions)
     {
-        var allProjects = new ConcurrentBag<TestModule>();
+        var allProjects = new ConcurrentBag<ParallelizableTestModuleGroupWithSequentialInnerModules>();
 
         Parallel.ForEach(
             projects,
             new ParallelOptions { MaxDegreeOfParallelism = buildOptions.DegreeOfParallelism },
             (project) =>
             {
-                IEnumerable<TestModule> projectsMetadata = SolutionAndProjectUtility.GetProjectProperties(project, projectCollection, buildOptions.NoLaunchProfile);
+                IEnumerable<ParallelizableTestModuleGroupWithSequentialInnerModules> projectsMetadata = SolutionAndProjectUtility.GetProjectProperties(project, projectCollection, buildOptions.NoLaunchProfile);
                 foreach (var projectMetadata in projectsMetadata)
                 {
                     allProjects.Add(projectMetadata);

src/Cli/dotnet/Commands/Test/Models.cs

@@ -1,11 +1,113 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections;
 using Microsoft.DotNet.Cli.Commands.Run;
 using Microsoft.DotNet.Cli.Commands.Run.LaunchSettings;
 
 namespace Microsoft.DotNet.Cli.Commands.Test;
 
+/// <summary>
+/// Groups test modules that should NOT be run in parallel.
+/// The whole group is still parallelizable with other groups, just that the inner modules are run sequentially.
+/// This class serves only the purpose of disabling parallelizing of TFMs when user sets TestTfmsInParallel to false.
+/// For a single TFM project, we will use the constructor with a single module. Meaning it will be parallelized.
+/// For a multi TFM project:
+/// - If parallelization is enabled, we will create multiple ParallelizableTestModuleGroupWithSequentialInnerModuless, each with a single module.
+/// - If parallelization is not enabled, we will create a single ParallelizableTestModuleGroupWithSequentialInnerModules with all modules.
+/// </summary>
+internal sealed class ParallelizableTestModuleGroupWithSequentialInnerModules : IEnumerable<TestModule>
+{
+    public ParallelizableTestModuleGroupWithSequentialInnerModules(List<TestModule> modules)
+    {
+        Modules = modules;
+    }
+
+    public ParallelizableTestModuleGroupWithSequentialInnerModules(TestModule module)
+    {
+        // This constructor is used when there is only one module.
+        Module = module;
+    }
+
+    public List<TestModule>? Modules { get; }
+
+    public TestModule? Module { get; }
+
+    public TestModule[] GetVSTestAndNotMTPModules()
+    {
+        if (Modules is not null)
+        {
+            return Modules.Where(module => !module.IsTestingPlatformApplication).ToArray();
+        }
+
+        if (!Module.IsTestingPlatformApplication)
+        {
+            return [Module];
+        }
+
+        return Array.Empty<TestModule>();
+    }
+
+    public Enumerator GetEnumerator()
+        => new Enumerator(this);
+
+    IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+
+    IEnumerator<TestModule> IEnumerable<TestModule>.GetEnumerator() => GetEnumerator();
+
+    internal struct Enumerator : IEnumerator<TestModule>
+    {
+        private readonly ParallelizableTestModuleGroupWithSequentialInnerModules _group;
+        private int _index = -1;
+
+        public Enumerator(ParallelizableTestModuleGroupWithSequentialInnerModules group)
+        {
+            _group = group;
+        }
+
+        public TestModule Current
+        {
+            get
+            {
+                if (_index < 0)
+                {
+                    throw new InvalidOperationException();
+                }
+
+                if (_group.Modules is not null)
+                {
+                    return _group.Modules[_index];
+                }
+
+                if (_index != 0)
+                {
+                    throw new InvalidOperationException();
+                }
+
+                return _group.Module;
+            }
+        }
+
+        object IEnumerator.Current => Current;
+
+        public void Dispose() { }
+
+        public bool MoveNext()
+        {
+            _index++;
+
+            if (_group.Modules is not null)
+            {
+                return _index < _group.Modules.Count;
+            }
+
+            return _index == 0;
+        }
+
+        public void Reset() => _index = -1;
+    }
+}
+
 internal sealed record TestModule(RunProperties RunProperties, string? ProjectFullPath, string? TargetFramework, bool IsTestingPlatformApplication, bool IsTestProject, ProjectLaunchSettingsModel? LaunchSettings);
 
 internal sealed record Handshake(Dictionary<byte, string>? Properties);