Fix missing outcomes.

xkbeyer · May 15, 2020 · f4b727e · f4b727e
1 parent 3a775d8
commit f4b727e
Show file tree

Hide file tree

Showing 4 changed files with 231 additions and 61 deletions.
diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
-[![Github Releases](https://img.shields.io/github/release/xkbeyer/CatchTestAdapter/all.svg?label=pre-release)](https://github.com/xkbeyer/CatchTestAdapter/releases)
 [![Github Releases](https://img.shields.io/github/release/xkbeyer/CatchTestAdapter.svg)](https://github.com/xkbeyer/CatchTestAdapter/releases)
+[![MIT license](http://img.shields.io/badge/license-MIT-brightgreen.svg)](http://opensource.org/licenses/MIT)
 # CatchTestAdapter
 A Visual Studio Extension to run [Catch2](https://github.com/catchorg/Catch2) unit tests within the Visual Studio TestExplorer. 
 
@@ -8,26 +8,25 @@ Use the latest [CatchTestAdapter.vsix](https://github.com/xkbeyer/CatchTestAdapt
 It can be installed by double clicking on the downloaded file.
 
 ### Visual Studio Compatibility
-v1.6.x: Visual Studio 2019 v16.2 and newer 
-
+##### v1.6.x: Works with Visual Studio 2019 v16.2 and newer 
 Beginning with Visual Studio 2019 v16.2 the CatchTestAdapter 1.5.1 is broken. 
-The new TextExplorer Window doesn't accept new test cases as a sub test case.
-Therefore Catch SECTIONs are no longer shown as test cases after the discovery phase. 
-They are shown as sub results.
+The new TestExplorer Window doesn't accept new test cases as a sub test case.
+As a result the Catch2 `SECTION`s are no longer shown as sub test cases after the first run. 
+Now they are shown as sub results of a test case.
 
-v1.5.1: Visual Studio prior v16.2 
+##### v1.5.1: Works with Visual Studio prior to v16.2 
 
 ### Status
 
 - Test cases are shown after discovery process.
 - Stack trace link to the source line.
-- Catch2 TAGS are implemented as Traits.
-- SECTION and SCENARIO are shown as sub results.
+- Catch2 `TAGS` are implemented as Traits.
+- `SECTION` and `SCENARIO` are shown as sub results.
 
 ### Testing
 To run the unit tests against the `CatchTestAdapter.dll` of the solution, the `Local.runsettings` file must be loaded.
 The `TestAdaptersPaths` should be adapted to point to the Solution directory.
-```
+```xml
 <TestAdaptersPaths>c:\Path\to\the\Solution\bin\Debug</TestAdaptersPaths> 
 ```
 

diff --git a/TestAdapter/CatchTestCase.cs b/TestAdapter/CatchTestCase.cs
@@ -42,6 +42,8 @@ public TestCase[] Sections {
         }
         [XmlElement("OverallResult", typeof(OverallResult))]
         public OverallResult Result { get; set; }
+        [XmlElement("OverallResults", typeof(OverallResults), IsNullable = true)]   
+        public OverallResults Results { get; set; }
     }
     public class Failure
     {
@@ -75,4 +77,15 @@ public class OverallResult
         [XmlAttribute("durationInSeconds")]
         public string Duration = "";
     }
+    public class OverallResults
+    {
+        [XmlAttribute("successes")]
+        public string Successes = "";
+        [XmlAttribute("failures")]
+        public string Failures = "";
+        [XmlAttribute("expectedFailures")]
+        public string ExpectedFailures = "";
+        [XmlAttribute("durationInSeconds")]
+        public string Duration = "";
+    }
 }
diff --git a/TestAdapter/TestExecutor.cs b/TestAdapter/TestExecutor.cs
@@ -151,11 +151,21 @@ private void CreateResult(Tests.TestCase element, TestCase testCase, string name
                 DisplayName = name.Replace(".", "\n\t"),
                 ErrorMessage = $"",
                 ErrorStackTrace = "",
+                Outcome = TestOutcome.None
             };
+
             if (element.Result != null)
+            {
+                subResult.Outcome = element.Result.Success == "true" ? TestOutcome.Passed : TestOutcome.Failed;
                 subResult.Duration = TimeSpan.FromSeconds(Double.Parse(element.Result.Duration, CultureInfo.InvariantCulture));
+            }
 
             int failedExpressions = ConstructResult(element.Expressions, subResult);
+            // Make sure the outcome is failed if the expressions have failed.
+            if (failedExpressions != 0)
+            {
+                subResult.Outcome = TestOutcome.Failed;
+            }
 
             foreach (var s in (element.Warning ?? new string[] { }))
             {
@@ -179,7 +189,18 @@ private void CreateResult(Tests.TestCase element, TestCase testCase, string name
                 subResult.ErrorStackTrace += $"at #{failedExpressions} - {name}() in {FilePath}:line {LineNumber}{Environment.NewLine}";
             }
 
-            subResult.Outcome = failedExpressions == 0 ? TestOutcome.Passed : TestOutcome.Failed;
+            if( subResult.Outcome == TestOutcome.None )
+            {
+                // Check if the OverallResults is set with an outcome.
+                if (element.Results != null)
+                {
+                    if (Int32.Parse(element.Results.Failures) != 0)
+                        subResult.Outcome = TestOutcome.Failed;
+                    else
+                        subResult.Outcome = TestOutcome.Passed;
+                }
+            }
+
             results.Add(subResult);
 
             // Try to find the failure from a subsection of this element.