diff --git a/README.md b/README.md
index c152529..7414479 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
-[](https://github.com/xkbeyer/CatchTestAdapter/releases)
[](https://github.com/xkbeyer/CatchTestAdapter/releases)
+[](http://opensource.org/licenses/MIT)
# CatchTestAdapter
A Visual Studio Extension to run [Catch2](https://github.com/catchorg/Catch2) unit tests within the Visual Studio TestExplorer.
@@ -8,26 +8,25 @@ Use the latest [CatchTestAdapter.vsix](https://github.com/xkbeyer/CatchTestAdapt
It can be installed by double clicking on the downloaded file.
### Visual Studio Compatibility
-v1.6.x: Visual Studio 2019 v16.2 and newer
-
+##### v1.6.x: Works with Visual Studio 2019 v16.2 and newer
Beginning with Visual Studio 2019 v16.2 the CatchTestAdapter 1.5.1 is broken.
-The new TextExplorer Window doesn't accept new test cases as a sub test case.
-Therefore Catch SECTIONs are no longer shown as test cases after the discovery phase.
-They are shown as sub results.
+The new TestExplorer Window doesn't accept new test cases as a sub test case.
+As a result the Catch2 `SECTION`s are no longer shown as sub test cases after the first run.
+Now they are shown as sub results of a test case.
-v1.5.1: Visual Studio prior v16.2
+##### v1.5.1: Works with Visual Studio prior to v16.2
### Status
- Test cases are shown after discovery process.
- Stack trace link to the source line.
-- Catch2 TAGS are implemented as Traits.
-- SECTION and SCENARIO are shown as sub results.
+- Catch2 `TAGS` are implemented as Traits.
+- `SECTION` and `SCENARIO` are shown as sub results.
### Testing
To run the unit tests against the `CatchTestAdapter.dll` of the solution, the `Local.runsettings` file must be loaded.
The `TestAdaptersPaths` should be adapted to point to the Solution directory.
-```
+```xml
c:\Path\to\the\Solution\bin\Debug
```
diff --git a/TestAdapter/CatchTestCase.cs b/TestAdapter/CatchTestCase.cs
index 08e47d0..eeaa108 100644
--- a/TestAdapter/CatchTestCase.cs
+++ b/TestAdapter/CatchTestCase.cs
@@ -42,6 +42,8 @@ public TestCase[] Sections {
}
[XmlElement("OverallResult", typeof(OverallResult))]
public OverallResult Result { get; set; }
+ [XmlElement("OverallResults", typeof(OverallResults), IsNullable = true)]
+ public OverallResults Results { get; set; }
}
public class Failure
{
@@ -75,4 +77,15 @@ public class OverallResult
[XmlAttribute("durationInSeconds")]
public string Duration = "";
}
+ public class OverallResults
+ {
+ [XmlAttribute("successes")]
+ public string Successes = "";
+ [XmlAttribute("failures")]
+ public string Failures = "";
+ [XmlAttribute("expectedFailures")]
+ public string ExpectedFailures = "";
+ [XmlAttribute("durationInSeconds")]
+ public string Duration = "";
+ }
}
diff --git a/TestAdapter/TestExecutor.cs b/TestAdapter/TestExecutor.cs
index a39b174..f1e65e5 100644
--- a/TestAdapter/TestExecutor.cs
+++ b/TestAdapter/TestExecutor.cs
@@ -151,11 +151,21 @@ private void CreateResult(Tests.TestCase element, TestCase testCase, string name
DisplayName = name.Replace(".", "\n\t"),
ErrorMessage = $"",
ErrorStackTrace = "",
+ Outcome = TestOutcome.None
};
+
if (element.Result != null)
+ {
+ subResult.Outcome = element.Result.Success == "true" ? TestOutcome.Passed : TestOutcome.Failed;
subResult.Duration = TimeSpan.FromSeconds(Double.Parse(element.Result.Duration, CultureInfo.InvariantCulture));
+ }
int failedExpressions = ConstructResult(element.Expressions, subResult);
+ // Make sure the outcome is failed if the expressions have failed.
+ if (failedExpressions != 0)
+ {
+ subResult.Outcome = TestOutcome.Failed;
+ }
foreach (var s in (element.Warning ?? new string[] { }))
{
@@ -179,7 +189,18 @@ private void CreateResult(Tests.TestCase element, TestCase testCase, string name
subResult.ErrorStackTrace += $"at #{failedExpressions} - {name}() in {FilePath}:line {LineNumber}{Environment.NewLine}";
}
- subResult.Outcome = failedExpressions == 0 ? TestOutcome.Passed : TestOutcome.Failed;
+ if( subResult.Outcome == TestOutcome.None )
+ {
+ // Check if the OverallResults is set with an outcome.
+ if (element.Results != null)
+ {
+ if (Int32.Parse(element.Results.Failures) != 0)
+ subResult.Outcome = TestOutcome.Failed;
+ else
+ subResult.Outcome = TestOutcome.Passed;
+ }
+ }
+
results.Add(subResult);
// Try to find the failure from a subsection of this element.
diff --git a/TestAdapterTest/TestTestExecutor.cs b/TestAdapterTest/TestTestExecutor.cs
index 442e166..15ff54d 100644
--- a/TestAdapterTest/TestTestExecutor.cs
+++ b/TestAdapterTest/TestTestExecutor.cs
@@ -135,17 +135,48 @@ public void WarningAndInfoMessage()
}
+ [TestMethod]
+ public void OneTestResult()
+ {
+ // Set up a fake testing context.
+ var framework = new MockFrameworkHandle();
+
+ // Execute all tests.
+ TestExecutor executor = new TestExecutor();
+ IList tests = new List();
+ tests.Add(new TestCase("Has forced failure", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject") { CodeFilePath = @"ReferenceCatchProject\Tests.cpp", LineNumber = 37 });
+ executor.RunTests(tests, new MockRunContext(), framework);
+ Assert.AreEqual(2, framework.Results.Count);
+ Assert.IsTrue(framework.Results.All(r => r.Outcome == TestOutcome.Failed));
+ }
+
+ [TestMethod]
+ public void SubResult()
+ {
+ // Set up a fake testing context.
+ var framework = new MockFrameworkHandle();
+
+ // Execute all tests.
+ TestExecutor executor = new TestExecutor();
+ executor.RunTests(Common.ReferenceExeList, new MockRunContext(), framework);
+ Assert.IsTrue(framework.Results.Count != 0);
+ var resultsOfFoo = framework.Results.Where(r => r.TestCase.DisplayName == "Foo");
+ Assert.AreEqual(5, resultsOfFoo.Count());
+ Assert.IsTrue(resultsOfFoo.All(r => r.Outcome == TestOutcome.Failed));
+ }
+
[TestMethod]
public void BrokenXmlWithSingleTest()
{
var framework = new MockFrameworkHandle();
var runcontext = new MockRunContext();
var executor = new MockTestExecutor();
- IList xml_output = new List() { @"",
-@"",
-@" ",
-@" "
-};
+ IList xml_output = new List() {
+ @"",
+ @"",
+ @" ",
+ @" "
+ };
IList tests = new List();
tests.Add(new TestCase("C++ assert", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject") { CodeFilePath = "ReferenceCatchProject\testrunnertest.cpp", LineNumber = 45 });
executor.MockComposeResults(xml_output, tests, framework);
@@ -159,50 +190,51 @@ public void BrokenXml()
var framework = new MockFrameworkHandle();
var runcontext = new MockRunContext();
var executor = new MockTestExecutor();
- IList xml_output = new List() { @"",
-@"",
-@" ",
-@" ",
-@" ",
-@" ",
-@" 3 == 4",
-@" ",
-@" ",
-@" 3 == 4",
-@" ",
-@" ",
-@" ",
-@" ",
-@" 9 == 0 ",
-@" ",
-@" ",
-@" 9 == 0 ",
-@" ",
-@" ",
-@" ",
-@" ",
-@" ",
-@" ",
-@" ",
-@" a == b ",
-@" ",
-@" ",
-@" 5 == 6 ",
-@" ",
-@" ",
-@" ",
-@" ",
-@" ",
-@" ",
-@" ",
-@" Fail check",
-@" ",
-@" ",
-@" ",
-@" ",
-};
+ IList xml_output = new List() {
+ @"",
+ @"",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" 3 == 4",
+ @" ",
+ @" ",
+ @" 3 == 4",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" 9 == 0 ",
+ @" ",
+ @" ",
+ @" 9 == 0 ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" a == b ",
+ @" ",
+ @" ",
+ @" 5 == 6 ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" Fail check",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ };
IList tests = new List();
tests.Add(new TestCase("Simple test case", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject") { CodeFilePath = "ReferenceCatchProject\testrunnertest.cpp", LineNumber = 15 });
tests.Add(new TestCase("Another test case", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject") { CodeFilePath = "ReferenceCatchProject\testrunnertest.cpp", LineNumber = 23 });
@@ -211,7 +243,7 @@ public void BrokenXml()
tests.Add(new TestCase("Last test case", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject") { CodeFilePath = "ReferenceCatchProject\testrunnertest.cpp", LineNumber = 54 });
executor.MockComposeResults(xml_output, tests, framework);
Assert.AreEqual(6, framework.Results.Count);
- for (int i = 0; i < 3; ++i)
+ for (int i = 0; i < 4; ++i)
{
Assert.AreNotEqual(TestOutcome.None, framework.Results[i].Outcome);
}
@@ -290,5 +322,110 @@ public void XmlReadTest()
Assert.AreEqual(TestOutcome.Failed, framework.Results[i].Outcome);
}
}
+
+ [TestMethod]
+ public void XmlFooTest()
+ {
+ var framework = new MockFrameworkHandle();
+ var runcontext = new MockRunContext();
+ var executor = new MockTestExecutor();
+ IList xml_output = new List() {
+ @"",
+ @"",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" x < 100",
+ @" ",
+ @" ",
+ @" 168 < 100",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" x == 42",
+ @" ",
+ @" ",
+ @" 43 == 42",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" x == 42",
+ @" ",
+ @" ",
+ @" 44 == 42",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" x == 42",
+ @" ",
+ @" ",
+ @" 43 == 42",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @""
+ };
+
+ IList tests = new List();
+ tests.Add(new TestCase("Foo", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject") { CodeFilePath = @"ReferenceCatchProject\Tests.cpp", LineNumber = 59 });
+ executor.MockComposeResults(xml_output, tests, framework);
+ var results = framework.Results.ToArray();
+ Assert.AreEqual(5, framework.Results.Count);
+ Assert.IsTrue(results.All(r => r.Outcome == TestOutcome.Failed));
+ }
+
+ [TestMethod]
+ public void XmlPassedTestWithInfo()
+ {
+ var framework = new MockFrameworkHandle();
+ var runcontext = new MockRunContext();
+ var executor = new MockTestExecutor();
+ IList xml_output = new List() {
+ @"",
+ @"",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" expected == actual",
+ @" ",
+ @" ",
+ @" true == true",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @" ",
+ @""
+ };
+ IList tests = new List();
+ tests.Add(new TestCase("First fixture", new Uri(TestExecutor.ExecutorUriString), "ReferenceCatchProject") { CodeFilePath = @"ReferenceCatchProject\fixture_test.cpp", LineNumber = 13 });
+ executor.MockComposeResults(xml_output, tests, framework);
+ var results = framework.Results.ToArray();
+ Assert.AreEqual(1, framework.Results.Count);
+ Assert.AreEqual(TestOutcome.Passed, results[0].Outcome);
+ }
}
}