From 51b29ced1ac2bde83d977283f6e0b4f871898b8a Mon Sep 17 00:00:00 2001 From: sp-dani-garcia Date: Thu, 29 Aug 2019 17:47:56 +0200 Subject: [PATCH] Add SonarQube Generic Test Data reporter It outputs reports in the `Generic Execution Test Data` format, see https://docs.sonarqube.org/latest/analysis/generic-test/, specifically https://docs.sonarqube.org/latest/analysis/generic-test/#header-2 Close #1738 (this is a cherry-pick and fixup of that PR) --- codecov.yml | 1 + docs/ci-and-misc.md | 6 +- docs/release-process.md | 4 +- docs/reporters.md | 1 + .../reporters/catch_reporter_sonarqube.hpp | 181 ++ projects/CMakeLists.txt | 1 + .../Baselines/sonarqube.sw.approved.txt | 1730 +++++++++++++++++ projects/SelfTest/TestMain.cpp | 1 + scripts/approvalTests.py | 4 + scripts/releaseCommon.py | 2 +- 10 files changed, 1927 insertions(+), 4 deletions(-) create mode 100644 include/reporters/catch_reporter_sonarqube.hpp create mode 100644 projects/SelfTest/Baselines/sonarqube.sw.approved.txt diff --git a/codecov.yml b/codecov.yml index 94d88d8331..75809ee2e8 100644 --- a/codecov.yml +++ b/codecov.yml @@ -14,6 +14,7 @@ coverage: - "**/catch_reporter_tap.hpp" - "**/catch_reporter_automake.hpp" - "**/catch_reporter_teamcity.hpp" + - "**/catch_reporter_sonarqube.hpp" - "**/external/clara.hpp" diff --git a/docs/ci-and-misc.md b/docs/ci-and-misc.md index 8c33087256..40b7cec927 100644 --- a/docs/ci-and-misc.md +++ b/docs/ci-and-misc.md @@ -12,7 +12,7 @@ Build Systems may refer to low-level tools, like CMake, or larger systems that r ## Continuous Integration systems -Probably the most important aspect to using Catch with a build server is the use of different reporters. Catch comes bundled with three reporters that should cover the majority of build servers out there - although adding more for better integration with some is always a possibility (currently we also offer TeamCity, TAP and Automake reporters). +Probably the most important aspect to using Catch with a build server is the use of different reporters. Catch comes bundled with three reporters that should cover the majority of build servers out there - although adding more for better integration with some is always a possibility (currently we also offer TeamCity, TAP, Automake and SonarQube reporters). Two of these reporters are built in (XML and JUnit) and the third (TeamCity) is included as a separate header. It's possible that the other two may be split out in the future too - as that would make the core of Catch smaller for those that don't need them. @@ -65,6 +65,10 @@ The Automake Reporter writes out the [meta tags](https://www.gnu.org/software/au Because of the incremental nature of Catch's test suites and ability to run specific tests, our implementation of TAP reporter writes out the number of tests in a suite last. +### SonarQube Reporter +```-r sonarqube``` +[SonarQube Generic Test Data](https://docs.sonarqube.org/latest/analysis/generic-test/) XML format for tests metrics. + ## Low-level tools ### Precompiled headers (PCHs) diff --git a/docs/release-process.md b/docs/release-process.md index 7fa5f7e170..ca48da0330 100644 --- a/docs/release-process.md +++ b/docs/release-process.md @@ -42,8 +42,8 @@ Tag version and release title should be same as the new version, description should contain the release notes for the current release. Single header version of `catch.hpp` *needs* to be attached as a binary, as that is where the official download link links to. Preferably -it should use linux line endings. All non-bundled reporters (Automake, -TAP, TeamCity) should also be attached as binaries, as they might be +it should use linux line endings. All non-bundled reporters (Automake, TAP, +TeamCity, SonarQube) should also be attached as binaries, as they might be dependent on a specific version of the single-include header. Since 2.5.0, the release tag and the "binaries" (headers) should be PGP diff --git a/docs/reporters.md b/docs/reporters.md index 32b3419b5a..a33e55bf11 100644 --- a/docs/reporters.md +++ b/docs/reporters.md @@ -29,6 +29,7 @@ Do this in one source file - the same one you have `CATCH_CONFIG_MAIN` or `CATCH Use this when building as part of a TeamCity build to see results as they happen ([code example](../examples/207-Rpt-TeamCityReporter.cpp)). * `tap` writes in the TAP ([Test Anything Protocol](https://en.wikipedia.org/wiki/Test_Anything_Protocol)) format. * `automake` writes in a format that correspond to [automake .trs](https://www.gnu.org/software/automake/manual/html_node/Log-files-generation-and-test-results-recording.html) files +* `sonarqube` writes the [SonarQube Generic Test Data](https://docs.sonarqube.org/latest/analysis/generic-test/) XML format. You see what reporters are available from the command line by running with `--list-reporters`. diff --git a/include/reporters/catch_reporter_sonarqube.hpp b/include/reporters/catch_reporter_sonarqube.hpp new file mode 100644 index 0000000000..b860293cf7 --- /dev/null +++ b/include/reporters/catch_reporter_sonarqube.hpp @@ -0,0 +1,181 @@ +/* + * Created by Daniel Garcia on 2018-12-04. + * Copyright Social Point SL. All rights reserved. + * + * Distributed under the Boost Software License, Version 1.0. (See accompanying + * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + */ +#ifndef CATCH_REPORTER_SONARQUBE_HPP_INCLUDED +#define CATCH_REPORTER_SONARQUBE_HPP_INCLUDED + + +// Don't #include any Catch headers here - we can assume they are already +// included before this header. +// This is not good practice in general but is necessary in this case so this +// file can be distributed as a single header that works with the main +// Catch single header. + +#include + +namespace Catch { + + struct SonarQubeReporter : CumulativeReporterBase { + + SonarQubeReporter(ReporterConfig const& config) + : CumulativeReporterBase(config) + , xml(config.stream()) { + m_reporterPrefs.shouldRedirectStdOut = true; + m_reporterPrefs.shouldReportAllAssertions = true; + } + + ~SonarQubeReporter() override; + + static std::string getDescription() { + return "Reports test results in the Generic Test Data SonarQube XML format"; + } + + static std::set getSupportedVerbosities() { + return { Verbosity::Normal }; + } + + void noMatchingTestCases(std::string const& /*spec*/) override {} + + void testRunStarting(TestRunInfo const& testRunInfo) override { + CumulativeReporterBase::testRunStarting(testRunInfo); + xml.startElement("testExecutions"); + xml.writeAttribute("version", "1"); + } + + void testGroupEnded(TestGroupStats const& testGroupStats) override { + CumulativeReporterBase::testGroupEnded(testGroupStats); + writeGroup(*m_testGroups.back()); + } + + void testRunEndedCumulative() override { + xml.endElement(); + } + + void writeGroup(TestGroupNode const& groupNode) { + std::map testsPerFile; + for(auto const& child : groupNode.children) + testsPerFile[child->value.testInfo.lineInfo.file].push_back(child); + + for(auto const& kv : testsPerFile) + writeTestFile(kv.first.c_str(), kv.second); + } + + void writeTestFile(const char* filename, TestGroupNode::ChildNodes const& testCaseNodes) { + XmlWriter::ScopedElement e = xml.scopedElement("file"); + xml.writeAttribute("path", filename); + + for(auto const& child : testCaseNodes) + writeTestCase(*child); + } + + void writeTestCase(TestCaseNode const& testCaseNode) { + // All test cases have exactly one section - which represents the + // test case itself. That section may have 0-n nested sections + assert(testCaseNode.children.size() == 1); + SectionNode const& rootSection = *testCaseNode.children.front(); + writeSection("", rootSection, testCaseNode.value.testInfo.okToFail()); + } + + void writeSection(std::string const& rootName, SectionNode const& sectionNode, bool okToFail) { + std::string name = trim(sectionNode.stats.sectionInfo.name); + if(!rootName.empty()) + name = rootName + '/' + name; + + if(!sectionNode.assertions.empty() || !sectionNode.stdOut.empty() || !sectionNode.stdErr.empty()) { + XmlWriter::ScopedElement e = xml.scopedElement("testCase"); + xml.writeAttribute("name", name); + xml.writeAttribute("duration", static_cast(sectionNode.stats.durationInSeconds * 1000)); + + writeAssertions(sectionNode, okToFail); + } + + for(auto const& childNode : sectionNode.childSections) + writeSection(name, *childNode, okToFail); + } + + void writeAssertions(SectionNode const& sectionNode, bool okToFail) { + for(auto const& assertion : sectionNode.assertions) + writeAssertion( assertion, okToFail); + } + + void writeAssertion(AssertionStats const& stats, bool okToFail) { + AssertionResult const& result = stats.assertionResult; + if(!result.isOk()) { + std::string elementName; + if(okToFail) { + elementName = "skipped"; + } + else { + switch(result.getResultType()) { + case ResultWas::ThrewException: + case ResultWas::FatalErrorCondition: + elementName = "error"; + break; + case ResultWas::ExplicitFailure: + elementName = "failure"; + break; + case ResultWas::ExpressionFailed: + elementName = "failure"; + break; + case ResultWas::DidntThrowException: + elementName = "failure"; + break; + + // We should never see these here: + case ResultWas::Info: + case ResultWas::Warning: + case ResultWas::Ok: + case ResultWas::Unknown: + case ResultWas::FailureBit: + case ResultWas::Exception: + elementName = "internalError"; + break; + } + } + + XmlWriter::ScopedElement e = xml.scopedElement(elementName); + + ReusableStringStream messageRss; + messageRss << result.getTestMacroName() << "(" << result.getExpression() << ")"; + xml.writeAttribute("message", messageRss.str()); + + ReusableStringStream textRss; + if (stats.totals.assertions.total() > 0) { + textRss << "FAILED:\n"; + if (result.hasExpression()) { + textRss << "\t" << result.getExpressionInMacro() << "\n"; + } + if (result.hasExpandedExpression()) { + textRss << "with expansion:\n\t" << result.getExpandedExpression() << "\n"; + } + } + + if(!result.getMessage().empty()) + textRss << result.getMessage() << "\n"; + + for(auto const& msg : stats.infoMessages) + if(msg.type == ResultWas::Info) + textRss << msg.message << "\n"; + + textRss << "at " << result.getSourceInfo(); + xml.writeText(textRss.str(), false); + } + } + + private: + XmlWriter xml; + }; + +#ifdef CATCH_IMPL + SonarQubeReporter::~SonarQubeReporter() {} +#endif + + CATCH_REGISTER_REPORTER( "sonarqube", SonarQubeReporter ) + +} // end namespace Catch + +#endif // CATCH_REPORTER_SONARQUBE_HPP_INCLUDED \ No newline at end of file diff --git a/projects/CMakeLists.txt b/projects/CMakeLists.txt index cbe21fcff5..0aafc1780d 100644 --- a/projects/CMakeLists.txt +++ b/projects/CMakeLists.txt @@ -282,6 +282,7 @@ set(REPORTER_HEADERS ${HEADER_DIR}/reporters/catch_reporter_tap.hpp ${HEADER_DIR}/reporters/catch_reporter_teamcity.hpp ${HEADER_DIR}/reporters/catch_reporter_xml.h + ${HEADER_DIR}/reporters/catch_reporter_sonarqube.hpp ) set(REPORTER_SOURCES ${HEADER_DIR}/reporters/catch_reporter_bases.cpp diff --git a/projects/SelfTest/Baselines/sonarqube.sw.approved.txt b/projects/SelfTest/Baselines/sonarqube.sw.approved.txt new file mode 100644 index 0000000000..65f4655af7 --- /dev/null +++ b/projects/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -0,0 +1,1730 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +FAILED: + REQUIRE( s == "world" ) +with expansion: + "hello" == "world" +Class.tests.cpp: + + + + + +FAILED: + REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) +with expansion: + 0 == 1 +Class.tests.cpp: + + + + +FAILED: + REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) +with expansion: + 0 == 1 +Class.tests.cpp: + + + + +FAILED: + REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) +with expansion: + 0 == 1 +Class.tests.cpp: + + + + +FAILED: + REQUIRE( Template_Fixture_2<TestType>::m_a.size() == 1 ) +with expansion: + 0 == 1 +Class.tests.cpp: + + + + + + + + +FAILED: + REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) +with expansion: + 6 < 2 +Class.tests.cpp: + + + + +FAILED: + REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) +with expansion: + 2 < 2 +Class.tests.cpp: + + + + +FAILED: + REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) +with expansion: + 6 < 2 +Class.tests.cpp: + + + + +FAILED: + REQUIRE( Template_Fixture_2<TestType>{}.m_a.size() < 2 ) +with expansion: + 2 < 2 +Class.tests.cpp: + + + + + + + + +FAILED: + REQUIRE( Template_Fixture<TestType>::m_a == 2 ) +with expansion: + 1.0 == 2 +Class.tests.cpp: + + + + +FAILED: + REQUIRE( Template_Fixture<TestType>::m_a == 2 ) +with expansion: + 1.0f == 2 +Class.tests.cpp: + + + + +FAILED: + REQUIRE( Template_Fixture<TestType>::m_a == 2 ) +with expansion: + 1 == 2 +Class.tests.cpp: + + + + + + + +FAILED: + REQUIRE( Nttp_Fixture<V>::value == 0 ) +with expansion: + 1 == 0 +Class.tests.cpp: + + + + +FAILED: + REQUIRE( Nttp_Fixture<V>::value == 0 ) +with expansion: + 3 == 0 +Class.tests.cpp: + + + + +FAILED: + REQUIRE( Nttp_Fixture<V>::value == 0 ) +with expansion: + 6 == 0 +Class.tests.cpp: + + + + + + + +FAILED: + REQUIRE( m_a == 2 ) +with expansion: + 1 == 2 +Class.tests.cpp: + + + + + + + + + + + + + + + + + + + + + + + +FAILED: + CHECK( false != false ) +Condition.tests.cpp: + + +FAILED: + CHECK( true != true ) +Condition.tests.cpp: + + +FAILED: + CHECK( !true ) +with expansion: + false +Condition.tests.cpp: + + +FAILED: + CHECK_FALSE( true ) +with expansion: + !true +Condition.tests.cpp: + + +FAILED: + CHECK( !trueValue ) +with expansion: + false +Condition.tests.cpp: + + +FAILED: + CHECK_FALSE( trueValue ) +with expansion: + !true +Condition.tests.cpp: + + +FAILED: + CHECK( !(1 == 1) ) +with expansion: + false +Condition.tests.cpp: + + +FAILED: + CHECK_FALSE( 1 == 1 ) +Condition.tests.cpp: + + + + + + + + +FAILED: + CHECK( data.int_seven == 6 ) +with expansion: + 7 == 6 +Condition.tests.cpp: + + +FAILED: + CHECK( data.int_seven == 8 ) +with expansion: + 7 == 8 +Condition.tests.cpp: + + +FAILED: + CHECK( data.int_seven == 0 ) +with expansion: + 7 == 0 +Condition.tests.cpp: + + +FAILED: + CHECK( data.float_nine_point_one == Approx( 9.11f ) ) +with expansion: + 9.1f == Approx( 9.1099996567 ) +Condition.tests.cpp: + + +FAILED: + CHECK( data.float_nine_point_one == Approx( 9.0f ) ) +with expansion: + 9.1f == Approx( 9.0 ) +Condition.tests.cpp: + + +FAILED: + CHECK( data.float_nine_point_one == Approx( 1 ) ) +with expansion: + 9.1f == Approx( 1.0 ) +Condition.tests.cpp: + + +FAILED: + CHECK( data.float_nine_point_one == Approx( 0 ) ) +with expansion: + 9.1f == Approx( 0.0 ) +Condition.tests.cpp: + + +FAILED: + CHECK( data.double_pi == Approx( 3.1415 ) ) +with expansion: + 3.1415926535 == Approx( 3.1415 ) +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello == "goodbye" ) +with expansion: + "hello" == "goodbye" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello == "hell" ) +with expansion: + "hello" == "hell" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello == "hello1" ) +with expansion: + "hello" == "hello1" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello.size() == 6 ) +with expansion: + 5 == 6 +Condition.tests.cpp: + + +FAILED: + CHECK( x == Approx( 1.301 ) ) +with expansion: + 1.3 == Approx( 1.301 ) +Condition.tests.cpp: + + + + + +FAILED: + CHECK( data.int_seven != 7 ) +with expansion: + 7 != 7 +Condition.tests.cpp: + + +FAILED: + CHECK( data.float_nine_point_one != Approx( 9.1f ) ) +with expansion: + 9.1f != Approx( 9.1000003815 ) +Condition.tests.cpp: + + +FAILED: + CHECK( data.double_pi != Approx( 3.1415926535 ) ) +with expansion: + 3.1415926535 != Approx( 3.1415926535 ) +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello != "hello" ) +with expansion: + "hello" != "hello" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello.size() != 5 ) +with expansion: + 5 != 5 +Condition.tests.cpp: + + + + + +FAILED: + CHECK( data.int_seven > 7 ) +with expansion: + 7 > 7 +Condition.tests.cpp: + + +FAILED: + CHECK( data.int_seven < 7 ) +with expansion: + 7 < 7 +Condition.tests.cpp: + + +FAILED: + CHECK( data.int_seven > 8 ) +with expansion: + 7 > 8 +Condition.tests.cpp: + + +FAILED: + CHECK( data.int_seven < 6 ) +with expansion: + 7 < 6 +Condition.tests.cpp: + + +FAILED: + CHECK( data.int_seven < 0 ) +with expansion: + 7 < 0 +Condition.tests.cpp: + + +FAILED: + CHECK( data.int_seven < -1 ) +with expansion: + 7 < -1 +Condition.tests.cpp: + + +FAILED: + CHECK( data.int_seven >= 8 ) +with expansion: + 7 >= 8 +Condition.tests.cpp: + + +FAILED: + CHECK( data.int_seven <= 6 ) +with expansion: + 7 <= 6 +Condition.tests.cpp: + + +FAILED: + CHECK( data.float_nine_point_one < 9 ) +with expansion: + 9.1f < 9 +Condition.tests.cpp: + + +FAILED: + CHECK( data.float_nine_point_one > 10 ) +with expansion: + 9.1f > 10 +Condition.tests.cpp: + + +FAILED: + CHECK( data.float_nine_point_one > 9.2 ) +with expansion: + 9.1f > 9.2 +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello > "hello" ) +with expansion: + "hello" > "hello" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello < "hello" ) +with expansion: + "hello" < "hello" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello > "hellp" ) +with expansion: + "hello" > "hellp" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello > "z" ) +with expansion: + "hello" > "z" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello < "hellm" ) +with expansion: + "hello" < "hellm" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello < "a" ) +with expansion: + "hello" < "a" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello >= "z" ) +with expansion: + "hello" >= "z" +Condition.tests.cpp: + + +FAILED: + CHECK( data.str_hello <= "a" ) +with expansion: + "hello" <= "a" +Condition.tests.cpp: + + + + + + + + + + + +FAILED: + CHECK( truthy(false) ) +with expansion: + Hey, its truthy! +Decomposition.tests.cpp: + + + + + + + + + + + + + + +FAILED: +expected exception +answer := 42 +Exception.tests.cpp: + + + + +FAILED: + REQUIRE_NOTHROW( thisThrows() ) +expected exception +answer := 42 +Exception.tests.cpp: + + + + + +FAILED: + {Unknown expression after the reported line} +unexpected exception +Exception.tests.cpp: + + + + +FAILED: + REQUIRE_NOTHROW( throwCustom() ) +custom exception - not std +Exception.tests.cpp: + + + + +FAILED: + REQUIRE_THROWS_AS( throwCustom(), std::exception ) +custom exception - not std +Exception.tests.cpp: + + + + +FAILED: +custom std exception +Exception.tests.cpp: + + + + + + + +FAILED: + CHECK_THROWS_AS( thisThrows(), std::string ) +expected exception +Exception.tests.cpp: + + +FAILED: + CHECK_THROWS_AS( thisDoesntThrow(), std::domain_error ) +Exception.tests.cpp: + + +FAILED: + CHECK_NOTHROW( thisThrows() ) +expected exception +Exception.tests.cpp: + + + + +FAILED: + REQUIRE_THROWS_WITH( thisThrows(), "should fail" ) +with expansion: + "expected exception" equals: "should fail" +Exception.tests.cpp: + + + + +FAILED: +custom exception +Exception.tests.cpp: + + + + +FAILED: +For some reason someone is throwing a string literal! +Exception.tests.cpp: + + + + +FAILED: +3.14 +Exception.tests.cpp: + + + + + +FAILED: +unexpected exception +Exception.tests.cpp: + + + + +FAILED: + CHECK( thisThrows() == 0 ) +expected exception +Exception.tests.cpp: + + + + +FAILED: + REQUIRE( thisThrows() == 0 ) +expected exception +Exception.tests.cpp: + + + + +FAILED: + CHECK( thisThrows() == 0 ) +expected exception +Exception.tests.cpp: + + + + +FAILED: +unexpected exception +Exception.tests.cpp: + + + + +FAILED: +Why would you throw a std::string? +Exception.tests.cpp: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +FAILED: + CHECK_THAT( testStringForMatching(), Contains("not there", Catch::CaseSensitive::No) ) +with expansion: + "this string contains 'abc' as a substring" contains: "not there" (case insensitive) +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( testStringForMatching(), Contains("STRING") ) +with expansion: + "this string contains 'abc' as a substring" contains: "STRING" +Matchers.tests.cpp: + + + + +FAILED: + CHECK_THAT( testStringForMatching(), EndsWith("Substring") ) +with expansion: + "this string contains 'abc' as a substring" ends with: "Substring" +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( testStringForMatching(), EndsWith("this", Catch::CaseSensitive::No) ) +with expansion: + "this string contains 'abc' as a substring" ends with: "this" (case insensitive) +Matchers.tests.cpp: + + + + + +FAILED: + CHECK_THAT( testStringForMatching(), Equals("this string contains 'ABC' as a substring") ) +with expansion: + "this string contains 'abc' as a substring" equals: "this string contains 'ABC' as a substring" +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( testStringForMatching(), Equals("something else", Catch::CaseSensitive::No) ) +with expansion: + "this string contains 'abc' as a substring" equals: "something else" (case insensitive) +Matchers.tests.cpp: + + + + +FAILED: + CHECK_THROWS_MATCHES( doesNotThrow(), SpecialException, ExceptionMatcher{1} ) +Matchers.tests.cpp: + + +FAILED: + REQUIRE_THROWS_MATCHES( doesNotThrow(), SpecialException, ExceptionMatcher{1} ) +Matchers.tests.cpp: + + + + +FAILED: + CHECK_THROWS_MATCHES( throwsAsInt(1), SpecialException, ExceptionMatcher{1} ) +Unknown exception +Matchers.tests.cpp: + + +FAILED: + REQUIRE_THROWS_MATCHES( throwsAsInt(1), SpecialException, ExceptionMatcher{1} ) +Unknown exception +Matchers.tests.cpp: + + + + +FAILED: + CHECK_THROWS_MATCHES( throwsSpecialException(3), SpecialException, ExceptionMatcher{1} ) +with expansion: + SpecialException::what special exception has value of 1 +Matchers.tests.cpp: + + +FAILED: + REQUIRE_THROWS_MATCHES( throwsSpecialException(4), SpecialException, ExceptionMatcher{1} ) +with expansion: + SpecialException::what special exception has value of 1 +Matchers.tests.cpp: + + + + + + + + + + + + + + + + + + + + + +FAILED: + CHECK_THAT( testStringForMatching(), (Contains("string") || Contains("different")) && Contains("random") ) +with expansion: + "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "random" ) +Matchers.tests.cpp: + + + + + +FAILED: + CHECK_THAT( testStringForMatching(), !Contains("substring") ) +with expansion: + "this string contains 'abc' as a substring" not contains: "substring" +Matchers.tests.cpp: + + + + + +FAILED: + CHECK_THAT( testStringForMatching(), Matches("this STRING contains 'abc' as a substring") ) +with expansion: + "this string contains 'abc' as a substring" matches "this STRING contains 'abc' as a substring" case sensitively +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( testStringForMatching(), Matches("contains 'abc' as a substring") ) +with expansion: + "this string contains 'abc' as a substring" matches "contains 'abc' as a substring" case sensitively +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( testStringForMatching(), Matches("this string contains 'abc' as a") ) +with expansion: + "this string contains 'abc' as a substring" matches "this string contains 'abc' as a" case sensitively +Matchers.tests.cpp: + + + + + +FAILED: + CHECK_THAT( testStringForMatching(), StartsWith("This String") ) +with expansion: + "this string contains 'abc' as a substring" starts with: "This String" +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( testStringForMatching(), StartsWith("string", Catch::CaseSensitive::No) ) +with expansion: + "this string contains 'abc' as a substring" starts with: "string" (case insensitive) +Matchers.tests.cpp: + + + + + + + + + +FAILED: + CHECK_THAT( empty, Approx(t1) ) +with expansion: + { } is approx: { 1.0, 2.0 } +Matchers.tests.cpp: + + + + +FAILED: + CHECK_THAT( v1, Approx(v2) ) +with expansion: + { 2.0, 4.0, 6.0 } is approx: { 1.0, 3.0, 5.0 } +Matchers.tests.cpp: + + + + + + + + + +FAILED: + CHECK_THAT( v, VectorContains(-1) ) +with expansion: + { 1, 2, 3 } Contains: -1 +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( empty, VectorContains(1) ) +with expansion: + { } Contains: 1 +Matchers.tests.cpp: + + + + +FAILED: + CHECK_THAT( empty, Contains(v) ) +with expansion: + { } Contains: { 1, 2, 3 } +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( v, Contains(v2) ) +with expansion: + { 1, 2, 3 } Contains: { 1, 2, 4 } +Matchers.tests.cpp: + + + + +FAILED: + CHECK_THAT( v, Equals(v2) ) +with expansion: + { 1, 2, 3 } Equals: { 1, 2 } +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( v2, Equals(v) ) +with expansion: + { 1, 2 } Equals: { 1, 2, 3 } +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( empty, Equals(v) ) +with expansion: + { } Equals: { 1, 2, 3 } +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( v, Equals(empty) ) +with expansion: + { 1, 2, 3 } Equals: { } +Matchers.tests.cpp: + + + + +FAILED: + CHECK_THAT( v, UnorderedEquals(empty) ) +with expansion: + { 1, 2, 3 } UnorderedEquals: { } +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( empty, UnorderedEquals(v) ) +with expansion: + { } UnorderedEquals: { 1, 2, 3 } +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( permuted, UnorderedEquals(v) ) +with expansion: + { 1, 3 } UnorderedEquals: { 1, 2, 3 } +Matchers.tests.cpp: + + +FAILED: + CHECK_THAT( permuted, UnorderedEquals(v) ) +with expansion: + { 3, 1 } UnorderedEquals: { 1, 2, 3 } +Matchers.tests.cpp: + + + + + + + + + + +FAILED: +This is a failure +Message.tests.cpp: + + + + +FAILED: +Message.tests.cpp: + + + + +FAILED: +This is a failure +Message.tests.cpp: + + + + + +FAILED: + REQUIRE( a == 1 ) +with expansion: + 2 == 1 +this message should be logged +so should this +Message.tests.cpp: + + + + +FAILED: + CHECK( a == 1 ) +with expansion: + 2 == 1 +this message may be logged later +this message should be logged +Message.tests.cpp: + + +FAILED: + CHECK( a == 0 ) +with expansion: + 2 == 0 +this message may be logged later +this message should be logged +and this, but later +Message.tests.cpp: + + + + +FAILED: + REQUIRE( i < 10 ) +with expansion: + 10 < 10 +current counter 10 +i := 10 +Message.tests.cpp: + + + + +FAILED: +Message from section one +Message.tests.cpp: + + + + +FAILED: +Message from section two +Message.tests.cpp: + + + + + + + + +FAILED: +Previous info should not be seen +Message.tests.cpp: + + + + +FAILED: +previous unscoped info SHOULD not be seen +Message.tests.cpp: + + + + + +FAILED: + REQUIRE( false ) +this SHOULD be seen +Message.tests.cpp: + + + + + +FAILED: + REQUIRE( false ) +this SHOULD be seen +this SHOULD also be seen +Message.tests.cpp: + + + + +FAILED: + CHECK( false ) +this SHOULD be seen only ONCE +Message.tests.cpp: + + + + +FAILED: + REQUIRE( false ) +hi +i := 7 +Message.tests.cpp: + + + + +FAILED: + CHECK( false ) +Count 1 to 3... +1 +2 +3 +Message.tests.cpp: + + +FAILED: + CHECK( false ) +Count 4 to 6... +4 +5 +6 +Message.tests.cpp: + + + + + + + + +FAILED: + CHECK( f() == 0 ) +with expansion: + 1 == 0 +Misc.tests.cpp: + + + + + + + + + + + + + + + + + +FAILED: +to infinity and beyond +Misc.tests.cpp: + + + + + + + + + + + + +FAILED: + CHECK( s1 == s2 ) +with expansion: + "if ($b == 10) { + $a = 20; +}" +== +"if ($b == 10) { + $a = 20; +} +" +Misc.tests.cpp: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +FAILED: + CHECKED_ELSE( flag ) +with expansion: + false +Misc.tests.cpp: + + +FAILED: + REQUIRE( testCheckedElse( false ) ) +with expansion: + false +Misc.tests.cpp: + + + + + +FAILED: + CHECKED_IF( flag ) +with expansion: + false +Misc.tests.cpp: + + +FAILED: + REQUIRE( testCheckedIf( false ) ) +with expansion: + false +Misc.tests.cpp: + + + + + + + + +FAILED: + CHECK( b > a ) +with expansion: + 0 > 1 +Misc.tests.cpp: + + + + +FAILED: + CHECK( b > a ) +with expansion: + 1 > 1 +Misc.tests.cpp: + + + + + + + + + + + + +FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +Testing if fib[0] (1) is even +Misc.tests.cpp: + + +FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +Testing if fib[1] (1) is even +Misc.tests.cpp: + + +FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +Testing if fib[3] (3) is even +Misc.tests.cpp: + + +FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +Testing if fib[4] (5) is even +Misc.tests.cpp: + + +FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +Testing if fib[6] (13) is even +Misc.tests.cpp: + + +FAILED: + CHECK( ( fib[i] % 2 ) == 0 ) +with expansion: + 1 == 0 +Testing if fib[7] (21) is even +Misc.tests.cpp: + + + + +FAILED: + REQUIRE( a == b ) +with expansion: + 1 == 2 +Misc.tests.cpp: + + + + + + + + + + + + +FAILED: + REQUIRE( false ) +3 +Misc.tests.cpp: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +FAILED: +1514 +Tricky.tests.cpp: + + + + + + + + + +FAILED: + CHECK( &o1 == &o2 ) +with expansion: + 0x == 0x +Tricky.tests.cpp: + + +FAILED: + CHECK( o1 == o2 ) +with expansion: + {?} == {?} +Tricky.tests.cpp: + + + + + + + + + + + + + + + + + + + + + + + + +FAILED: + REQUIRE( std::string( "first" ) == "second" ) +with expansion: + "first" == "second" +Tricky.tests.cpp: + + + + + + + + + diff --git a/projects/SelfTest/TestMain.cpp b/projects/SelfTest/TestMain.cpp index 1c023ce91f..a16f0c22bd 100644 --- a/projects/SelfTest/TestMain.cpp +++ b/projects/SelfTest/TestMain.cpp @@ -13,6 +13,7 @@ #include "reporters/catch_reporter_teamcity.hpp" #include "reporters/catch_reporter_tap.hpp" #include "reporters/catch_reporter_automake.hpp" +#include "reporters/catch_reporter_sonarqube.hpp" // Some example tag aliases diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py index bb01e6d570..dad2a96b7f 100755 --- a/scripts/approvalTests.py +++ b/scripts/approvalTests.py @@ -29,6 +29,7 @@ lineNumberParser = re.compile(r' line="[0-9]*"') hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b') durationsParser = re.compile(r' time="[0-9]*\.[0-9]*"') +sonarqubeDurationParser = re.compile(r' duration="[0-9]+"') timestampsParser = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}Z') versionParser = re.compile(r'Catch v[0-9]+\.[0-9]+\.[0-9]+(-develop\.[0-9]+)?') nullParser = re.compile(r'\b(__null|nullptr)\b') @@ -138,6 +139,7 @@ def filterLine(line, isCompact): # strip durations and timestamps line = durationsParser.sub(' time="{duration}"', line) + line = sonarqubeDurationParser.sub(' duration="{duration}"', line) line = timestampsParser.sub('{iso8601-timestamp}', line) line = specialCaseParser.sub('file:\g<1>', line) line = errnoParser.sub('errno', line) @@ -204,6 +206,8 @@ def approve(baseName, args): approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex", "--rng-seed", "1"]) # compact reporter, include passes, warn about No Assertions approve('compact.sw', ['~[!nonportable]~[!benchmark]~[approvals]', '-s', '-w', 'NoAssertions', '-r', 'compact', '--order', 'lex', "--rng-seed", "1"]) +# sonarqube reporter, include passes, warn about No Assertions +approve("sonarqube.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "sonarqube", "--order", "lex", "--rng-seed", "1"]) if overallResult != 0: print("If these differences are expected, run approve.py to approve new baselines.") diff --git a/scripts/releaseCommon.py b/scripts/releaseCommon.py index 102fe46a09..283337d409 100644 --- a/scripts/releaseCommon.py +++ b/scripts/releaseCommon.py @@ -156,7 +156,7 @@ def performUpdates(version): # We probably should have some kind of convention to select which reporters need to be copied automagically, # but this works for now import shutil - for rep in ('automake', 'tap', 'teamcity'): + for rep in ('automake', 'tap', 'teamcity', 'sonarqube'): sourceFile = os.path.join(catchPath, 'include/reporters/catch_reporter_{}.hpp'.format(rep)) destFile = os.path.join(catchPath, 'single_include', 'catch2', 'catch_reporter_{}.hpp'.format(rep)) shutil.copyfile(sourceFile, destFile)