1 #include "catch_run_context.h" 2 #include "catch_compiler_capabilities.h" 3 #include "catch_context.h" 4 #include "catch_enforce.h" 5 #include "catch_random_number_generator.h" 6 #include "catch_stream.h" 7 #include "catch_output_redirect.h" 8 9 #include <cassert> 10 #include <algorithm> 11 #include <sstream> 12 13 namespace Catch { 14 15 namespace Generators { 16 struct GeneratorTracker : TestCaseTracking::TrackerBase, IGeneratorTracker { 17 GeneratorBasePtr m_generator; 18 GeneratorTrackerCatch::Generators::GeneratorTracker19 GeneratorTracker( TestCaseTracking::NameAndLocation const& nameAndLocation, TrackerContext& ctx, ITracker* parent ) 20 : TrackerBase( nameAndLocation, ctx, parent ) 21 {} 22 ~GeneratorTracker(); 23 acquireCatch::Generators::GeneratorTracker24 static GeneratorTracker& acquire( TrackerContext& ctx, TestCaseTracking::NameAndLocation const& nameAndLocation ) { 25 std::shared_ptr<GeneratorTracker> tracker; 26 27 ITracker& currentTracker = ctx.currentTracker(); 28 if( TestCaseTracking::ITrackerPtr childTracker = currentTracker.findChild( nameAndLocation ) ) { 29 assert( childTracker ); 30 assert( childTracker->isGeneratorTracker() ); 31 tracker = std::static_pointer_cast<GeneratorTracker>( childTracker ); 32 } 33 else { 34 tracker = std::make_shared<GeneratorTracker>( nameAndLocation, ctx, ¤tTracker ); 35 currentTracker.addChild( tracker ); 36 } 37 38 if( !ctx.completedCycle() && !tracker->isComplete() ) { 39 tracker->open(); 40 } 41 42 return *tracker; 43 } 44 45 // TrackerBase interface isGeneratorTrackerCatch::Generators::GeneratorTracker46 bool isGeneratorTracker() const override { return true; } hasGeneratorCatch::Generators::GeneratorTracker47 auto hasGenerator() const -> bool override { 48 return !!m_generator; 49 } closeCatch::Generators::GeneratorTracker50 void close() override { 51 TrackerBase::close(); 52 // Generator interface only finds out if it has another item on atual move 53 if (m_runState == CompletedSuccessfully && m_generator->next()) { 54 m_children.clear(); 55 m_runState = Executing; 56 } 57 } 58 59 // IGeneratorTracker interface getGeneratorCatch::Generators::GeneratorTracker60 auto getGenerator() const -> GeneratorBasePtr const& override { 61 return m_generator; 62 } setGeneratorCatch::Generators::GeneratorTracker63 void setGenerator( GeneratorBasePtr&& generator ) override { 64 m_generator = std::move( generator ); 65 } 66 }; ~GeneratorTracker()67 GeneratorTracker::~GeneratorTracker() {} 68 } 69 RunContext(IConfigPtr const & _config,IStreamingReporterPtr && reporter)70 RunContext::RunContext(IConfigPtr const& _config, IStreamingReporterPtr&& reporter) 71 : m_runInfo(_config->name()), 72 m_context(getCurrentMutableContext()), 73 m_config(_config), 74 m_reporter(std::move(reporter)), 75 m_lastAssertionInfo{ StringRef(), SourceLineInfo("",0), StringRef(), ResultDisposition::Normal }, 76 m_includeSuccessfulResults( m_config->includeSuccessfulResults() || m_reporter->getPreferences().shouldReportAllAssertions ) 77 { 78 m_context.setRunner(this); 79 m_context.setConfig(m_config); 80 m_context.setResultCapture(this); 81 m_reporter->testRunStarting(m_runInfo); 82 } 83 ~RunContext()84 RunContext::~RunContext() { 85 m_reporter->testRunEnded(TestRunStats(m_runInfo, m_totals, aborting())); 86 } 87 testGroupStarting(std::string const & testSpec,std::size_t groupIndex,std::size_t groupsCount)88 void RunContext::testGroupStarting(std::string const& testSpec, std::size_t groupIndex, std::size_t groupsCount) { 89 m_reporter->testGroupStarting(GroupInfo(testSpec, groupIndex, groupsCount)); 90 } 91 testGroupEnded(std::string const & testSpec,Totals const & totals,std::size_t groupIndex,std::size_t groupsCount)92 void RunContext::testGroupEnded(std::string const& testSpec, Totals const& totals, std::size_t groupIndex, std::size_t groupsCount) { 93 m_reporter->testGroupEnded(TestGroupStats(GroupInfo(testSpec, groupIndex, groupsCount), totals, aborting())); 94 } 95 runTest(TestCase const & testCase)96 Totals RunContext::runTest(TestCase const& testCase) { 97 Totals prevTotals = m_totals; 98 99 std::string redirectedCout; 100 std::string redirectedCerr; 101 102 auto const& testInfo = testCase.getTestCaseInfo(); 103 104 m_reporter->testCaseStarting(testInfo); 105 106 m_activeTestCase = &testCase; 107 108 109 ITracker& rootTracker = m_trackerContext.startRun(); 110 assert(rootTracker.isSectionTracker()); 111 static_cast<SectionTracker&>(rootTracker).addInitialFilters(m_config->getSectionsToRun()); 112 do { 113 m_trackerContext.startCycle(); 114 m_testCaseTracker = &SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(testInfo.name, testInfo.lineInfo)); 115 runCurrentTest(redirectedCout, redirectedCerr); 116 } while (!m_testCaseTracker->isSuccessfullyCompleted() && !aborting()); 117 118 Totals deltaTotals = m_totals.delta(prevTotals); 119 if (testInfo.expectedToFail() && deltaTotals.testCases.passed > 0) { 120 deltaTotals.assertions.failed++; 121 deltaTotals.testCases.passed--; 122 deltaTotals.testCases.failed++; 123 } 124 m_totals.testCases += deltaTotals.testCases; 125 m_reporter->testCaseEnded(TestCaseStats(testInfo, 126 deltaTotals, 127 redirectedCout, 128 redirectedCerr, 129 aborting())); 130 131 m_activeTestCase = nullptr; 132 m_testCaseTracker = nullptr; 133 134 return deltaTotals; 135 } 136 config() const137 IConfigPtr RunContext::config() const { 138 return m_config; 139 } 140 reporter() const141 IStreamingReporter& RunContext::reporter() const { 142 return *m_reporter; 143 } 144 assertionEnded(AssertionResult const & result)145 void RunContext::assertionEnded(AssertionResult const & result) { 146 if (result.getResultType() == ResultWas::Ok) { 147 m_totals.assertions.passed++; 148 m_lastAssertionPassed = true; 149 } else if (!result.isOk()) { 150 m_lastAssertionPassed = false; 151 if( m_activeTestCase->getTestCaseInfo().okToFail() ) 152 m_totals.assertions.failedButOk++; 153 else 154 m_totals.assertions.failed++; 155 } 156 else { 157 m_lastAssertionPassed = true; 158 } 159 160 // We have no use for the return value (whether messages should be cleared), because messages were made scoped 161 // and should be let to clear themselves out. 162 static_cast<void>(m_reporter->assertionEnded(AssertionStats(result, m_messages, m_totals))); 163 164 if (result.getResultType() != ResultWas::Warning) 165 m_messageScopes.clear(); 166 167 // Reset working state 168 resetAssertionInfo(); 169 m_lastResult = result; 170 } resetAssertionInfo()171 void RunContext::resetAssertionInfo() { 172 m_lastAssertionInfo.macroName = StringRef(); 173 m_lastAssertionInfo.capturedExpression = "{Unknown expression after the reported line}"_sr; 174 } 175 sectionStarted(SectionInfo const & sectionInfo,Counts & assertions)176 bool RunContext::sectionStarted(SectionInfo const & sectionInfo, Counts & assertions) { 177 ITracker& sectionTracker = SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(sectionInfo.name, sectionInfo.lineInfo)); 178 if (!sectionTracker.isOpen()) 179 return false; 180 m_activeSections.push_back(§ionTracker); 181 182 m_lastAssertionInfo.lineInfo = sectionInfo.lineInfo; 183 184 m_reporter->sectionStarting(sectionInfo); 185 186 assertions = m_totals.assertions; 187 188 return true; 189 } acquireGeneratorTracker(SourceLineInfo const & lineInfo)190 auto RunContext::acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& { 191 using namespace Generators; 192 GeneratorTracker& tracker = GeneratorTracker::acquire( m_trackerContext, TestCaseTracking::NameAndLocation( "generator", lineInfo ) ); 193 assert( tracker.isOpen() ); 194 m_lastAssertionInfo.lineInfo = lineInfo; 195 return tracker; 196 } 197 testForMissingAssertions(Counts & assertions)198 bool RunContext::testForMissingAssertions(Counts& assertions) { 199 if (assertions.total() != 0) 200 return false; 201 if (!m_config->warnAboutMissingAssertions()) 202 return false; 203 if (m_trackerContext.currentTracker().hasChildren()) 204 return false; 205 m_totals.assertions.failed++; 206 assertions.failed++; 207 return true; 208 } 209 sectionEnded(SectionEndInfo const & endInfo)210 void RunContext::sectionEnded(SectionEndInfo const & endInfo) { 211 Counts assertions = m_totals.assertions - endInfo.prevAssertions; 212 bool missingAssertions = testForMissingAssertions(assertions); 213 214 if (!m_activeSections.empty()) { 215 m_activeSections.back()->close(); 216 m_activeSections.pop_back(); 217 } 218 219 m_reporter->sectionEnded(SectionStats(endInfo.sectionInfo, assertions, endInfo.durationInSeconds, missingAssertions)); 220 m_messages.clear(); 221 m_messageScopes.clear(); 222 } 223 sectionEndedEarly(SectionEndInfo const & endInfo)224 void RunContext::sectionEndedEarly(SectionEndInfo const & endInfo) { 225 if (m_unfinishedSections.empty()) 226 m_activeSections.back()->fail(); 227 else 228 m_activeSections.back()->close(); 229 m_activeSections.pop_back(); 230 231 m_unfinishedSections.push_back(endInfo); 232 } benchmarkStarting(BenchmarkInfo const & info)233 void RunContext::benchmarkStarting( BenchmarkInfo const& info ) { 234 m_reporter->benchmarkStarting( info ); 235 } benchmarkEnded(BenchmarkStats const & stats)236 void RunContext::benchmarkEnded( BenchmarkStats const& stats ) { 237 m_reporter->benchmarkEnded( stats ); 238 } 239 pushScopedMessage(MessageInfo const & message)240 void RunContext::pushScopedMessage(MessageInfo const & message) { 241 m_messages.push_back(message); 242 } 243 popScopedMessage(MessageInfo const & message)244 void RunContext::popScopedMessage(MessageInfo const & message) { 245 m_messages.erase(std::remove(m_messages.begin(), m_messages.end(), message), m_messages.end()); 246 } 247 emplaceUnscopedMessage(MessageBuilder const & builder)248 void RunContext::emplaceUnscopedMessage( MessageBuilder const& builder ) { 249 m_messageScopes.emplace_back( builder ); 250 } 251 getCurrentTestName() const252 std::string RunContext::getCurrentTestName() const { 253 return m_activeTestCase 254 ? m_activeTestCase->getTestCaseInfo().name 255 : std::string(); 256 } 257 getLastResult() const258 const AssertionResult * RunContext::getLastResult() const { 259 return &(*m_lastResult); 260 } 261 exceptionEarlyReported()262 void RunContext::exceptionEarlyReported() { 263 m_shouldReportUnexpected = false; 264 } 265 handleFatalErrorCondition(StringRef message)266 void RunContext::handleFatalErrorCondition( StringRef message ) { 267 // First notify reporter that bad things happened 268 m_reporter->fatalErrorEncountered(message); 269 270 // Don't rebuild the result -- the stringification itself can cause more fatal errors 271 // Instead, fake a result data. 272 AssertionResultData tempResult( ResultWas::FatalErrorCondition, { false } ); 273 tempResult.message = message; 274 AssertionResult result(m_lastAssertionInfo, tempResult); 275 276 assertionEnded(result); 277 278 handleUnfinishedSections(); 279 280 // Recreate section for test case (as we will lose the one that was in scope) 281 auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); 282 SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name); 283 284 Counts assertions; 285 assertions.failed = 1; 286 SectionStats testCaseSectionStats(testCaseSection, assertions, 0, false); 287 m_reporter->sectionEnded(testCaseSectionStats); 288 289 auto const& testInfo = m_activeTestCase->getTestCaseInfo(); 290 291 Totals deltaTotals; 292 deltaTotals.testCases.failed = 1; 293 deltaTotals.assertions.failed = 1; 294 m_reporter->testCaseEnded(TestCaseStats(testInfo, 295 deltaTotals, 296 std::string(), 297 std::string(), 298 false)); 299 m_totals.testCases.failed++; 300 testGroupEnded(std::string(), m_totals, 1, 1); 301 m_reporter->testRunEnded(TestRunStats(m_runInfo, m_totals, false)); 302 } 303 lastAssertionPassed()304 bool RunContext::lastAssertionPassed() { 305 return m_lastAssertionPassed; 306 } 307 assertionPassed()308 void RunContext::assertionPassed() { 309 m_lastAssertionPassed = true; 310 ++m_totals.assertions.passed; 311 resetAssertionInfo(); 312 m_messageScopes.clear(); 313 } 314 aborting() const315 bool RunContext::aborting() const { 316 return m_totals.assertions.failed >= static_cast<std::size_t>(m_config->abortAfter()); 317 } 318 runCurrentTest(std::string & redirectedCout,std::string & redirectedCerr)319 void RunContext::runCurrentTest(std::string & redirectedCout, std::string & redirectedCerr) { 320 auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); 321 SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name); 322 m_reporter->sectionStarting(testCaseSection); 323 Counts prevAssertions = m_totals.assertions; 324 double duration = 0; 325 m_shouldReportUnexpected = true; 326 m_lastAssertionInfo = { "TEST_CASE"_sr, testCaseInfo.lineInfo, StringRef(), ResultDisposition::Normal }; 327 328 seedRng(*m_config); 329 330 Timer timer; 331 CATCH_TRY { 332 if (m_reporter->getPreferences().shouldRedirectStdOut) { 333 #if !defined(CATCH_CONFIG_EXPERIMENTAL_REDIRECT) 334 RedirectedStreams redirectedStreams(redirectedCout, redirectedCerr); 335 336 timer.start(); 337 invokeActiveTestCase(); 338 #else 339 OutputRedirect r(redirectedCout, redirectedCerr); 340 timer.start(); 341 invokeActiveTestCase(); 342 #endif 343 } else { 344 timer.start(); 345 invokeActiveTestCase(); 346 } 347 duration = timer.getElapsedSeconds(); 348 } CATCH_CATCH_ANON (TestFailureException&) { 349 // This just means the test was aborted due to failure 350 } CATCH_CATCH_ALL { 351 // Under CATCH_CONFIG_FAST_COMPILE, unexpected exceptions under REQUIRE assertions 352 // are reported without translation at the point of origin. 353 if( m_shouldReportUnexpected ) { 354 AssertionReaction dummyReaction; 355 handleUnexpectedInflightException( m_lastAssertionInfo, translateActiveException(), dummyReaction ); 356 } 357 } 358 Counts assertions = m_totals.assertions - prevAssertions; 359 bool missingAssertions = testForMissingAssertions(assertions); 360 361 m_testCaseTracker->close(); 362 handleUnfinishedSections(); 363 m_messages.clear(); 364 m_messageScopes.clear(); 365 366 SectionStats testCaseSectionStats(testCaseSection, assertions, duration, missingAssertions); 367 m_reporter->sectionEnded(testCaseSectionStats); 368 } 369 invokeActiveTestCase()370 void RunContext::invokeActiveTestCase() { 371 FatalConditionHandler fatalConditionHandler; // Handle signals 372 m_activeTestCase->invoke(); 373 fatalConditionHandler.reset(); 374 } 375 handleUnfinishedSections()376 void RunContext::handleUnfinishedSections() { 377 // If sections ended prematurely due to an exception we stored their 378 // infos here so we can tear them down outside the unwind process. 379 for (auto it = m_unfinishedSections.rbegin(), 380 itEnd = m_unfinishedSections.rend(); 381 it != itEnd; 382 ++it) 383 sectionEnded(*it); 384 m_unfinishedSections.clear(); 385 } 386 handleExpr(AssertionInfo const & info,ITransientExpression const & expr,AssertionReaction & reaction)387 void RunContext::handleExpr( 388 AssertionInfo const& info, 389 ITransientExpression const& expr, 390 AssertionReaction& reaction 391 ) { 392 m_reporter->assertionStarting( info ); 393 394 bool negated = isFalseTest( info.resultDisposition ); 395 bool result = expr.getResult() != negated; 396 397 if( result ) { 398 if (!m_includeSuccessfulResults) { 399 assertionPassed(); 400 } 401 else { 402 reportExpr(info, ResultWas::Ok, &expr, negated); 403 } 404 } 405 else { 406 reportExpr(info, ResultWas::ExpressionFailed, &expr, negated ); 407 populateReaction( reaction ); 408 } 409 } reportExpr(AssertionInfo const & info,ResultWas::OfType resultType,ITransientExpression const * expr,bool negated)410 void RunContext::reportExpr( 411 AssertionInfo const &info, 412 ResultWas::OfType resultType, 413 ITransientExpression const *expr, 414 bool negated ) { 415 416 m_lastAssertionInfo = info; 417 AssertionResultData data( resultType, LazyExpression( negated ) ); 418 419 AssertionResult assertionResult{ info, data }; 420 assertionResult.m_resultData.lazyExpression.m_transientExpression = expr; 421 422 assertionEnded( assertionResult ); 423 } 424 handleMessage(AssertionInfo const & info,ResultWas::OfType resultType,StringRef const & message,AssertionReaction & reaction)425 void RunContext::handleMessage( 426 AssertionInfo const& info, 427 ResultWas::OfType resultType, 428 StringRef const& message, 429 AssertionReaction& reaction 430 ) { 431 m_reporter->assertionStarting( info ); 432 433 m_lastAssertionInfo = info; 434 435 AssertionResultData data( resultType, LazyExpression( false ) ); 436 data.message = message; 437 AssertionResult assertionResult{ m_lastAssertionInfo, data }; 438 assertionEnded( assertionResult ); 439 if( !assertionResult.isOk() ) 440 populateReaction( reaction ); 441 } handleUnexpectedExceptionNotThrown(AssertionInfo const & info,AssertionReaction & reaction)442 void RunContext::handleUnexpectedExceptionNotThrown( 443 AssertionInfo const& info, 444 AssertionReaction& reaction 445 ) { 446 handleNonExpr(info, Catch::ResultWas::DidntThrowException, reaction); 447 } 448 handleUnexpectedInflightException(AssertionInfo const & info,std::string const & message,AssertionReaction & reaction)449 void RunContext::handleUnexpectedInflightException( 450 AssertionInfo const& info, 451 std::string const& message, 452 AssertionReaction& reaction 453 ) { 454 m_lastAssertionInfo = info; 455 456 AssertionResultData data( ResultWas::ThrewException, LazyExpression( false ) ); 457 data.message = message; 458 AssertionResult assertionResult{ info, data }; 459 assertionEnded( assertionResult ); 460 populateReaction( reaction ); 461 } 462 populateReaction(AssertionReaction & reaction)463 void RunContext::populateReaction( AssertionReaction& reaction ) { 464 reaction.shouldDebugBreak = m_config->shouldDebugBreak(); 465 reaction.shouldThrow = aborting() || (m_lastAssertionInfo.resultDisposition & ResultDisposition::Normal); 466 } 467 handleIncomplete(AssertionInfo const & info)468 void RunContext::handleIncomplete( 469 AssertionInfo const& info 470 ) { 471 m_lastAssertionInfo = info; 472 473 AssertionResultData data( ResultWas::ThrewException, LazyExpression( false ) ); 474 data.message = "Exception translation was disabled by CATCH_CONFIG_FAST_COMPILE"; 475 AssertionResult assertionResult{ info, data }; 476 assertionEnded( assertionResult ); 477 } handleNonExpr(AssertionInfo const & info,ResultWas::OfType resultType,AssertionReaction & reaction)478 void RunContext::handleNonExpr( 479 AssertionInfo const &info, 480 ResultWas::OfType resultType, 481 AssertionReaction &reaction 482 ) { 483 m_lastAssertionInfo = info; 484 485 AssertionResultData data( resultType, LazyExpression( false ) ); 486 AssertionResult assertionResult{ info, data }; 487 assertionEnded( assertionResult ); 488 489 if( !assertionResult.isOk() ) 490 populateReaction( reaction ); 491 } 492 493 getResultCapture()494 IResultCapture& getResultCapture() { 495 if (auto* capture = getCurrentContext().getResultCapture()) 496 return *capture; 497 else 498 CATCH_INTERNAL_ERROR("No result capture instance"); 499 } 500 } 501