/*------------------------------------------------------------------------- * drawElements Quality Program Tester Core * ---------------------------------------- * * Copyright 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * *//*! * \file * \brief Test executor. *//*--------------------------------------------------------------------*/ #include "tcuTestSessionExecutor.hpp" #include "tcuCommandLine.hpp" #include "tcuTestLog.hpp" #include "deClock.h" namespace tcu { using std::vector; static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType) { switch (nodeType) { case NODETYPE_SELF_VALIDATE: return QP_TEST_CASE_TYPE_SELF_VALIDATE; case NODETYPE_PERFORMANCE: return QP_TEST_CASE_TYPE_PERFORMANCE; case NODETYPE_CAPABILITY: return QP_TEST_CASE_TYPE_CAPABILITY; case NODETYPE_ACCURACY: return QP_TEST_CASE_TYPE_ACCURACY; default: DE_ASSERT(false); return QP_TEST_CASE_TYPE_LAST; } } TestSessionExecutor::TestSessionExecutor (TestPackageRoot& root, TestContext& testCtx) : m_testCtx (testCtx) , m_inflater (testCtx) , m_caseListFilter (testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive())) , m_iterator (root, m_inflater, *m_caseListFilter) , m_state (STATE_TRAVERSE_HIERARCHY) , m_abortSession (false) , m_isInTestCase (false) , m_testStartTime (0) , m_packageStartTime (0) { } TestSessionExecutor::~TestSessionExecutor (void) { } bool TestSessionExecutor::iterate (void) { while (!m_abortSession) { switch (m_state) { case STATE_TRAVERSE_HIERARCHY: { const TestHierarchyIterator::State hierIterState = m_iterator.getState(); if (hierIterState == TestHierarchyIterator::STATE_ENTER_NODE || hierIterState == TestHierarchyIterator::STATE_LEAVE_NODE) { TestNode* const curNode = m_iterator.getNode(); const TestNodeType nodeType = curNode->getNodeType(); const bool isEnter = hierIterState == TestHierarchyIterator::STATE_ENTER_NODE; switch (nodeType) { case NODETYPE_PACKAGE: { TestPackage* const testPackage = static_cast(curNode); isEnter ? enterTestPackage(testPackage) : leaveTestPackage(testPackage); break; } case NODETYPE_GROUP: { isEnter ? enterTestGroup(m_iterator.getNodePath()) : leaveTestGroup(m_iterator.getNodePath()); break; // nada } case NODETYPE_SELF_VALIDATE: case NODETYPE_PERFORMANCE: case NODETYPE_CAPABILITY: case NODETYPE_ACCURACY: { TestCase* const testCase = static_cast(curNode); if (isEnter) { if (enterTestCase(testCase, m_iterator.getNodePath())) m_state = STATE_EXECUTE_TEST_CASE; // else remain in TRAVERSING_HIERARCHY => node will be exited from in the next iteration } else leaveTestCase(testCase); break; } default: DE_ASSERT(false); break; } m_iterator.next(); break; } else { DE_ASSERT(hierIterState == TestHierarchyIterator::STATE_FINISHED); m_status.isComplete = true; return false; } } case STATE_EXECUTE_TEST_CASE: { DE_ASSERT(m_iterator.getState() == TestHierarchyIterator::STATE_LEAVE_NODE && isTestNodeTypeExecutable(m_iterator.getNode()->getNodeType())); TestCase* const testCase = static_cast(m_iterator.getNode()); const TestCase::IterateResult iterResult = iterateTestCase(testCase); if (iterResult == TestCase::STOP) m_state = STATE_TRAVERSE_HIERARCHY; return true; } default: DE_ASSERT(false); break; } } return false; } void TestSessionExecutor::enterTestPackage (TestPackage* testPackage) { // Create test case wrapper DE_ASSERT(!m_caseExecutor); m_caseExecutor = de::MovePtr(testPackage->createExecutor()); m_packageStartTime = deGetMicroseconds(); } void TestSessionExecutor::leaveTestPackage (TestPackage* testPackage) { DE_UNREF(testPackage); m_caseExecutor.clear(); m_testCtx.getLog().startTestsCasesTime(); { const deInt64 duration = deGetMicroseconds() - m_packageStartTime; m_packageStartTime = 0; m_testCtx.getLog() << TestLog::Integer(testPackage->getName(), "Total tests case duration in microseconds", "us", QP_KEY_TAG_TIME, duration); } for(std::map::iterator it=m_groupsDurationTime.begin(); it != m_groupsDurationTime.end(); ++it) m_testCtx.getLog() << TestLog::Integer(it->first, "The test group case duration in microseconds", "us", QP_KEY_TAG_TIME, it->second); m_testCtx.getLog().endTestsCasesTime(); } void TestSessionExecutor::enterTestGroup (const std::string& casePath) { m_groupsDurationTime[casePath] = deGetMicroseconds(); } void TestSessionExecutor::leaveTestGroup (const std::string& casePath) { m_groupsDurationTime[casePath] = deGetMicroseconds() - m_groupsDurationTime[casePath]; } bool TestSessionExecutor::enterTestCase (TestCase* testCase, const std::string& casePath) { TestLog& log = m_testCtx.getLog(); const qpTestCaseType caseType = nodeTypeToTestCaseType(testCase->getNodeType()); bool initOk = false; print("\nTest case '%s'..\n", casePath.c_str()); m_testCtx.setTestResult(QP_TEST_RESULT_LAST, ""); m_testCtx.setTerminateAfter(false); log.startCase(casePath.c_str(), caseType); m_isInTestCase = true; m_testStartTime = deGetMicroseconds(); try { m_caseExecutor->init(testCase, casePath); initOk = true; } catch (const std::bad_alloc&) { DE_ASSERT(!initOk); m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory in test case init"); m_testCtx.setTerminateAfter(true); } catch (const tcu::TestException& e) { DE_ASSERT(!initOk); DE_ASSERT(e.getTestResult() != QP_TEST_RESULT_LAST); m_testCtx.setTestResult(e.getTestResult(), e.getMessage()); m_testCtx.setTerminateAfter(e.isFatal()); log << e; } catch (const tcu::Exception& e) { DE_ASSERT(!initOk); m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage()); log << e; } DE_ASSERT(initOk || m_testCtx.getTestResult() != QP_TEST_RESULT_LAST); return initOk; } void TestSessionExecutor::leaveTestCase (TestCase* testCase) { TestLog& log = m_testCtx.getLog(); // De-init case. try { m_caseExecutor->deinit(testCase); } catch (const tcu::Exception& e) { log << e << TestLog::Message << "Error in test case deinit, test program will terminate." << TestLog::EndMessage; m_testCtx.setTerminateAfter(true); } { const deInt64 duration = deGetMicroseconds()-m_testStartTime; m_testStartTime = 0; m_testCtx.getLog() << TestLog::Integer("TestDuration", "Test case duration in microseconds", "us", QP_KEY_TAG_TIME, duration); } { const qpTestResult testResult = m_testCtx.getTestResult(); const char* const testResultDesc = m_testCtx.getTestResultDesc(); const bool terminateAfter = m_testCtx.getTerminateAfter(); DE_ASSERT(testResult != QP_TEST_RESULT_LAST); m_isInTestCase = false; m_testCtx.getLog().endCase(testResult, testResultDesc); // Update statistics. print(" %s (%s)\n", qpGetTestResultName(testResult), testResultDesc); m_status.numExecuted += 1; switch (testResult) { case QP_TEST_RESULT_PASS: m_status.numPassed += 1; break; case QP_TEST_RESULT_NOT_SUPPORTED: m_status.numNotSupported += 1; break; case QP_TEST_RESULT_QUALITY_WARNING: m_status.numWarnings += 1; break; case QP_TEST_RESULT_COMPATIBILITY_WARNING: m_status.numWarnings += 1; break; case QP_TEST_RESULT_WAIVER: m_status.numWaived += 1; break; default: m_status.numFailed += 1; break; } // terminateAfter, Resource error or any error in deinit means that execution should end if (terminateAfter || testResult == QP_TEST_RESULT_RESOURCE_ERROR) m_abortSession = true; } if (m_testCtx.getWatchDog()) qpWatchDog_reset(m_testCtx.getWatchDog()); } TestCase::IterateResult TestSessionExecutor::iterateTestCase (TestCase* testCase) { TestLog& log = m_testCtx.getLog(); TestCase::IterateResult iterateResult = TestCase::STOP; m_testCtx.touchWatchdog(); try { iterateResult = m_caseExecutor->iterate(testCase); } catch (const std::bad_alloc&) { m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory during test execution"); m_testCtx.setTerminateAfter(true); } catch (const tcu::TestException& e) { log << e; m_testCtx.setTestResult(e.getTestResult(), e.getMessage()); m_testCtx.setTerminateAfter(e.isFatal()); } catch (const tcu::Exception& e) { log << e; m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage()); } return iterateResult; } } // tcu