1 /*-------------------------------------------------------------------------
2 * drawElements Quality Program Tester Core
3 * ----------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Test executor.
22 *//*--------------------------------------------------------------------*/
23
24 #include "tcuTestSessionExecutor.hpp"
25 #include "tcuCommandLine.hpp"
26 #include "tcuTestLog.hpp"
27
28 #include "deClock.h"
29
30 namespace tcu
31 {
32
33 using std::vector;
34
nodeTypeToTestCaseType(TestNodeType nodeType)35 static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType)
36 {
37 switch (nodeType)
38 {
39 case NODETYPE_SELF_VALIDATE: return QP_TEST_CASE_TYPE_SELF_VALIDATE;
40 case NODETYPE_PERFORMANCE: return QP_TEST_CASE_TYPE_PERFORMANCE;
41 case NODETYPE_CAPABILITY: return QP_TEST_CASE_TYPE_CAPABILITY;
42 case NODETYPE_ACCURACY: return QP_TEST_CASE_TYPE_ACCURACY;
43 default:
44 DE_ASSERT(false);
45 return QP_TEST_CASE_TYPE_LAST;
46 }
47 }
48
TestSessionExecutor(TestPackageRoot & root,TestContext & testCtx)49 TestSessionExecutor::TestSessionExecutor (TestPackageRoot& root, TestContext& testCtx)
50 : m_testCtx (testCtx)
51 , m_inflater (testCtx)
52 , m_caseListFilter (testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive()))
53 , m_iterator (root, m_inflater, *m_caseListFilter)
54 , m_state (STATE_TRAVERSE_HIERARCHY)
55 , m_abortSession (false)
56 , m_isInTestCase (false)
57 , m_testStartTime (0)
58 , m_packageStartTime (0)
59 {
60 }
61
~TestSessionExecutor(void)62 TestSessionExecutor::~TestSessionExecutor (void)
63 {
64 }
65
iterate(void)66 bool TestSessionExecutor::iterate (void)
67 {
68 while (!m_abortSession)
69 {
70 switch (m_state)
71 {
72 case STATE_TRAVERSE_HIERARCHY:
73 {
74 const TestHierarchyIterator::State hierIterState = m_iterator.getState();
75
76 if (hierIterState == TestHierarchyIterator::STATE_ENTER_NODE ||
77 hierIterState == TestHierarchyIterator::STATE_LEAVE_NODE)
78 {
79 TestNode* const curNode = m_iterator.getNode();
80 const TestNodeType nodeType = curNode->getNodeType();
81 const bool isEnter = hierIterState == TestHierarchyIterator::STATE_ENTER_NODE;
82
83 switch (nodeType)
84 {
85 case NODETYPE_PACKAGE:
86 {
87 TestPackage* const testPackage = static_cast<TestPackage*>(curNode);
88 isEnter ? enterTestPackage(testPackage) : leaveTestPackage(testPackage);
89 break;
90 }
91
92 case NODETYPE_GROUP:
93 {
94 isEnter ? enterTestGroup(m_iterator.getNodePath()) : leaveTestGroup(m_iterator.getNodePath());
95 break; // nada
96 }
97
98 case NODETYPE_SELF_VALIDATE:
99 case NODETYPE_PERFORMANCE:
100 case NODETYPE_CAPABILITY:
101 case NODETYPE_ACCURACY:
102 {
103 TestCase* const testCase = static_cast<TestCase*>(curNode);
104
105 if (isEnter)
106 {
107 if (enterTestCase(testCase, m_iterator.getNodePath()))
108 m_state = STATE_EXECUTE_TEST_CASE;
109 // else remain in TRAVERSING_HIERARCHY => node will be exited from in the next iteration
110 }
111 else
112 leaveTestCase(testCase);
113
114 break;
115 }
116
117 default:
118 DE_ASSERT(false);
119 break;
120 }
121
122 m_iterator.next();
123 break;
124 }
125 else
126 {
127 DE_ASSERT(hierIterState == TestHierarchyIterator::STATE_FINISHED);
128 m_status.isComplete = true;
129 return false;
130 }
131 }
132
133 case STATE_EXECUTE_TEST_CASE:
134 {
135 DE_ASSERT(m_iterator.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
136 isTestNodeTypeExecutable(m_iterator.getNode()->getNodeType()));
137
138 TestCase* const testCase = static_cast<TestCase*>(m_iterator.getNode());
139 const TestCase::IterateResult iterResult = iterateTestCase(testCase);
140
141 if (iterResult == TestCase::STOP)
142 m_state = STATE_TRAVERSE_HIERARCHY;
143
144 return true;
145 }
146
147 default:
148 DE_ASSERT(false);
149 break;
150 }
151 }
152
153 return false;
154 }
155
enterTestPackage(TestPackage * testPackage)156 void TestSessionExecutor::enterTestPackage (TestPackage* testPackage)
157 {
158 // Create test case wrapper
159 DE_ASSERT(!m_caseExecutor);
160 m_caseExecutor = de::MovePtr<TestCaseExecutor>(testPackage->createExecutor());
161 testPackage->setCaseListFilter(m_caseListFilter.get());
162 m_packageStartTime = deGetMicroseconds();
163 }
164
leaveTestPackage(TestPackage * testPackage)165 void TestSessionExecutor::leaveTestPackage (TestPackage* testPackage)
166 {
167 DE_UNREF(testPackage);
168 m_caseExecutor->deinitTestPackage(m_testCtx);
169 // If m_caseExecutor uses local status then it may perform some tests in deinitTestPackage(). We have to update TestSessionExecutor::m_status
170 if (m_caseExecutor->usesLocalStatus())
171 m_caseExecutor->updateGlobalStatus(m_status);
172
173 const deInt64 duration = deGetMicroseconds() - m_packageStartTime;
174 m_packageStartTime = 0;
175
176 if (!std::string(m_testCtx.getCommandLine().getServerAddress()).empty())
177 m_caseExecutor->reportDurations(m_testCtx, std::string(testPackage->getName()), duration, m_groupsDurationTime);
178
179 m_caseExecutor.clear();
180
181 if (!std::string(m_testCtx.getCommandLine().getServerAddress()).empty())
182 {
183 m_testCtx.getLog().startTestsCasesTime();
184
185 m_testCtx.getLog() << TestLog::Integer(testPackage->getName(), "Total tests case duration in microseconds", "us", QP_KEY_TAG_TIME, duration);
186
187 for (std::map<std::string, deUint64>::iterator it = m_groupsDurationTime.begin(); it != m_groupsDurationTime.end(); ++it)
188 m_testCtx.getLog() << TestLog::Integer(it->first, "The test group case duration in microseconds", "us", QP_KEY_TAG_TIME, it->second);
189
190 m_testCtx.getLog().endTestsCasesTime();
191 }
192 }
193
enterTestGroup(const std::string & casePath)194 void TestSessionExecutor::enterTestGroup (const std::string& casePath)
195 {
196 m_groupsDurationTime[casePath] = deGetMicroseconds();
197 }
198
leaveTestGroup(const std::string & casePath)199 void TestSessionExecutor::leaveTestGroup (const std::string& casePath)
200 {
201 m_groupsDurationTime[casePath] = deGetMicroseconds() - m_groupsDurationTime[casePath];
202 }
203
enterTestCase(TestCase * testCase,const std::string & casePath)204 bool TestSessionExecutor::enterTestCase (TestCase* testCase, const std::string& casePath)
205 {
206 TestLog& log = m_testCtx.getLog();
207 const qpTestCaseType caseType = nodeTypeToTestCaseType(testCase->getNodeType());
208 bool initOk = false;
209
210 print("\nTest case '%s'..\n", casePath.c_str());
211
212 #if (DE_OS == DE_OS_WIN32)
213 fflush(stdout);
214 #endif
215
216 m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
217 m_testCtx.setTerminateAfter(false);
218 log.startCase(casePath.c_str(), caseType);
219
220 m_isInTestCase = true;
221 m_testStartTime = deGetMicroseconds();
222
223 try
224 {
225 m_caseExecutor->init(testCase, casePath);
226 initOk = true;
227 }
228 catch (const std::bad_alloc&)
229 {
230 DE_ASSERT(!initOk);
231 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory in test case init");
232 m_testCtx.setTerminateAfter(true);
233 }
234 catch (const tcu::TestException& e)
235 {
236 DE_ASSERT(!initOk);
237 DE_ASSERT(e.getTestResult() != QP_TEST_RESULT_LAST);
238 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
239 m_testCtx.setTerminateAfter(e.isFatal());
240 log << e;
241 }
242 catch (const tcu::Exception& e)
243 {
244 DE_ASSERT(!initOk);
245 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
246 log << e;
247 }
248
249 DE_ASSERT(initOk || m_testCtx.getTestResult() != QP_TEST_RESULT_LAST);
250
251 return initOk;
252 }
253
leaveTestCase(TestCase * testCase)254 void TestSessionExecutor::leaveTestCase (TestCase* testCase)
255 {
256 TestLog& log = m_testCtx.getLog();
257
258 // De-init case.
259 try
260 {
261 m_caseExecutor->deinit(testCase);
262 }
263 catch (const tcu::Exception& e)
264 {
265 const bool suppressLogging = m_testCtx.getLog().isSupressLogging();
266
267 if (suppressLogging)
268 m_testCtx.getLog().supressLogging(false);
269
270 log << e << TestLog::Message << "Error in test case deinit, test program will terminate." << TestLog::EndMessage;
271 m_testCtx.setTerminateAfter(true);
272
273 m_testCtx.getLog().supressLogging(suppressLogging);
274 }
275
276 {
277 const deInt64 duration = deGetMicroseconds()-m_testStartTime;
278 m_testStartTime = 0;
279 m_testCtx.getLog() << TestLog::Integer("TestDuration", "Test case duration in microseconds", "us", QP_KEY_TAG_TIME, duration);
280 }
281
282 {
283 const qpTestResult testResult = m_testCtx.getTestResult();
284 const char* const testResultDesc = m_testCtx.getTestResultDesc();
285 const bool terminateAfter = m_testCtx.getTerminateAfter();
286 DE_ASSERT(testResult != QP_TEST_RESULT_LAST);
287
288 m_isInTestCase = false;
289 m_testCtx.getLog().endCase(testResult, testResultDesc);
290
291 // Update statistics.
292 print(" %s (%s)\n", qpGetTestResultName(testResult), testResultDesc);
293
294 #if (DE_OS == DE_OS_WIN32)
295 fflush(stdout);
296 #endif
297 if(!m_caseExecutor->usesLocalStatus())
298 {
299 m_status.numExecuted += 1;
300 switch (testResult)
301 {
302 case QP_TEST_RESULT_PASS: m_status.numPassed += 1; break;
303 case QP_TEST_RESULT_NOT_SUPPORTED: m_status.numNotSupported += 1; break;
304 case QP_TEST_RESULT_QUALITY_WARNING: m_status.numWarnings += 1; break;
305 case QP_TEST_RESULT_COMPATIBILITY_WARNING: m_status.numWarnings += 1; break;
306 case QP_TEST_RESULT_WAIVER: m_status.numWaived += 1; break;
307 default: m_status.numFailed += 1; break;
308 }
309 }
310 else
311 {
312 m_caseExecutor->updateGlobalStatus(m_status);
313 }
314
315 // terminateAfter, Resource error or any error in deinit means that execution should end
316 if (terminateAfter || testResult == QP_TEST_RESULT_RESOURCE_ERROR ||
317 (m_status.numFailed > 0 && m_testCtx.getCommandLine().isTerminateOnFailEnabled()))
318
319 m_abortSession = true;
320 }
321
322 if (m_testCtx.getWatchDog())
323 qpWatchDog_reset(m_testCtx.getWatchDog());
324 }
325
iterateTestCase(TestCase * testCase)326 TestCase::IterateResult TestSessionExecutor::iterateTestCase (TestCase* testCase)
327 {
328 TestLog& log = m_testCtx.getLog();
329 TestCase::IterateResult iterateResult = TestCase::STOP;
330
331 m_testCtx.touchWatchdog();
332
333 try
334 {
335 iterateResult = m_caseExecutor->iterate(testCase);
336 }
337 catch (const std::bad_alloc&)
338 {
339 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory during test execution");
340 m_testCtx.setTerminateAfter(true);
341 }
342 catch (const tcu::TestException& e)
343 {
344 log << e;
345 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
346 m_testCtx.setTerminateAfter(e.isFatal());
347 }
348 catch (const tcu::Exception& e)
349 {
350 log << e;
351 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
352 }
353
354 return iterateResult;
355 }
356
357 } // tcu
358