1 /*-------------------------------------------------------------------------
2 * drawElements Quality Program Tester Core
3 * ----------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Executor which can run randomly accessed tests.
22 *//*--------------------------------------------------------------------*/
23
24 #include "tcuRandomOrderExecutor.h"
25
26 #include "deClock.h"
27 #include "deStringUtil.hpp"
28 #include "tcuCommandLine.hpp"
29 #include "tcuTestLog.hpp"
30
31 #include <algorithm>
32 #include <cstdio>
33 #include <fstream>
34
35 using std::string;
36 using std::vector;
37
38 namespace tcu
39 {
40
RandomOrderExecutor(TestPackageRoot & root,TestContext & testCtx,bool enableRenderDocCapture)41 RandomOrderExecutor::RandomOrderExecutor(TestPackageRoot &root,
42 TestContext &testCtx,
43 bool enableRenderDocCapture)
44 : m_testCtx(testCtx), m_inflater(testCtx)
45 {
46 m_nodeStack.push_back(NodeStackEntry(&root));
47 root.getChildren(m_nodeStack[0].children);
48
49 if (enableRenderDocCapture)
50 {
51 mRenderDoc.attach();
52 }
53 }
54
~RandomOrderExecutor(void)55 RandomOrderExecutor::~RandomOrderExecutor(void)
56 {
57 pruneStack(1);
58 }
59
pruneStack(size_t newStackSize)60 void RandomOrderExecutor::pruneStack(size_t newStackSize)
61 {
62 // \note Root is not managed by us
63 DE_ASSERT(de::inRange(newStackSize, size_t(1), m_nodeStack.size()));
64
65 while (m_nodeStack.size() > newStackSize)
66 {
67 NodeStackEntry &curEntry = m_nodeStack.back();
68 const bool isPkg = curEntry.node->getNodeType() == NODETYPE_PACKAGE;
69
70 DE_ASSERT((m_nodeStack.size() == 2) == isPkg);
71
72 if (curEntry.node) // Just in case we are in
73 // cleanup path after partial
74 // prune
75 {
76 if (curEntry.node->getNodeType() == NODETYPE_GROUP)
77 m_inflater.leaveGroupNode(static_cast<TestCaseGroup *>(curEntry.node));
78 else if (curEntry.node->getNodeType() == NODETYPE_PACKAGE)
79 m_inflater.leaveTestPackage(static_cast<TestPackage *>(curEntry.node));
80 else
81 DE_ASSERT(curEntry.children.empty());
82
83 curEntry.node = DE_NULL;
84 curEntry.children.clear();
85 }
86
87 if (isPkg)
88 m_caseExecutor.clear();
89
90 m_nodeStack.pop_back();
91 }
92 }
93
findNodeByName(vector<TestNode * > & nodes,const std::string & name)94 static TestNode *findNodeByName(vector<TestNode *> &nodes, const std::string &name)
95 {
96 for (vector<TestNode *>::const_iterator node = nodes.begin(); node != nodes.end(); ++node)
97 {
98 if (name == (*node)->getName())
99 return *node;
100 }
101
102 return DE_NULL;
103 }
104
seekToCase(const string & path)105 TestCase *RandomOrderExecutor::seekToCase(const string &path)
106 {
107 const vector<string> components = de::splitString(path, '.');
108 size_t compNdx;
109
110 DE_ASSERT(!m_nodeStack.empty() && m_nodeStack.front().node->getNodeType() == NODETYPE_ROOT);
111
112 for (compNdx = 0; compNdx < components.size(); compNdx++)
113 {
114 const size_t stackPos = compNdx + 1;
115
116 if (stackPos >= m_nodeStack.size())
117 break; // Stack end
118 else if (components[compNdx] != m_nodeStack[stackPos].node->getName())
119 {
120 // Current stack doesn't match any more, prune.
121 pruneStack(stackPos);
122 break;
123 }
124 }
125
126 DE_ASSERT(m_nodeStack.size() == compNdx + 1);
127
128 for (; compNdx < components.size(); compNdx++)
129 {
130 const size_t parentStackPos = compNdx;
131 TestNode *const curNode =
132 findNodeByName(m_nodeStack[parentStackPos].children, components[compNdx]);
133
134 if (!curNode)
135 throw Exception(string("Test hierarchy node not found: ") + path);
136
137 m_nodeStack.push_back(NodeStackEntry(curNode));
138
139 if (curNode->getNodeType() == NODETYPE_PACKAGE)
140 {
141 TestPackage *const testPackage = static_cast<TestPackage *>(curNode);
142
143 m_inflater.enterTestPackage(testPackage, m_nodeStack.back().children);
144 DE_ASSERT(!m_caseExecutor);
145 m_caseExecutor = de::MovePtr<TestCaseExecutor>(testPackage->createExecutor());
146 }
147 else if (curNode->getNodeType() == NODETYPE_GROUP)
148 m_inflater.enterGroupNode(static_cast<TestCaseGroup *>(curNode),
149 m_nodeStack.back().children);
150 else if (compNdx + 1 != components.size())
151 throw Exception(string("Invalid test hierarchy path: ") + path);
152 }
153
154 DE_ASSERT(m_nodeStack.size() == components.size() + 1);
155
156 if (isTestNodeTypeExecutable(m_nodeStack.back().node->getNodeType()))
157 return dynamic_cast<TestCase *>(m_nodeStack.back().node);
158 else
159 throw Exception(string("Not a test case: ") + path);
160 }
161
nodeTypeToTestCaseType(TestNodeType nodeType)162 static qpTestCaseType nodeTypeToTestCaseType(TestNodeType nodeType)
163 {
164 switch (nodeType)
165 {
166 case NODETYPE_SELF_VALIDATE:
167 return QP_TEST_CASE_TYPE_SELF_VALIDATE;
168 case NODETYPE_PERFORMANCE:
169 return QP_TEST_CASE_TYPE_PERFORMANCE;
170 case NODETYPE_CAPABILITY:
171 return QP_TEST_CASE_TYPE_CAPABILITY;
172 case NODETYPE_ACCURACY:
173 return QP_TEST_CASE_TYPE_ACCURACY;
174 default:
175 DE_ASSERT(false);
176 return QP_TEST_CASE_TYPE_LAST;
177 }
178 }
179
execute(const std::string & casePath)180 TestStatus RandomOrderExecutor::execute(const std::string &casePath)
181 {
182 TestCase *const testCase = seekToCase(casePath);
183 TestLog &log = m_testCtx.getLog();
184 const qpTestCaseType caseType = nodeTypeToTestCaseType(testCase->getNodeType());
185
186 m_testCtx.setTerminateAfter(false);
187 log.startCase(casePath.c_str(), caseType);
188
189 {
190 const TestStatus result = executeInner(testCase, casePath);
191 log.endCase(result.getCode(), result.getDescription().c_str());
192 return result;
193 }
194 }
195
executeInner(TestCase * testCase,const std::string & casePath)196 tcu::TestStatus RandomOrderExecutor::executeInner(TestCase *testCase, const std::string &casePath)
197 {
198 TestLog &log = m_testCtx.getLog();
199 const deUint64 testStartTime = deGetMicroseconds();
200
201 m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
202
203 // Initialize, will return immediately if fails
204 try
205 {
206 mRenderDoc.startFrame();
207 m_caseExecutor->init(testCase, casePath);
208 }
209 catch (const std::bad_alloc &)
210 {
211 m_testCtx.setTerminateAfter(true);
212 return TestStatus(QP_TEST_RESULT_RESOURCE_ERROR,
213 "Failed to allocate memory in test case init");
214 }
215 catch (const TestException &e)
216 {
217 DE_ASSERT(e.getTestResult() != QP_TEST_RESULT_LAST);
218 m_testCtx.setTerminateAfter(e.isFatal());
219 log << e;
220 return TestStatus(e.getTestResult(), e.getMessage());
221 }
222 catch (const Exception &e)
223 {
224 log << e;
225 return TestStatus(QP_TEST_RESULT_FAIL, e.getMessage());
226 }
227
228 bool isFirstFrameBeingCaptured = true;
229
230 // Execute
231 for (;;)
232 {
233 TestCase::IterateResult iterateResult = TestCase::STOP;
234
235 m_testCtx.touchWatchdog();
236
237 try
238 {
239 // Make every iteration produce one renderdoc frame. Include the init code in the first
240 // frame, and the deinit code in the last frame.
241 if (!isFirstFrameBeingCaptured)
242 {
243 mRenderDoc.endFrame();
244 mRenderDoc.startFrame();
245 }
246 isFirstFrameBeingCaptured = false;
247
248 iterateResult = m_caseExecutor->iterate(testCase);
249 }
250 catch (const std::bad_alloc &)
251 {
252 m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR,
253 "Failed to allocate memory during test "
254 "execution");
255 }
256 catch (const TestException &e)
257 {
258 log << e;
259 m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
260 m_testCtx.setTerminateAfter(e.isFatal());
261 }
262 catch (const Exception &e)
263 {
264 log << e;
265 m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
266 }
267
268 if (iterateResult == TestCase::STOP)
269 break;
270 }
271
272 DE_ASSERT(m_testCtx.getTestResult() != QP_TEST_RESULT_LAST);
273
274 if (m_testCtx.getTestResult() == QP_TEST_RESULT_RESOURCE_ERROR)
275 m_testCtx.setTerminateAfter(true);
276
277 // De-initialize
278 try
279 {
280 m_caseExecutor->deinit(testCase);
281 mRenderDoc.endFrame();
282 }
283 catch (const tcu::Exception &e)
284 {
285 log << e << TestLog::Message
286 << "Error in test case deinit, test program "
287 "will terminate."
288 << TestLog::EndMessage;
289 m_testCtx.setTerminateAfter(true);
290 }
291
292 if (m_testCtx.getWatchDog())
293 qpWatchDog_reset(m_testCtx.getWatchDog());
294
295 {
296 const TestStatus result =
297 TestStatus(m_testCtx.getTestResult(), m_testCtx.getTestResultDesc());
298 m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
299 return result;
300 }
301 }
302
303 } // namespace tcu
304