1 /*-------------------------------------------------------------------------
2 * OpenGL Conformance Test Suite
3 * -----------------------------
4 *
5 * Copyright (c) 2016 Google Inc.
6 * Copyright (c) 2016 The Khronos Group Inc.
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 */ /*!
21 * \file
22 * \brief CTS runner.
23 */ /*-------------------------------------------------------------------*/
24
25 #include "glcTestRunner.hpp"
26 #include "deFilePath.hpp"
27 #include "deStringUtil.hpp"
28 #include "deUniquePtr.hpp"
29 #include "glcConfigList.hpp"
30 #include "qpXmlWriter.h"
31 #include "tcuApp.hpp"
32 #include "tcuCommandLine.hpp"
33 #include "tcuTestLog.hpp"
34 #include "tcuTestSessionExecutor.hpp"
35
36 #include <iterator>
37
38 namespace glcts
39 {
40
41 using std::vector;
42 using std::string;
43
44 // RunSession
45
46 class RunSession
47 {
48 public:
RunSession(tcu::Platform & platform,tcu::Archive & archive,const int numArgs,const char * const * args)49 RunSession(tcu::Platform& platform, tcu::Archive& archive, const int numArgs, const char* const* args)
50 : m_cmdLine(numArgs, args)
51 , m_log(m_cmdLine.getLogFileName(), (numArgs - 1), (char**)(args + 1), m_cmdLine.getLogFlags())
52 , m_app(platform, archive, m_log, m_cmdLine)
53 {
54 }
55
iterate(void)56 inline bool iterate(void)
57 {
58 return m_app.iterate();
59 }
60
getResult(void) const61 inline const tcu::TestRunStatus& getResult(void) const
62 {
63 return m_app.getResult();
64 }
65
66 private:
67 tcu::CommandLine m_cmdLine;
68 tcu::TestLog m_log;
69 tcu::App m_app;
70 };
71
appendConfigArgs(const Config & config,std::vector<std::string> & args,const char * fboConfig)72 static void appendConfigArgs(const Config& config, std::vector<std::string>& args, const char* fboConfig)
73 {
74 if (fboConfig != NULL)
75 {
76 args.push_back(string("--deqp-gl-config-name=") + fboConfig);
77 args.push_back("--deqp-surface-type=fbo");
78 }
79
80 if (config.type != CONFIGTYPE_DEFAULT)
81 {
82 // \todo [2013-05-06 pyry] Test all surface types for some configs?
83 if (fboConfig == NULL)
84 {
85 if (config.surfaceTypes & SURFACETYPE_WINDOW)
86 args.push_back("--deqp-surface-type=window");
87 else if (config.surfaceTypes & SURFACETYPE_PBUFFER)
88 args.push_back("--deqp-surface-type=pbuffer");
89 else if (config.surfaceTypes & SURFACETYPE_PIXMAP)
90 args.push_back("--deqp-surface-type=pixmap");
91 }
92
93 args.push_back(string("--deqp-gl-config-id=") + de::toString(config.id));
94
95 if (config.type == CONFIGTYPE_EGL)
96 args.push_back("--deqp-gl-context-type=egl");
97 else if (config.type == CONFIGTYPE_WGL)
98 args.push_back("--deqp-gl-context-type=wgl");
99 }
100 }
101
102 typedef struct configInfo
103 {
104 deInt32 redBits;
105 deInt32 greenBits;
106 deInt32 blueBits;
107 deInt32 alphaBits;
108 deInt32 depthBits;
109 deInt32 stencilBits;
110 deInt32 samples;
111 } configInfo;
112
parseConfigBitsFromName(const char * configName)113 static configInfo parseConfigBitsFromName(const char* configName)
114 {
115 configInfo cfgInfo;
116 static const struct
117 {
118 const char* name;
119 int redBits;
120 int greenBits;
121 int blueBits;
122 int alphaBits;
123 } colorCfgs[] = {
124 { "rgba8888", 8, 8, 8, 8 }, { "rgb565", 5, 6, 5, 0 },
125 };
126 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(colorCfgs); ndx++)
127 {
128 if (!strncmp(configName, colorCfgs[ndx].name, strlen(colorCfgs[ndx].name)))
129 {
130 cfgInfo.redBits = colorCfgs[ndx].redBits;
131 cfgInfo.greenBits = colorCfgs[ndx].greenBits;
132 cfgInfo.blueBits = colorCfgs[ndx].blueBits;
133 cfgInfo.alphaBits = colorCfgs[ndx].alphaBits;
134
135 configName += strlen(colorCfgs[ndx].name);
136 break;
137 }
138 }
139
140 static const struct
141 {
142 const char* name;
143 int depthBits;
144 } depthCfgs[] = {
145 { "d0", 0 }, { "d24", 24 },
146 };
147 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(depthCfgs); ndx++)
148 {
149 if (!strncmp(configName, depthCfgs[ndx].name, strlen(depthCfgs[ndx].name)))
150 {
151 cfgInfo.depthBits = depthCfgs[ndx].depthBits;
152
153 configName += strlen(depthCfgs[ndx].name);
154 break;
155 }
156 }
157
158 static const struct
159 {
160 const char* name;
161 int stencilBits;
162 } stencilCfgs[] = {
163 { "s0", 0 }, { "s8", 8 },
164 };
165 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(stencilCfgs); ndx++)
166 {
167 if (!strncmp(configName, stencilCfgs[ndx].name, strlen(stencilCfgs[ndx].name)))
168 {
169 cfgInfo.stencilBits = stencilCfgs[ndx].stencilBits;
170
171 configName += strlen(stencilCfgs[ndx].name);
172 break;
173 }
174 }
175
176 static const struct
177 {
178 const char* name;
179 int samples;
180 } multiSampleCfgs[] = {
181 { "ms0", 0 }, { "ms4", 4 },
182 };
183 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(multiSampleCfgs); ndx++)
184 {
185 if (!strncmp(configName, multiSampleCfgs[ndx].name, strlen(multiSampleCfgs[ndx].name)))
186 {
187 cfgInfo.samples = multiSampleCfgs[ndx].samples;
188
189 configName += strlen(multiSampleCfgs[ndx].name);
190 break;
191 }
192 }
193
194 return cfgInfo;
195 }
196
getApiName(glu::ApiType apiType)197 static const char* getApiName(glu::ApiType apiType)
198 {
199 if (apiType == glu::ApiType::es(2, 0))
200 return "gles2";
201 else if (apiType == glu::ApiType::es(3, 0))
202 return "gles3";
203 else if (apiType == glu::ApiType::es(3, 1))
204 return "gles31";
205 else if (apiType == glu::ApiType::es(3, 2))
206 return "gles32";
207 else if (apiType == glu::ApiType::core(3, 0))
208 return "gl30";
209 else if (apiType == glu::ApiType::core(3, 1))
210 return "gl31";
211 else if (apiType == glu::ApiType::core(3, 2))
212 return "gl32";
213 else if (apiType == glu::ApiType::core(3, 3))
214 return "gl33";
215 else if (apiType == glu::ApiType::core(4, 0))
216 return "gl40";
217 else if (apiType == glu::ApiType::core(4, 1))
218 return "gl41";
219 else if (apiType == glu::ApiType::core(4, 2))
220 return "gl42";
221 else if (apiType == glu::ApiType::core(4, 3))
222 return "gl43";
223 else if (apiType == glu::ApiType::core(4, 4))
224 return "gl44";
225 else if (apiType == glu::ApiType::core(4, 5))
226 return "gl45";
227 else if (apiType == glu::ApiType::core(4, 6))
228 return "gl46";
229 else
230 throw std::runtime_error("Unknown context type");
231 }
232
getCaseListFileOption(const char * mustpassDir,const char * apiName,const char * mustpassName)233 static const string getCaseListFileOption(const char* mustpassDir, const char* apiName, const char* mustpassName)
234 {
235 #if DE_OS == DE_OS_ANDROID
236 const string case_list_option = "--deqp-caselist-resource=";
237 #else
238 const string case_list_option = "--deqp-caselist-file=";
239 #endif
240 return case_list_option + mustpassDir + apiName + "-" + mustpassName + ".txt";
241 }
242
getLogFileName(const char * apiName,const char * configName,const int iterId,const int runId,const int width,const int height,const int seed)243 static const string getLogFileName(const char* apiName, const char* configName, const int iterId, const int runId,
244 const int width, const int height, const int seed)
245 {
246 string res = string("config-") + apiName + "-" + configName + "-cfg-" + de::toString(iterId) + "-run-" +
247 de::toString(runId) + "-width-" + de::toString(width) + "-height-" + de::toString(height);
248 if (seed != -1)
249 {
250 res += "-seed-" + de::toString(seed);
251 }
252 res += ".qpa";
253
254 return res;
255 }
256
getBaseOptions(std::vector<std::string> & args,const char * mustpassDir,const char * apiName,const char * configName,const char * screenRotation,int width,int height)257 static void getBaseOptions(std::vector<std::string>& args, const char* mustpassDir, const char* apiName,
258 const char* configName, const char* screenRotation, int width, int height)
259 {
260 args.push_back(getCaseListFileOption(mustpassDir, apiName, configName));
261 args.push_back(string("--deqp-screen-rotation=") + screenRotation);
262 args.push_back(string("--deqp-surface-width=") + de::toString(width));
263 args.push_back(string("--deqp-surface-height=") + de::toString(height));
264 args.push_back("--deqp-watchdog=disable");
265 }
266
isGLConfigCompatible(configInfo cfgInfo,const AOSPConfig & config)267 static bool isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig& config)
268 {
269 return cfgInfo.redBits == config.redBits && cfgInfo.greenBits == config.greenBits &&
270 cfgInfo.blueBits == config.blueBits && cfgInfo.alphaBits == config.alphaBits &&
271 cfgInfo.depthBits == config.depthBits && cfgInfo.stencilBits == config.stencilBits &&
272 cfgInfo.samples == config.samples;
273 }
274
getTestRunsForAOSPEGL(vector<TestRunParams> & runs,const ConfigList & configs)275 static void getTestRunsForAOSPEGL(vector<TestRunParams>& runs, const ConfigList& configs)
276 {
277 #include "glcAospMustpassEgl.hpp"
278
279 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_egl_first_cfg); ++i)
280 {
281 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_egl_first_cfg[i].glConfigName);
282
283 vector<AOSPConfig>::const_iterator cfgIter;
284 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
285 {
286 // find first compatible config
287 if ((*cfgIter).type == CONFIGTYPE_EGL && isGLConfigCompatible(cfgInfo, *cfgIter))
288 {
289 break;
290 }
291 }
292
293 if (cfgIter == configs.aospConfigs.end())
294 {
295 // No suitable configuration found. Skipping EGL tests
296 continue;
297 }
298
299 const char* apiName = "egl";
300
301 const int width = aosp_mustpass_egl_first_cfg[i].surfaceWidth;
302 const int height = aosp_mustpass_egl_first_cfg[i].surfaceHeight;
303
304 TestRunParams params;
305 params.logFilename =
306 getLogFileName(apiName, aosp_mustpass_egl_first_cfg[i].configName, 1, i, width, height, -1);
307 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_egl_first_cfg[i].configName,
308 aosp_mustpass_egl_first_cfg[i].screenRotation, width, height);
309
310 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_egl_first_cfg[i].glConfigName));
311
312 runs.push_back(params);
313 }
314 }
315
getTestRunsForAOSPES(vector<TestRunParams> & runs,const ConfigList & configs,const glu::ApiType apiType)316 static void getTestRunsForAOSPES(vector<TestRunParams>& runs, const ConfigList& configs, const glu::ApiType apiType)
317 {
318 #include "glcAospMustpassEs.hpp"
319
320 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_es_first_cfg); ++i)
321 {
322 if (!glu::contextSupports(glu::ContextType(apiType), aosp_mustpass_es_first_cfg[i].apiType))
323 continue;
324
325 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_es_first_cfg[i].glConfigName);
326
327 vector<AOSPConfig>::const_iterator cfgIter;
328 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
329 {
330 // find first compatible config
331 if (isGLConfigCompatible(cfgInfo, *cfgIter))
332 {
333 break;
334 }
335 }
336
337 if (cfgIter == configs.aospConfigs.end())
338 {
339 TCU_FAIL(("No suitable configuration found for GL config " +
340 de::toString(aosp_mustpass_es_first_cfg[i].glConfigName))
341 .c_str());
342 return;
343 }
344
345 const char* apiName = getApiName(aosp_mustpass_es_first_cfg[i].apiType);
346
347 const int width = aosp_mustpass_es_first_cfg[i].surfaceWidth;
348 const int height = aosp_mustpass_es_first_cfg[i].surfaceHeight;
349
350 TestRunParams params;
351 params.logFilename = getLogFileName(apiName, aosp_mustpass_es_first_cfg[i].configName, 1, i, width, height, -1);
352 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_es_first_cfg[i].configName,
353 aosp_mustpass_es_first_cfg[i].screenRotation, width, height);
354
355 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_es_first_cfg[i].glConfigName));
356
357 //set surface type
358 if ((*cfgIter).surfaceTypes & SURFACETYPE_WINDOW)
359 params.args.push_back("--deqp-surface-type=window");
360 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PBUFFER)
361 params.args.push_back("--deqp-surface-type=pbuffer");
362 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PIXMAP)
363 params.args.push_back("--deqp-surface-type=pixmap");
364 runs.push_back(params);
365 }
366 }
367
getTestRunsForNoContext(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs,const RunParams * runParams,const int numRunParams,const char * mustpassDir)368 static void getTestRunsForNoContext(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams,
369 const int numRunParams, const char* mustpassDir)
370 {
371 vector<Config>::const_iterator cfgIter = configs.configs.begin();
372
373 for (int i = 0; i < numRunParams; ++i)
374 {
375 if (!glu::contextSupports(glu::ContextType(type), runParams[i].apiType))
376 continue;
377
378 const char* apiName = getApiName(runParams[i].apiType);
379
380 const int width = runParams[i].surfaceWidth;
381 const int height = runParams[i].surfaceHeight;
382 const int seed = runParams[i].baseSeed;
383
384 TestRunParams params;
385 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
386
387 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
388 height);
389
390 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
391
392 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
393
394 runs.push_back(params);
395 }
396 }
397
getTestRunsForNoContextES(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)398 static void getTestRunsForNoContextES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
399 {
400 #include "glcKhronosMustpassEsNocontext.hpp"
401 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_es_nocontext_first_cfg,
402 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_nocontext_first_cfg), mustpassDir);
403 }
404
getTestRunsForSingleConfig(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs,const RunParams * runParams,const int numRunParams,const char * mustpassDir)405 static void getTestRunsForSingleConfig(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams,
406 const int numRunParams, const char* mustpassDir)
407 {
408 vector<Config>::const_iterator cfgIter = configs.configs.begin();
409
410 for (int i = 0; i < numRunParams; ++i)
411 {
412 if (type != runParams[i].apiType)
413 continue;
414
415 const char* apiName = getApiName(runParams[i].apiType);
416
417 const int width = runParams[i].surfaceWidth;
418 const int height = runParams[i].surfaceHeight;
419 const int seed = runParams[i].baseSeed;
420
421 TestRunParams params;
422 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
423
424 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
425 height);
426
427 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
428
429 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
430
431 runs.push_back(params);
432 }
433 }
getTestRunsForSingleConfigES(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)434 static void getTestRunsForSingleConfigES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
435 {
436 #include "glcKhronosMustpassEsSingleConfig.hpp"
437 getTestRunsForSingleConfig(type, runs, configs, khronos_mustpass_es_single_config_first_cfg,
438 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_single_config_first_cfg), mustpassDir);
439 }
440
getTestRunsForES(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)441 static void getTestRunsForES(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
442 {
443 getTestRunsForAOSPEGL(runs, configs);
444 getTestRunsForAOSPES(runs, configs, type);
445 getTestRunsForNoContextES(type, runs, configs);
446 getTestRunsForSingleConfigES(type, runs, configs);
447
448 #include "glcKhronosMustpassEs.hpp"
449
450 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
451 {
452 const bool isFirst = cfgIter == configs.configs.begin();
453 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_es_first_cfg) :
454 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_other_cfg);
455 const RunParams* runParams = isFirst ? khronos_mustpass_es_first_cfg : khronos_mustpass_es_other_cfg;
456
457 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
458 {
459 if (!glu::contextSupports(glu::ContextType(type), runParams[runNdx].apiType))
460 continue;
461
462 const char* apiName = getApiName(runParams[runNdx].apiType);
463
464 const int width = runParams[runNdx].surfaceWidth;
465 const int height = runParams[runNdx].surfaceHeight;
466 const int seed = runParams[runNdx].baseSeed;
467
468 TestRunParams params;
469
470 params.logFilename =
471 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
472
473 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
474 runParams[runNdx].screenRotation, width, height);
475 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
476
477 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
478
479 runs.push_back(params);
480 }
481 }
482 }
483
getTestRunsForNoContextGL(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)484 static void getTestRunsForNoContextGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
485 {
486 #include "glcKhronosMustpassGlNocontext.hpp"
487 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_gl_nocontext_first_cfg,
488 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_nocontext_first_cfg), mustpassDir);
489 }
getTestRunsForSingleConfigGL(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)490 static void getTestRunsForSingleConfigGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
491 {
492 #include "glcKhronosMustpassGlSingleConfig.hpp"
493 getTestRunsForSingleConfig(type, runs, configs, khronos_mustpass_gl_single_config_first_cfg,
494 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_single_config_first_cfg), mustpassDir);
495 }
496
getTestRunsForGL(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)497 static void getTestRunsForGL(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
498 {
499 getTestRunsForNoContextGL(type, runs, configs);
500 getTestRunsForSingleConfigGL(type, runs, configs);
501 #include "glcKhronosMustpassGl.hpp"
502
503 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
504 {
505 const bool isFirst = cfgIter == configs.configs.begin();
506 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_first_cfg) :
507 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_other_cfg);
508 const RunParams* runParams = isFirst ? khronos_mustpass_gl_first_cfg : khronos_mustpass_gl_other_cfg;
509
510 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
511 {
512 if (type != runParams[runNdx].apiType)
513 continue;
514
515 const char* apiName = getApiName(runParams[runNdx].apiType);
516
517 const int width = runParams[runNdx].surfaceWidth;
518 const int height = runParams[runNdx].surfaceHeight;
519 const int seed = runParams[runNdx].baseSeed;
520
521 TestRunParams params;
522
523 params.logFilename =
524 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
525
526 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
527 runParams[runNdx].screenRotation, width, height);
528 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
529
530 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
531
532 runs.push_back(params);
533 }
534 }
535 }
536
getTestRunParams(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)537 static void getTestRunParams(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
538 {
539 switch (type.getProfile())
540 {
541 case glu::PROFILE_CORE:
542 getTestRunsForGL(type, configs, runs);
543 break;
544 case glu::PROFILE_ES:
545 getTestRunsForES(type, configs, runs);
546 break;
547 default:
548 throw std::runtime_error("Unknown context type");
549 }
550 }
551
552 struct FileDeleter
553 {
operator ()glcts::FileDeleter554 void operator()(FILE* file) const
555 {
556 if (file)
557 fclose(file);
558 }
559 };
560
561 struct XmlWriterDeleter
562 {
operator ()glcts::XmlWriterDeleter563 void operator()(qpXmlWriter* writer) const
564 {
565 if (writer)
566 qpXmlWriter_destroy(writer);
567 }
568 };
569
getRunTypeName(glu::ApiType type)570 static const char* getRunTypeName(glu::ApiType type)
571 {
572 if (type == glu::ApiType::es(2, 0))
573 return "es2";
574 else if (type == glu::ApiType::es(3, 0))
575 return "es3";
576 else if (type == glu::ApiType::es(3, 1))
577 return "es31";
578 else if (type == glu::ApiType::es(3, 2))
579 return "es32";
580 else if (type == glu::ApiType::core(3, 0))
581 return "gl30";
582 else if (type == glu::ApiType::core(3, 1))
583 return "gl31";
584 else if (type == glu::ApiType::core(3, 2))
585 return "gl32";
586 else if (type == glu::ApiType::core(3, 3))
587 return "gl33";
588 else if (type == glu::ApiType::core(4, 0))
589 return "gl40";
590 else if (type == glu::ApiType::core(4, 1))
591 return "gl41";
592 else if (type == glu::ApiType::core(4, 2))
593 return "gl42";
594 else if (type == glu::ApiType::core(4, 3))
595 return "gl43";
596 else if (type == glu::ApiType::core(4, 4))
597 return "gl44";
598 else if (type == glu::ApiType::core(4, 5))
599 return "gl45";
600 else if (type == glu::ApiType::core(4, 6))
601 return "gl46";
602 else
603 return DE_NULL;
604 }
605
606 #define XML_CHECK(X) \
607 if (!(X)) \
608 throw tcu::Exception("Writing XML failed")
609
writeRunSummary(const TestRunSummary & summary,const char * filename)610 static void writeRunSummary(const TestRunSummary& summary, const char* filename)
611 {
612 de::UniquePtr<FILE, FileDeleter> out(fopen(filename, "wb"));
613 if (!out)
614 throw tcu::Exception(string("Failed to open ") + filename);
615
616 de::UniquePtr<qpXmlWriter, XmlWriterDeleter> writer(qpXmlWriter_createFileWriter(out.get(), DE_FALSE, DE_FALSE));
617 if (!writer)
618 throw std::bad_alloc();
619
620 XML_CHECK(qpXmlWriter_startDocument(writer.get()));
621
622 {
623 qpXmlAttribute attribs[2];
624
625 attribs[0] = qpSetStringAttrib("Type", getRunTypeName(summary.runType));
626 attribs[1] = qpSetBoolAttrib("Conformant", summary.isConformant ? DE_TRUE : DE_FALSE);
627
628 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Summary", DE_LENGTH_OF_ARRAY(attribs), attribs));
629 }
630
631 // Config run
632 {
633 qpXmlAttribute attribs[1];
634 attribs[0] = qpSetStringAttrib("FileName", summary.configLogFilename.c_str());
635 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Configs", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
636 qpXmlWriter_endElement(writer.get(), "Configs"));
637 }
638
639 // Record test run parameters (log filename & command line).
640 for (vector<TestRunParams>::const_iterator runIter = summary.runParams.begin(); runIter != summary.runParams.end();
641 ++runIter)
642 {
643 string cmdLine;
644 qpXmlAttribute attribs[2];
645
646 for (vector<string>::const_iterator argIter = runIter->args.begin(); argIter != runIter->args.end(); ++argIter)
647 {
648 if (argIter != runIter->args.begin())
649 cmdLine += " ";
650 cmdLine += *argIter;
651 }
652
653 attribs[0] = qpSetStringAttrib("FileName", runIter->logFilename.c_str());
654 attribs[1] = qpSetStringAttrib("CmdLine", cmdLine.c_str());
655
656 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestRun", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
657 qpXmlWriter_endElement(writer.get(), "TestRun"));
658 }
659
660 XML_CHECK(qpXmlWriter_endElement(writer.get(), "Summary"));
661 XML_CHECK(qpXmlWriter_endDocument(writer.get()));
662 }
663
664 #undef XML_CHECK
665
TestRunner(tcu::Platform & platform,tcu::Archive & archive,const char * logDirPath,glu::ApiType type,deUint32 flags)666 TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* logDirPath, glu::ApiType type,
667 deUint32 flags)
668 : m_platform(platform)
669 , m_archive(archive)
670 , m_logDirPath(logDirPath)
671 , m_type(type)
672 , m_flags(flags)
673 , m_iterState(ITERATE_INIT)
674 , m_curSession(DE_NULL)
675 , m_sessionsExecuted(0)
676 , m_sessionsPassed(0)
677 , m_sessionsFailed(0)
678 {
679 }
680
~TestRunner(void)681 TestRunner::~TestRunner(void)
682 {
683 delete m_curSession;
684 }
685
iterate(void)686 bool TestRunner::iterate(void)
687 {
688 switch (m_iterState)
689 {
690 case ITERATE_INIT:
691 init();
692 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
693 return true;
694
695 case ITERATE_DEINIT:
696 deinit();
697 m_iterState = ITERATE_INIT;
698 return false;
699
700 case ITERATE_INIT_SESSION:
701 DE_ASSERT(m_sessionIter != m_runSessions.end());
702 initSession(*m_sessionIter);
703 if (m_flags & PRINT_SUMMARY)
704 m_iterState = ITERATE_DEINIT_SESSION;
705 else
706 m_iterState = ITERATE_ITERATE_SESSION;
707 return true;
708
709 case ITERATE_DEINIT_SESSION:
710 deinitSession();
711 ++m_sessionIter;
712 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
713 return true;
714
715 case ITERATE_ITERATE_SESSION:
716 if (!iterateSession())
717 m_iterState = ITERATE_DEINIT_SESSION;
718 return true;
719
720 default:
721 DE_ASSERT(false);
722 return false;
723 }
724 }
725
init(void)726 void TestRunner::init(void)
727 {
728 DE_ASSERT(m_runSessions.empty() && m_summary.runParams.empty());
729
730 tcu::print("Running %s conformance\n", glu::getApiTypeDescription(m_type));
731
732 m_summary.runType = m_type;
733
734 // Get list of configs to test.
735 ConfigList configList;
736 getDefaultConfigList(m_platform, m_type, configList);
737
738 tcu::print(" found %d compatible and %d excluded configs\n", (int)configList.configs.size(),
739 (int)configList.excludedConfigs.size());
740
741 // Config list run.
742 {
743 const char* configLogFilename = "configs.qpa";
744 TestRunParams configRun;
745
746 configRun.logFilename = configLogFilename;
747 configRun.args.push_back("--deqp-case=CTS-Configs.*");
748 m_runSessions.push_back(configRun);
749
750 m_summary.configLogFilename = configLogFilename;
751 }
752
753 // Conformance test type specific runs
754 getTestRunParams(m_type, configList, m_runSessions);
755
756 // Record run params for summary.
757 for (std::vector<TestRunParams>::const_iterator runIter = m_runSessions.begin() + 1; runIter != m_runSessions.end();
758 ++runIter)
759 m_summary.runParams.push_back(*runIter);
760
761 // Session iterator
762 m_sessionIter = m_runSessions.begin();
763 }
764
deinit(void)765 void TestRunner::deinit(void)
766 {
767 // Print out totals.
768 bool isConformant_ = m_sessionsExecuted == m_sessionsPassed;
769 DE_ASSERT(m_sessionsExecuted == m_sessionsPassed + m_sessionsFailed);
770 tcu::print("\n%d/%d sessions passed, conformance test %s\n", m_sessionsPassed, m_sessionsExecuted,
771 isConformant_ ? "PASSED" : "FAILED");
772
773 m_summary.isConformant = isConformant_;
774
775 // Write out summary.
776 writeRunSummary(m_summary, de::FilePath::join(m_logDirPath, "cts-run-summary.xml").getPath());
777
778 m_runSessions.clear();
779 m_summary.clear();
780 }
781
initSession(const TestRunParams & runParams)782 void TestRunner::initSession(const TestRunParams& runParams)
783 {
784 DE_ASSERT(!m_curSession);
785
786 tcu::print("\n Test run %d / %d\n", (int)(m_sessionIter - m_runSessions.begin() + 1), (int)m_runSessions.size());
787
788 // Compute final args for run.
789 vector<string> args(runParams.args);
790 args.push_back(string("--deqp-log-filename=") + de::FilePath::join(m_logDirPath, runParams.logFilename).getPath());
791
792 if (!(m_flags & VERBOSE_IMAGES))
793 args.push_back("--deqp-log-images=disable");
794
795 if (!(m_flags & VERBOSE_SHADERS))
796 args.push_back("--deqp-log-shader-sources=disable");
797
798 std::ostringstream ostr;
799 std::ostream_iterator<string> out_it(ostr, ", ");
800 std::copy(args.begin(), args.end(), out_it);
801 tcu::print("\n Config: %s \n\n", ostr.str().c_str());
802
803 // Translate to argc, argv
804 vector<const char*> argv;
805 argv.push_back("cts-runner"); // Dummy binary name
806 for (vector<string>::const_iterator i = args.begin(); i != args.end(); i++)
807 argv.push_back(i->c_str());
808
809 // Create session
810 m_curSession = new RunSession(m_platform, m_archive, (int)argv.size(), &argv[0]);
811 }
812
deinitSession(void)813 void TestRunner::deinitSession(void)
814 {
815 DE_ASSERT(m_curSession);
816
817 // Collect results.
818 // \note NotSupported is treated as pass.
819 const tcu::TestRunStatus& result = m_curSession->getResult();
820 bool isOk =
821 result.numExecuted == (result.numPassed + result.numNotSupported + result.numWarnings) && result.isComplete;
822
823 DE_ASSERT(result.numExecuted == result.numPassed + result.numFailed + result.numNotSupported + result.numWarnings);
824
825 m_sessionsExecuted += 1;
826 (isOk ? m_sessionsPassed : m_sessionsFailed) += 1;
827
828 delete m_curSession;
829 m_curSession = DE_NULL;
830 }
831
iterateSession(void)832 inline bool TestRunner::iterateSession(void)
833 {
834 return m_curSession->iterate();
835 }
836
837 } // glcts
838