1 /*-------------------------------------------------------------------------
2 * OpenGL Conformance Test Suite
3 * -----------------------------
4 *
5 * Copyright (c) 2016 Google Inc.
6 * Copyright (c) 2016 The Khronos Group Inc.
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 */ /*!
21 * \file
22 * \brief CTS runner.
23 */ /*-------------------------------------------------------------------*/
24
25 #include "glcTestRunner.hpp"
26 #include "deFilePath.hpp"
27 #include "deStringUtil.hpp"
28 #include "deUniquePtr.hpp"
29 #include "glcConfigList.hpp"
30 #include "qpXmlWriter.h"
31 #include "tcuApp.hpp"
32 #include "tcuCommandLine.hpp"
33 #include "tcuTestLog.hpp"
34 #include "tcuTestSessionExecutor.hpp"
35
36 #include <iterator>
37
38 namespace glcts
39 {
40
41 using std::vector;
42 using std::string;
43
44 // RunSession
45
46 class RunSession
47 {
48 public:
RunSession(tcu::Platform & platform,tcu::Archive & archive,const int numArgs,const char * const * args)49 RunSession(tcu::Platform& platform, tcu::Archive& archive, const int numArgs, const char* const* args)
50 : m_cmdLine(numArgs, args)
51 , m_log(m_cmdLine.getLogFileName(), m_cmdLine.getLogFlags())
52 , m_app(platform, archive, m_log, m_cmdLine)
53 {
54 const std::string sessionInfo = "#sessionInfo commandLineParameters \"";
55 m_log.writeSessionInfo(sessionInfo + m_cmdLine.getInitialCmdLine() + "\"\n");
56 }
57
iterate(void)58 inline bool iterate(void)
59 {
60 return m_app.iterate();
61 }
62
getResult(void) const63 inline const tcu::TestRunStatus& getResult(void) const
64 {
65 return m_app.getResult();
66 }
67
68 private:
69 tcu::CommandLine m_cmdLine;
70 tcu::TestLog m_log;
71 tcu::App m_app;
72 };
73
appendConfigArgs(const Config & config,std::vector<std::string> & args,const char * fboConfig)74 static void appendConfigArgs(const Config& config, std::vector<std::string>& args, const char* fboConfig)
75 {
76 if (fboConfig != NULL)
77 {
78 args.push_back(string("--deqp-gl-config-name=") + fboConfig);
79 args.push_back("--deqp-surface-type=fbo");
80 }
81
82 if (config.type != CONFIGTYPE_DEFAULT)
83 {
84 // \todo [2013-05-06 pyry] Test all surface types for some configs?
85 if (fboConfig == NULL)
86 {
87 if (config.surfaceTypes & SURFACETYPE_WINDOW)
88 args.push_back("--deqp-surface-type=window");
89 else if (config.surfaceTypes & SURFACETYPE_PBUFFER)
90 args.push_back("--deqp-surface-type=pbuffer");
91 else if (config.surfaceTypes & SURFACETYPE_PIXMAP)
92 args.push_back("--deqp-surface-type=pixmap");
93 }
94
95 args.push_back(string("--deqp-gl-config-id=") + de::toString(config.id));
96
97 if (config.type == CONFIGTYPE_EGL)
98 args.push_back("--deqp-gl-context-type=egl");
99 else if (config.type == CONFIGTYPE_WGL)
100 args.push_back("--deqp-gl-context-type=wgl");
101 }
102 }
103
104 typedef struct configInfo
105 {
106 deInt32 redBits;
107 deInt32 greenBits;
108 deInt32 blueBits;
109 deInt32 alphaBits;
110 deInt32 depthBits;
111 deInt32 stencilBits;
112 deInt32 samples;
113 } configInfo;
114
parseConfigBitsFromName(const char * configName)115 static configInfo parseConfigBitsFromName(const char* configName)
116 {
117 configInfo cfgInfo;
118 static const struct
119 {
120 const char* name;
121 int redBits;
122 int greenBits;
123 int blueBits;
124 int alphaBits;
125 } colorCfgs[] = {
126 { "rgba8888", 8, 8, 8, 8 }, { "rgb565", 5, 6, 5, 0 },
127 };
128 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(colorCfgs); ndx++)
129 {
130 if (!strncmp(configName, colorCfgs[ndx].name, strlen(colorCfgs[ndx].name)))
131 {
132 cfgInfo.redBits = colorCfgs[ndx].redBits;
133 cfgInfo.greenBits = colorCfgs[ndx].greenBits;
134 cfgInfo.blueBits = colorCfgs[ndx].blueBits;
135 cfgInfo.alphaBits = colorCfgs[ndx].alphaBits;
136
137 configName += strlen(colorCfgs[ndx].name);
138 break;
139 }
140 }
141
142 static const struct
143 {
144 const char* name;
145 int depthBits;
146 } depthCfgs[] = {
147 { "d0", 0 }, { "d24", 24 },
148 };
149 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(depthCfgs); ndx++)
150 {
151 if (!strncmp(configName, depthCfgs[ndx].name, strlen(depthCfgs[ndx].name)))
152 {
153 cfgInfo.depthBits = depthCfgs[ndx].depthBits;
154
155 configName += strlen(depthCfgs[ndx].name);
156 break;
157 }
158 }
159
160 static const struct
161 {
162 const char* name;
163 int stencilBits;
164 } stencilCfgs[] = {
165 { "s0", 0 }, { "s8", 8 },
166 };
167 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(stencilCfgs); ndx++)
168 {
169 if (!strncmp(configName, stencilCfgs[ndx].name, strlen(stencilCfgs[ndx].name)))
170 {
171 cfgInfo.stencilBits = stencilCfgs[ndx].stencilBits;
172
173 configName += strlen(stencilCfgs[ndx].name);
174 break;
175 }
176 }
177
178 static const struct
179 {
180 const char* name;
181 int samples;
182 } multiSampleCfgs[] = {
183 { "ms0", 0 }, { "ms4", 4 },
184 };
185 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(multiSampleCfgs); ndx++)
186 {
187 if (!strncmp(configName, multiSampleCfgs[ndx].name, strlen(multiSampleCfgs[ndx].name)))
188 {
189 cfgInfo.samples = multiSampleCfgs[ndx].samples;
190
191 configName += strlen(multiSampleCfgs[ndx].name);
192 break;
193 }
194 }
195
196 return cfgInfo;
197 }
198
getApiName(glu::ApiType apiType)199 static const char* getApiName(glu::ApiType apiType)
200 {
201 if (apiType == glu::ApiType::es(2, 0))
202 return "gles2";
203 else if (apiType == glu::ApiType::es(3, 0))
204 return "gles3";
205 else if (apiType == glu::ApiType::es(3, 1))
206 return "gles31";
207 else if (apiType == glu::ApiType::es(3, 2))
208 return "gles32";
209 else if (apiType == glu::ApiType::core(3, 0))
210 return "gl30";
211 else if (apiType == glu::ApiType::core(3, 1))
212 return "gl31";
213 else if (apiType == glu::ApiType::core(3, 2))
214 return "gl32";
215 else if (apiType == glu::ApiType::core(3, 3))
216 return "gl33";
217 else if (apiType == glu::ApiType::core(4, 0))
218 return "gl40";
219 else if (apiType == glu::ApiType::core(4, 1))
220 return "gl41";
221 else if (apiType == glu::ApiType::core(4, 2))
222 return "gl42";
223 else if (apiType == glu::ApiType::core(4, 3))
224 return "gl43";
225 else if (apiType == glu::ApiType::core(4, 4))
226 return "gl44";
227 else if (apiType == glu::ApiType::core(4, 5))
228 return "gl45";
229 else if (apiType == glu::ApiType::core(4, 6))
230 return "gl46";
231 else
232 throw std::runtime_error("Unknown context type");
233 }
234
getCaseListFileOption(const char * mustpassDir,const char * apiName,const char * mustpassName)235 static const string getCaseListFileOption(const char* mustpassDir, const char* apiName, const char* mustpassName)
236 {
237 #if DE_OS == DE_OS_ANDROID
238 const string case_list_option = "--deqp-caselist-resource=";
239 #else
240 const string case_list_option = "--deqp-caselist-file=";
241 #endif
242 return case_list_option + mustpassDir + apiName + "-" + mustpassName + ".txt";
243 }
244
getLogFileName(const char * apiName,const char * configName,const int iterId,const int runId,const int width,const int height,const int seed)245 static const string getLogFileName(const char* apiName, const char* configName, const int iterId, const int runId,
246 const int width, const int height, const int seed)
247 {
248 string res = string("config-") + apiName + "-" + configName + "-cfg-" + de::toString(iterId) + "-run-" +
249 de::toString(runId) + "-width-" + de::toString(width) + "-height-" + de::toString(height);
250 if (seed != -1)
251 {
252 res += "-seed-" + de::toString(seed);
253 }
254 res += ".qpa";
255
256 return res;
257 }
258
getBaseOptions(std::vector<std::string> & args,const char * mustpassDir,const char * apiName,const char * configName,const char * screenRotation,int width,int height)259 static void getBaseOptions(std::vector<std::string>& args, const char* mustpassDir, const char* apiName,
260 const char* configName, const char* screenRotation, int width, int height)
261 {
262 args.push_back(getCaseListFileOption(mustpassDir, apiName, configName));
263 args.push_back(string("--deqp-screen-rotation=") + screenRotation);
264 args.push_back(string("--deqp-surface-width=") + de::toString(width));
265 args.push_back(string("--deqp-surface-height=") + de::toString(height));
266 args.push_back("--deqp-watchdog=disable");
267 }
268
isGLConfigCompatible(configInfo cfgInfo,const AOSPConfig & config)269 static bool isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig& config)
270 {
271 return cfgInfo.redBits == config.redBits && cfgInfo.greenBits == config.greenBits &&
272 cfgInfo.blueBits == config.blueBits && cfgInfo.alphaBits == config.alphaBits &&
273 cfgInfo.depthBits == config.depthBits && cfgInfo.stencilBits == config.stencilBits &&
274 cfgInfo.samples == config.samples;
275 }
276
getTestRunsForAOSPEGL(vector<TestRunParams> & runs,const ConfigList & configs)277 static void getTestRunsForAOSPEGL(vector<TestRunParams>& runs, const ConfigList& configs)
278 {
279 #include "glcAospMustpassEgl.hpp"
280
281 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_egl_first_cfg); ++i)
282 {
283 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_egl_first_cfg[i].glConfigName);
284
285 vector<AOSPConfig>::const_iterator cfgIter;
286 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
287 {
288 // find first compatible config
289 if ((*cfgIter).type == CONFIGTYPE_EGL && isGLConfigCompatible(cfgInfo, *cfgIter))
290 {
291 break;
292 }
293 }
294
295 if (cfgIter == configs.aospConfigs.end())
296 {
297 // No suitable configuration found. Skipping EGL tests
298 continue;
299 }
300
301 const char* apiName = "egl";
302
303 const int width = aosp_mustpass_egl_first_cfg[i].surfaceWidth;
304 const int height = aosp_mustpass_egl_first_cfg[i].surfaceHeight;
305
306 TestRunParams params;
307 params.logFilename =
308 getLogFileName(apiName, aosp_mustpass_egl_first_cfg[i].configName, 1, i, width, height, -1);
309 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_egl_first_cfg[i].configName,
310 aosp_mustpass_egl_first_cfg[i].screenRotation, width, height);
311
312 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_egl_first_cfg[i].glConfigName));
313
314 runs.push_back(params);
315 }
316 }
317
getTestRunsForAOSPES(vector<TestRunParams> & runs,const ConfigList & configs,const glu::ApiType apiType)318 static void getTestRunsForAOSPES(vector<TestRunParams>& runs, const ConfigList& configs, const glu::ApiType apiType)
319 {
320 #include "glcAospMustpassEs.hpp"
321
322 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_es_first_cfg); ++i)
323 {
324 if (!glu::contextSupports(glu::ContextType(apiType), aosp_mustpass_es_first_cfg[i].apiType))
325 continue;
326
327 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_es_first_cfg[i].glConfigName);
328
329 vector<AOSPConfig>::const_iterator cfgIter;
330 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
331 {
332 // find first compatible config
333 if (isGLConfigCompatible(cfgInfo, *cfgIter))
334 {
335 break;
336 }
337 }
338
339 if (cfgIter == configs.aospConfigs.end())
340 {
341 TCU_FAIL(("No suitable configuration found for GL config " +
342 de::toString(aosp_mustpass_es_first_cfg[i].glConfigName))
343 .c_str());
344 return;
345 }
346
347 const char* apiName = getApiName(aosp_mustpass_es_first_cfg[i].apiType);
348
349 const int width = aosp_mustpass_es_first_cfg[i].surfaceWidth;
350 const int height = aosp_mustpass_es_first_cfg[i].surfaceHeight;
351
352 TestRunParams params;
353 params.logFilename = getLogFileName(apiName, aosp_mustpass_es_first_cfg[i].configName, 1, i, width, height, -1);
354 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_es_first_cfg[i].configName,
355 aosp_mustpass_es_first_cfg[i].screenRotation, width, height);
356
357 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_es_first_cfg[i].glConfigName));
358
359 //set surface type
360 if ((*cfgIter).surfaceTypes & SURFACETYPE_WINDOW)
361 params.args.push_back("--deqp-surface-type=window");
362 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PBUFFER)
363 params.args.push_back("--deqp-surface-type=pbuffer");
364 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PIXMAP)
365 params.args.push_back("--deqp-surface-type=pixmap");
366 runs.push_back(params);
367 }
368 }
369
getTestRunsForNoContext(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs,const RunParams * runParams,const int numRunParams,const char * mustpassDir)370 static void getTestRunsForNoContext(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams,
371 const int numRunParams, const char* mustpassDir)
372 {
373 vector<Config>::const_iterator cfgIter = configs.configs.begin();
374
375 for (int i = 0; i < numRunParams; ++i)
376 {
377 if (!glu::contextSupports(glu::ContextType(type), runParams[i].apiType))
378 continue;
379
380 const char* apiName = getApiName(runParams[i].apiType);
381
382 const int width = runParams[i].surfaceWidth;
383 const int height = runParams[i].surfaceHeight;
384 const int seed = runParams[i].baseSeed;
385
386 TestRunParams params;
387 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
388
389 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
390 height);
391
392 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
393
394 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
395
396 runs.push_back(params);
397 }
398 }
399
getTestRunsForNoContextES(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)400 static void getTestRunsForNoContextES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
401 {
402 #include "glcKhronosMustpassEsNocontext.hpp"
403 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_es_nocontext_first_cfg,
404 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_nocontext_first_cfg), mustpassDir);
405 }
406
getTestRunsForSingleConfig(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs,const RunParams * runParams,const int numRunParams,const char * mustpassDir)407 static void getTestRunsForSingleConfig(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams,
408 const int numRunParams, const char* mustpassDir)
409 {
410 vector<Config>::const_iterator cfgIter = configs.configs.begin();
411
412 for (int i = 0; i < numRunParams; ++i)
413 {
414 if (type != runParams[i].apiType)
415 continue;
416
417 const char* apiName = getApiName(runParams[i].apiType);
418
419 const int width = runParams[i].surfaceWidth;
420 const int height = runParams[i].surfaceHeight;
421 const int seed = runParams[i].baseSeed;
422
423 TestRunParams params;
424 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
425
426 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
427 height);
428
429 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
430
431 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
432
433 runs.push_back(params);
434 }
435 }
getTestRunsForSingleConfigES(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)436 static void getTestRunsForSingleConfigES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
437 {
438 #include "glcKhronosMustpassEsSingleConfig.hpp"
439 getTestRunsForSingleConfig(type, runs, configs, khronos_mustpass_es_single_config_first_cfg,
440 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_single_config_first_cfg), mustpassDir);
441 }
442
getTestRunsForES(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)443 static void getTestRunsForES(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
444 {
445 getTestRunsForAOSPEGL(runs, configs);
446 getTestRunsForAOSPES(runs, configs, type);
447 getTestRunsForNoContextES(type, runs, configs);
448 getTestRunsForSingleConfigES(type, runs, configs);
449
450 #include "glcKhronosMustpassEs.hpp"
451
452 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
453 {
454 const bool isFirst = cfgIter == configs.configs.begin();
455 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_es_first_cfg) :
456 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_other_cfg);
457 const RunParams* runParams = isFirst ? khronos_mustpass_es_first_cfg : khronos_mustpass_es_other_cfg;
458
459 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
460 {
461 if (!glu::contextSupports(glu::ContextType(type), runParams[runNdx].apiType))
462 continue;
463
464 const char* apiName = getApiName(runParams[runNdx].apiType);
465
466 const int width = runParams[runNdx].surfaceWidth;
467 const int height = runParams[runNdx].surfaceHeight;
468 const int seed = runParams[runNdx].baseSeed;
469
470 TestRunParams params;
471
472 params.logFilename =
473 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
474
475 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
476 runParams[runNdx].screenRotation, width, height);
477 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
478
479 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
480
481 runs.push_back(params);
482 }
483 }
484 }
485
getTestRunsForNoContextGL(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)486 static void getTestRunsForNoContextGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
487 {
488 #include "glcKhronosMustpassGlNocontext.hpp"
489 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_gl_nocontext_first_cfg,
490 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_nocontext_first_cfg), mustpassDir);
491 }
getTestRunsForSingleConfigGL(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)492 static void getTestRunsForSingleConfigGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
493 {
494 #include "glcKhronosMustpassGlSingleConfig.hpp"
495 getTestRunsForSingleConfig(type, runs, configs, khronos_mustpass_gl_single_config_first_cfg,
496 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_single_config_first_cfg), mustpassDir);
497 }
498
getTestRunsForESForGL(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)499 static void getTestRunsForESForGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
500 {
501 DE_UNREF(type);
502 #include "glcKhronosMustpassAospForGl.hpp"
503
504 vector<Config>::const_iterator cfgIter = configs.configs.begin();
505 const int numRunParams = DE_LENGTH_OF_ARRAY(khronos_mustpass_aosp_for_gl_first_cfg);
506 const RunParams* runParams = khronos_mustpass_aosp_for_gl_first_cfg;
507
508 for (int i = 0; i < numRunParams; ++i)
509 {
510 const char* apiName = getApiName(runParams[i].apiType);
511
512 const int width = runParams[i].surfaceWidth;
513 const int height = runParams[i].surfaceHeight;
514 const int seed = runParams[i].baseSeed;
515
516 TestRunParams params;
517 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
518
519 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
520 height);
521
522 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
523
524 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
525
526 runs.push_back(params);
527 }
528 }
529
530
getTestRunsForGL(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)531 static void getTestRunsForGL(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
532 {
533 getTestRunsForESForGL(type, runs, configs);
534 getTestRunsForNoContextGL(type, runs, configs);
535 getTestRunsForSingleConfigGL(type, runs, configs);
536 #include "glcKhronosMustpassGl.hpp"
537
538 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
539 {
540 const bool isFirst = cfgIter == configs.configs.begin();
541 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_first_cfg) :
542 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_other_cfg);
543 const RunParams* runParams = isFirst ? khronos_mustpass_gl_first_cfg : khronos_mustpass_gl_other_cfg;
544
545 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
546 {
547 if (type != runParams[runNdx].apiType)
548 continue;
549
550 const char* apiName = getApiName(runParams[runNdx].apiType);
551
552 const int width = runParams[runNdx].surfaceWidth;
553 const int height = runParams[runNdx].surfaceHeight;
554 const int seed = runParams[runNdx].baseSeed;
555
556 TestRunParams params;
557
558 params.logFilename =
559 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
560
561 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
562 runParams[runNdx].screenRotation, width, height);
563 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
564
565 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
566
567 runs.push_back(params);
568 }
569 }
570 }
571
getTestRunParams(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)572 static void getTestRunParams(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
573 {
574 switch (type.getProfile())
575 {
576 case glu::PROFILE_CORE:
577 getTestRunsForGL(type, configs, runs);
578 break;
579 case glu::PROFILE_ES:
580 getTestRunsForES(type, configs, runs);
581 break;
582 default:
583 throw std::runtime_error("Unknown context type");
584 }
585 }
586
587 struct FileDeleter
588 {
operator ()glcts::FileDeleter589 void operator()(FILE* file) const
590 {
591 if (file)
592 fclose(file);
593 }
594 };
595
596 struct XmlWriterDeleter
597 {
operator ()glcts::XmlWriterDeleter598 void operator()(qpXmlWriter* writer) const
599 {
600 if (writer)
601 qpXmlWriter_destroy(writer);
602 }
603 };
604
getRunTypeName(glu::ApiType type)605 static const char* getRunTypeName(glu::ApiType type)
606 {
607 if (type == glu::ApiType::es(2, 0))
608 return "es2";
609 else if (type == glu::ApiType::es(3, 0))
610 return "es3";
611 else if (type == glu::ApiType::es(3, 1))
612 return "es31";
613 else if (type == glu::ApiType::es(3, 2))
614 return "es32";
615 else if (type == glu::ApiType::core(3, 0))
616 return "gl30";
617 else if (type == glu::ApiType::core(3, 1))
618 return "gl31";
619 else if (type == glu::ApiType::core(3, 2))
620 return "gl32";
621 else if (type == glu::ApiType::core(3, 3))
622 return "gl33";
623 else if (type == glu::ApiType::core(4, 0))
624 return "gl40";
625 else if (type == glu::ApiType::core(4, 1))
626 return "gl41";
627 else if (type == glu::ApiType::core(4, 2))
628 return "gl42";
629 else if (type == glu::ApiType::core(4, 3))
630 return "gl43";
631 else if (type == glu::ApiType::core(4, 4))
632 return "gl44";
633 else if (type == glu::ApiType::core(4, 5))
634 return "gl45";
635 else if (type == glu::ApiType::core(4, 6))
636 return "gl46";
637 else
638 return DE_NULL;
639 }
640
641 #define XML_CHECK(X) \
642 if (!(X)) \
643 throw tcu::Exception("Writing XML failed")
644
writeRunSummary(const TestRunSummary & summary,const char * filename)645 static void writeRunSummary(const TestRunSummary& summary, const char* filename)
646 {
647 de::UniquePtr<FILE, FileDeleter> out(fopen(filename, "wb"));
648 if (!out)
649 throw tcu::Exception(string("Failed to open ") + filename);
650
651 de::UniquePtr<qpXmlWriter, XmlWriterDeleter> writer(qpXmlWriter_createFileWriter(out.get(), DE_FALSE, DE_FALSE));
652 if (!writer)
653 throw std::bad_alloc();
654
655 XML_CHECK(qpXmlWriter_startDocument(writer.get()));
656
657 {
658 qpXmlAttribute attribs[2];
659
660 attribs[0] = qpSetStringAttrib("Type", getRunTypeName(summary.runType));
661 attribs[1] = qpSetBoolAttrib("Conformant", summary.isConformant ? DE_TRUE : DE_FALSE);
662
663 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Summary", DE_LENGTH_OF_ARRAY(attribs), attribs));
664 }
665
666 // Config run
667 {
668 qpXmlAttribute attribs[1];
669 attribs[0] = qpSetStringAttrib("FileName", summary.configLogFilename.c_str());
670 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Configs", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
671 qpXmlWriter_endElement(writer.get(), "Configs"));
672 }
673
674 // Record test run parameters (log filename & command line).
675 for (vector<TestRunParams>::const_iterator runIter = summary.runParams.begin(); runIter != summary.runParams.end();
676 ++runIter)
677 {
678 string cmdLine;
679 qpXmlAttribute attribs[2];
680
681 for (vector<string>::const_iterator argIter = runIter->args.begin(); argIter != runIter->args.end(); ++argIter)
682 {
683 if (argIter != runIter->args.begin())
684 cmdLine += " ";
685 cmdLine += *argIter;
686 }
687
688 attribs[0] = qpSetStringAttrib("FileName", runIter->logFilename.c_str());
689 attribs[1] = qpSetStringAttrib("CmdLine", cmdLine.c_str());
690
691 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestRun", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
692 qpXmlWriter_endElement(writer.get(), "TestRun"));
693 }
694
695 XML_CHECK(qpXmlWriter_endElement(writer.get(), "Summary"));
696 XML_CHECK(qpXmlWriter_endDocument(writer.get()));
697 }
698
699 #undef XML_CHECK
700
TestRunner(tcu::Platform & platform,tcu::Archive & archive,const char * waiverPath,const char * logDirPath,glu::ApiType type,deUint32 flags)701 TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* waiverPath,
702 const char* logDirPath, glu::ApiType type, deUint32 flags)
703 : m_platform(platform)
704 , m_archive(archive)
705 , m_waiverPath(waiverPath)
706 , m_logDirPath(logDirPath)
707 , m_type(type)
708 , m_flags(flags)
709 , m_iterState(ITERATE_INIT)
710 , m_curSession(DE_NULL)
711 , m_sessionsExecuted(0)
712 , m_sessionsPassed(0)
713 , m_sessionsFailed(0)
714 {
715 }
716
~TestRunner(void)717 TestRunner::~TestRunner(void)
718 {
719 delete m_curSession;
720 }
721
iterate(void)722 bool TestRunner::iterate(void)
723 {
724 switch (m_iterState)
725 {
726 case ITERATE_INIT:
727 init();
728 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
729 return true;
730
731 case ITERATE_DEINIT:
732 deinit();
733 m_iterState = ITERATE_INIT;
734 return false;
735
736 case ITERATE_INIT_SESSION:
737 DE_ASSERT(m_sessionIter != m_runSessions.end());
738 initSession(*m_sessionIter);
739 if (m_flags & PRINT_SUMMARY)
740 m_iterState = ITERATE_DEINIT_SESSION;
741 else
742 m_iterState = ITERATE_ITERATE_SESSION;
743 return true;
744
745 case ITERATE_DEINIT_SESSION:
746 deinitSession();
747 ++m_sessionIter;
748 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
749 return true;
750
751 case ITERATE_ITERATE_SESSION:
752 if (!iterateSession())
753 m_iterState = ITERATE_DEINIT_SESSION;
754 return true;
755
756 default:
757 DE_ASSERT(false);
758 return false;
759 }
760 }
761
init(void)762 void TestRunner::init(void)
763 {
764 DE_ASSERT(m_runSessions.empty() && m_summary.runParams.empty());
765
766 tcu::print("Running %s conformance\n", glu::getApiTypeDescription(m_type));
767
768 m_summary.runType = m_type;
769
770 // Get list of configs to test.
771 ConfigList configList;
772 getDefaultConfigList(m_platform, m_type, configList);
773
774 tcu::print(" found %d compatible and %d excluded configs\n", (int)configList.configs.size(),
775 (int)configList.excludedConfigs.size());
776
777 // Config list run.
778 {
779 const char* configLogFilename = "configs.qpa";
780 TestRunParams configRun;
781
782 configRun.logFilename = configLogFilename;
783 configRun.args.push_back("--deqp-case=CTS-Configs.*");
784 m_runSessions.push_back(configRun);
785
786 m_summary.configLogFilename = configLogFilename;
787 }
788
789 // Conformance test type specific runs
790 getTestRunParams(m_type, configList, m_runSessions);
791
792 // Record run params for summary.
793 for (std::vector<TestRunParams>::const_iterator runIter = m_runSessions.begin() + 1; runIter != m_runSessions.end();
794 ++runIter)
795 m_summary.runParams.push_back(*runIter);
796
797 // Session iterator
798 m_sessionIter = m_runSessions.begin();
799 }
800
deinit(void)801 void TestRunner::deinit(void)
802 {
803 // Print out totals.
804 bool isConformant_ = m_sessionsExecuted == m_sessionsPassed;
805 DE_ASSERT(m_sessionsExecuted == m_sessionsPassed + m_sessionsFailed);
806 tcu::print("\n%d/%d sessions passed, conformance test %s\n", m_sessionsPassed, m_sessionsExecuted,
807 isConformant_ ? "PASSED" : "FAILED");
808
809 m_summary.isConformant = isConformant_;
810
811 // Write out summary.
812 writeRunSummary(m_summary, de::FilePath::join(m_logDirPath, "cts-run-summary.xml").getPath());
813
814 m_runSessions.clear();
815 }
816
initSession(const TestRunParams & runParams)817 void TestRunner::initSession(const TestRunParams& runParams)
818 {
819 DE_ASSERT(!m_curSession);
820
821 tcu::print("\n Test run %d / %d\n", (int)(m_sessionIter - m_runSessions.begin() + 1), (int)m_runSessions.size());
822
823 // Compute final args for run.
824 vector<string> args(runParams.args);
825 args.push_back(string("--deqp-log-filename=") + de::FilePath::join(m_logDirPath, runParams.logFilename).getPath());
826
827 if (!(m_flags & VERBOSE_IMAGES))
828 args.push_back("--deqp-log-images=disable");
829
830 if (!(m_flags & VERBOSE_SHADERS))
831 args.push_back("--deqp-log-shader-sources=disable");
832
833 if (!m_waiverPath.empty())
834 args.push_back(string("--deqp-waiver-file=") + m_waiverPath);
835
836 std::ostringstream ostr;
837 std::ostream_iterator<string> out_it(ostr, ", ");
838 std::copy(args.begin(), args.end(), out_it);
839 tcu::print("\n Config: %s \n\n", ostr.str().c_str());
840
841 // Translate to argc, argv
842 vector<const char*> argv;
843 argv.push_back("cts-runner"); // Dummy binary name
844 for (vector<string>::const_iterator i = args.begin(); i != args.end(); i++)
845 argv.push_back(i->c_str());
846
847 // Create session
848 m_curSession = new RunSession(m_platform, m_archive, (int)argv.size(), &argv[0]);
849 }
850
deinitSession(void)851 void TestRunner::deinitSession(void)
852 {
853 DE_ASSERT(m_curSession);
854
855 // Collect results.
856 // \note NotSupported is treated as pass.
857 const tcu::TestRunStatus& result = m_curSession->getResult();
858 bool isOk = result.numFailed == 0 && result.isComplete;
859
860 DE_ASSERT(result.numExecuted == result.numPassed + result.numFailed + result.numNotSupported + result.numWarnings + result.numWaived);
861
862 m_sessionsExecuted += 1;
863 (isOk ? m_sessionsPassed : m_sessionsFailed) += 1;
864
865 delete m_curSession;
866 m_curSession = DE_NULL;
867 }
868
iterateSession(void)869 inline bool TestRunner::iterateSession(void)
870 {
871 return m_curSession->iterate();
872 }
873
874 } // glcts
875