1 //
2 // Copyright 2015 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // angle_deqp_gtest:
7 // dEQP and GoogleTest integration logic. Calls through to the random
8 // order executor.
9
10 #include <stdint.h>
11 #include <array>
12 #include <fstream>
13
14 #include <gtest/gtest.h>
15
16 #include "angle_deqp_libtester.h"
17 #include "common/Optional.h"
18 #include "common/angleutils.h"
19 #include "common/base/anglebase/no_destructor.h"
20 #include "common/debug.h"
21 #include "common/platform.h"
22 #include "common/string_utils.h"
23 #include "common/system_utils.h"
24 #include "platform/PlatformMethods.h"
25 #include "tests/test_utils/runner/TestSuite.h"
26 #include "util/OSWindow.h"
27 #include "util/test_utils.h"
28
29 namespace angle
30 {
31 namespace
32 {
33 #if !defined(NDEBUG)
34 constexpr bool kIsDebug = true;
35 #else
36 constexpr bool kIsDebug = false;
37 #endif // !defined(NDEBUG)
38
39 bool gGlobalError = false;
40 bool gExpectError = false;
41 bool gVerbose = false;
42
43 // Set this to true temporarily to enable image logging in release. Useful for diagnosing
44 // errors.
45 bool gLogImages = kIsDebug;
46
47 constexpr char kInfoTag[] = "*RESULT";
48
HandlePlatformError(PlatformMethods * platform,const char * errorMessage)49 void HandlePlatformError(PlatformMethods *platform, const char *errorMessage)
50 {
51 if (!gExpectError)
52 {
53 FAIL() << errorMessage;
54 }
55 gGlobalError = true;
56 }
57
58 // Relative to the ANGLE root folder.
59 constexpr char kCTSRootPath[] = "third_party/VK-GL-CTS/src/";
60 constexpr char kSupportPath[] = "src/tests/deqp_support/";
61
62 #define GLES_CTS_DIR(PATH) "external/openglcts/data/gl_cts/data/mustpass/gles/" PATH
63 #define GL_CTS_DIR(PATH) "external/openglcts/data/gl_cts/data/mustpass/gl/" PATH
64 #define EGL_CTS_DIR(PATH) "external/openglcts/data/gl_cts/data/mustpass/egl/" PATH
65
66 const char *gCaseListFiles[] = {
67 EGL_CTS_DIR("aosp_mustpass/main/egl-main.txt"),
68 GLES_CTS_DIR("aosp_mustpass/main/gles2-main.txt"),
69 GLES_CTS_DIR("aosp_mustpass/main/gles3-main.txt"),
70 GLES_CTS_DIR("aosp_mustpass/main/gles31-main.txt"),
71 GLES_CTS_DIR("khronos_mustpass/main/gles2-khr-main.txt"),
72 GLES_CTS_DIR("khronos_mustpass/main/gles3-khr-main.txt"),
73 GLES_CTS_DIR("khronos_mustpass/main/gles31-khr-main.txt"),
74 GLES_CTS_DIR("khronos_mustpass/main/gles32-khr-main.txt"),
75 GLES_CTS_DIR("khronos_mustpass_noctx/main/gles2-khr-noctx-main.txt"),
76 GLES_CTS_DIR("khronos_mustpass_noctx/main/gles32-khr-noctx-main.txt"),
77 GLES_CTS_DIR("khronos_mustpass_single/main/gles32-khr-single.txt"),
78 GLES_CTS_DIR("aosp_mustpass/main/gles3-rotate-landscape.txt"),
79 GLES_CTS_DIR("aosp_mustpass/main/gles3-rotate-reverse-portrait.txt"),
80 GLES_CTS_DIR("aosp_mustpass/main/gles3-rotate-reverse-landscape.txt"),
81 GLES_CTS_DIR("aosp_mustpass/main/gles31-rotate-landscape.txt"),
82 GLES_CTS_DIR("aosp_mustpass/main/gles31-rotate-reverse-portrait.txt"),
83 GLES_CTS_DIR("aosp_mustpass/main/gles31-rotate-reverse-landscape.txt"),
84 GLES_CTS_DIR("aosp_mustpass/main/gles3-multisample.txt"),
85 GLES_CTS_DIR("aosp_mustpass/main/gles3-565-no-depth-no-stencil.txt"),
86 GLES_CTS_DIR("aosp_mustpass/main/gles31-multisample.txt"),
87 GLES_CTS_DIR("aosp_mustpass/main/gles31-565-no-depth-no-stencil.txt"),
88 };
89
90 const std::vector<const char *> gTestSuiteConfigParameters[] = {
91 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // egl
92 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles2
93 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles3
94 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles31
95 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles2_khr
96 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles3_khr
97 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles31_khr
98 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles32_khr
99 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles2_khr_noctx
100 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles32_khr_noctx
101 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles32_khr_single
102 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles3_rotate90
103 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles3_rotate180
104 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles3_rotate270
105 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles31_rotate90
106 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles31_rotate180
107 {"--deqp-gl-config-name=rgba8888d24s8ms0"}, // gles31_rotate270
108 {"--deqp-gl-config-name=rgba8888d24s8ms4"}, // gles3_multisample
109 {"--deqp-gl-config-name=rgb565d0s0ms0"}, // gles3_rgb565_no_depth_no_stencil
110 {"--deqp-gl-config-name=rgba8888d24s8ms4"}, // gles31_multisample
111 {"--deqp-gl-config-name=rgb565d0s0ms0"}, // gles31_rgb565_no_depth_no_stencil
112 };
113
114 #undef GLES_CTS_DIR
115 #undef GL_CTS_DIR
116
117 const char *gTestExpectationsFiles[] = {
118 "deqp_egl_test_expectations.txt",
119 "deqp_gles2_test_expectations.txt",
120 "deqp_gles3_test_expectations.txt",
121 "deqp_gles31_test_expectations.txt",
122 "deqp_khr_gles2_test_expectations.txt",
123 "deqp_khr_gles3_test_expectations.txt",
124 "deqp_khr_gles31_test_expectations.txt",
125 "deqp_khr_gles32_test_expectations.txt",
126 "deqp_khr_noctx_gles2_test_expectations.txt",
127 "deqp_khr_noctx_gles32_test_expectations.txt",
128 "deqp_khr_single_gles32_test_expectations.txt",
129 "deqp_gles3_rotate_test_expectations.txt",
130 "deqp_gles3_rotate_test_expectations.txt",
131 "deqp_gles3_rotate_test_expectations.txt",
132 "deqp_gles31_rotate_test_expectations.txt",
133 "deqp_gles31_rotate_test_expectations.txt",
134 "deqp_gles31_rotate_test_expectations.txt",
135 "deqp_gles3_multisample_test_expectations.txt",
136 "deqp_gles3_565_no_depth_no_stencil_test_expectations.txt",
137 "deqp_gles31_multisample_test_expectations.txt",
138 "deqp_gles31_565_no_depth_no_stencil_test_expectations.txt",
139 };
140
141 using APIInfo = std::pair<const char *, GPUTestConfig::API>;
142
143 constexpr APIInfo kEGLDisplayAPIs[] = {
144 #if defined(ANGLE_PLATFORM_ANDROID)
145 {"native-gles", GPUTestConfig::kAPIGLES},
146 #endif
147 {"angle-d3d9", GPUTestConfig::kAPID3D9},
148 {"angle-d3d11", GPUTestConfig::kAPID3D11},
149 {"angle-d3d11-ref", GPUTestConfig::kAPID3D11},
150 {"angle-gl", GPUTestConfig::kAPIGLDesktop},
151 {"angle-gles", GPUTestConfig::kAPIGLES},
152 {"angle-metal", GPUTestConfig::kAPIMetal},
153 {"angle-null", GPUTestConfig::kAPIUnknown},
154 {"angle-swiftshader", GPUTestConfig::kAPISwiftShader},
155 {"angle-vulkan", GPUTestConfig::kAPIVulkan},
156 {"angle-webgpu", GPUTestConfig::kAPIWgpu},
157 {"win32", GPUTestConfig::kAPIUnknown},
158 {"x11", GPUTestConfig::kAPIUnknown}
159
160 };
161
162 constexpr char kdEQPEGLString[] = "--deqp-egl-display-type=";
163 constexpr char kANGLEEGLString[] = "--use-angle=";
164 constexpr char kANGLEPreRotation[] = "--emulated-pre-rotation=";
165 constexpr char kdEQPCaseString[] = "--deqp-case=";
166 constexpr char kVerboseString[] = "--verbose";
167 constexpr char kRenderDocString[] = "--renderdoc";
168 constexpr char kNoRenderDocString[] = "--no-renderdoc";
169 constexpr char kdEQPFlagsPrefix[] = "--deqp-";
170 constexpr char kGTestFilter[] = "--gtest_filter=";
171 constexpr char kdEQPSurfaceWidth[] = "--deqp-surface-width=";
172 constexpr char kdEQPSurfaceHeight[] = "--deqp-surface-height=";
173 constexpr char kdEQPBaseSeed[] = "--deqp-base-seed";
174 constexpr const char gdEQPLogImagesString[] = "--deqp-log-images=";
175
176 // Use the config name defined in gTestSuiteConfigParameters by default
177 // If gEGLConfigNameFromCmdLine is overwritten by --deqp-gl-config-name passed from command
178 // line arguments, for example:
179 // out/Debug/angle_deqp_egl_tests --verbose --deqp-gl-config-name=rgba8888d24s8
180 // use gEGLConfigNameFromCmdLine (rgba8888d24s8) instead.
181 // Invalid --deqp-gl-config-name value passed from command line arguments will be caught by
182 // glu::parseConfigBitsFromName() defined in gluRenderConfig.cpp, and it will cause tests
183 // to fail
184 constexpr const char gdEQPEGLConfigNameString[] = "--deqp-gl-config-name=";
185 const char *gEGLConfigNameFromCmdLine = "";
186
187 angle::base::NoDestructor<std::vector<char>> gFilterStringBuffer;
188
189 // For angle_deqp_gles3*_rotateN_tests, default gOptions.preRotation to N.
190 #if defined(ANGLE_DEQP_GLES3_ROTATE90_TESTS) || defined(ANGLE_DEQP_GLES31_ROTATE90_TESTS)
191 constexpr uint32_t kDefaultPreRotation = 90;
192 #elif defined(ANGLE_DEQP_GLES3_ROTATE180_TESTS) || defined(ANGLE_DEQP_GLES31_ROTATE180_TESTS)
193 constexpr uint32_t kDefaultPreRotation = 180;
194 #elif defined(ANGLE_DEQP_GLES3_ROTATE270_TESTS) || defined(ANGLE_DEQP_GLES31_ROTATE270_TESTS)
195 constexpr uint32_t kDefaultPreRotation = 270;
196 #else
197 constexpr uint32_t kDefaultPreRotation = 0;
198 #endif
199
200 #if defined(ANGLE_TEST_ENABLE_RENDERDOC_CAPTURE)
201 constexpr bool kEnableRenderDocCapture = true;
202 #else
203 constexpr bool kEnableRenderDocCapture = false;
204 #endif
205
206 constexpr dEQPDriverOption kDeqpDriverOption = dEQPDriverOption::ANGLE;
207
208 const APIInfo *gInitAPI = nullptr;
209 dEQPOptions gOptions = {
210 kDefaultPreRotation, // preRotation
211 kEnableRenderDocCapture, // enableRenderDocCapture
212 kDeqpDriverOption, // useANGLE
213 };
214
215 std::vector<const char *> gdEQPForwardFlags;
216
217 // Returns the default API for a platform.
GetDefaultAPIName()218 const char *GetDefaultAPIName()
219 {
220 #if defined(ANGLE_PLATFORM_ANDROID) || defined(ANGLE_PLATFORM_LINUX) || \
221 defined(ANGLE_PLATFORM_WINDOWS)
222 return "angle-vulkan";
223 #elif defined(ANGLE_PLATFORM_APPLE)
224 return "angle-gl";
225 #else
226 # error Unknown platform.
227 #endif
228 }
229
FindAPIInfo(const std::string & arg)230 const APIInfo *FindAPIInfo(const std::string &arg)
231 {
232 for (auto &displayAPI : kEGLDisplayAPIs)
233 {
234 if (arg == displayAPI.first)
235 {
236 return &displayAPI;
237 }
238 }
239 return nullptr;
240 }
241
GetDefaultAPIInfo()242 const APIInfo *GetDefaultAPIInfo()
243 {
244 const APIInfo *defaultInfo = FindAPIInfo(GetDefaultAPIName());
245 ASSERT(defaultInfo);
246 return defaultInfo;
247 }
248
GetTestStatLine(const std::string & key,const std::string & value)249 std::string GetTestStatLine(const std::string &key, const std::string &value)
250 {
251 return std::string(kInfoTag) + ": " + key + ": " + value + "\n";
252 }
253
254 // During the CaseList initialization we cannot use the GTEST FAIL macro to quit the program
255 // because the initialization is called outside of tests the first time.
Die()256 void Die()
257 {
258 exit(EXIT_FAILURE);
259 }
260
FindFileFromPath(const char * dirPath,const char * filePath)261 Optional<std::string> FindFileFromPath(const char *dirPath, const char *filePath)
262 {
263 std::stringstream strstr;
264 strstr << dirPath << filePath;
265 std::string path = strstr.str();
266
267 constexpr size_t kMaxFoundPathLen = 1000;
268 char foundPath[kMaxFoundPathLen];
269 if (angle::FindTestDataPath(path.c_str(), foundPath, kMaxFoundPathLen))
270 {
271 return std::string(foundPath);
272 }
273
274 return Optional<std::string>::Invalid();
275 }
276
FindCaseListPath(size_t testModuleIndex)277 Optional<std::string> FindCaseListPath(size_t testModuleIndex)
278 {
279 return FindFileFromPath(kCTSRootPath, gCaseListFiles[testModuleIndex]);
280 }
281
FindTestExpectationsPath(size_t testModuleIndex)282 Optional<std::string> FindTestExpectationsPath(size_t testModuleIndex)
283 {
284 return FindFileFromPath(kSupportPath, gTestExpectationsFiles[testModuleIndex]);
285 }
286
GetTestModuleIndex()287 size_t GetTestModuleIndex()
288 {
289 #ifdef ANGLE_DEQP_EGL_TESTS
290 return 0;
291 #endif
292
293 #ifdef ANGLE_DEQP_GLES2_TESTS
294 return 1;
295 #endif
296
297 #ifdef ANGLE_DEQP_GLES3_TESTS
298 return 2;
299 #endif
300
301 #ifdef ANGLE_DEQP_GLES31_TESTS
302 return 3;
303 #endif
304
305 #ifdef ANGLE_DEQP_KHR_GLES2_TESTS
306 return 4;
307 #endif
308
309 #ifdef ANGLE_DEQP_KHR_GLES3_TESTS
310 return 5;
311 #endif
312
313 #ifdef ANGLE_DEQP_KHR_GLES31_TESTS
314 return 6;
315 #endif
316
317 #ifdef ANGLE_DEQP_KHR_GLES32_TESTS
318 return 7;
319 #endif
320
321 #ifdef ANGLE_DEQP_KHR_NOCTX_GLES2_TESTS
322 return 8;
323 #endif
324
325 #ifdef ANGLE_DEQP_KHR_NOCTX_GLES32_TESTS
326 return 9;
327 #endif
328
329 #ifdef ANGLE_DEQP_KHR_SINGLE_GLES32_TESTS
330 return 10;
331 #endif
332
333 #ifdef ANGLE_DEQP_GLES3_ROTATE90_TESTS
334 return 11;
335 #endif
336
337 #ifdef ANGLE_DEQP_GLES3_ROTATE180_TESTS
338 return 12;
339 #endif
340
341 #ifdef ANGLE_DEQP_GLES3_ROTATE270_TESTS
342 return 13;
343 #endif
344
345 #ifdef ANGLE_DEQP_GLES31_ROTATE90_TESTS
346 return 14;
347 #endif
348
349 #ifdef ANGLE_DEQP_GLES31_ROTATE180_TESTS
350 return 15;
351 #endif
352
353 #ifdef ANGLE_DEQP_GLES31_ROTATE270_TESTS
354 return 16;
355 #endif
356
357 #ifdef ANGLE_DEQP_GLES3_MULTISAMPLE_TESTS
358 return 17;
359 #endif
360
361 #ifdef ANGLE_DEQP_GLES3_565_NO_DEPTH_NO_STENCIL_TESTS
362 return 18;
363 #endif
364
365 #ifdef ANGLE_DEQP_GLES31_MULTISAMPLE_TESTS
366 return 19;
367 #endif
368
369 #ifdef ANGLE_DEQP_GLES31_565_NO_DEPTH_NO_STENCIL_TESTS
370 return 20;
371 #endif
372 }
373
374 class dEQPCaseList
375 {
376 public:
377 dEQPCaseList(size_t testModuleIndex);
378
379 struct CaseInfo
380 {
CaseInfoangle::__anon6ddab4f20111::dEQPCaseList::CaseInfo381 CaseInfo(const std::string &testNameIn, int expectationIn)
382 : testName(testNameIn), expectation(expectationIn)
383 {}
384
385 std::string testName;
386 int expectation;
387 };
388
389 void initialize();
390
getCaseInfo(size_t caseIndex) const391 const CaseInfo &getCaseInfo(size_t caseIndex) const
392 {
393 ASSERT(mInitialized);
394 ASSERT(caseIndex < mCaseInfoList.size());
395 return mCaseInfoList[caseIndex];
396 }
397
numCases() const398 size_t numCases() const
399 {
400 ASSERT(mInitialized);
401 return mCaseInfoList.size();
402 }
403
404 private:
405 std::vector<CaseInfo> mCaseInfoList;
406 size_t mTestModuleIndex;
407 bool mInitialized = false;
408 };
409
dEQPCaseList(size_t testModuleIndex)410 dEQPCaseList::dEQPCaseList(size_t testModuleIndex) : mTestModuleIndex(testModuleIndex) {}
411
initialize()412 void dEQPCaseList::initialize()
413 {
414 if (mInitialized)
415 {
416 return;
417 }
418
419 mInitialized = true;
420
421 Optional<std::string> caseListPath = FindCaseListPath(mTestModuleIndex);
422 if (!caseListPath.valid())
423 {
424 std::cerr << "Failed to find case list file." << std::endl;
425 Die();
426 }
427
428 Optional<std::string> testExpectationsPath = FindTestExpectationsPath(mTestModuleIndex);
429 if (!testExpectationsPath.valid())
430 {
431 std::cerr << "Failed to find test expectations file." << std::endl;
432 Die();
433 }
434
435 GPUTestConfig::API api = GetDefaultAPIInfo()->second;
436 // Set the API from the command line, or using the default platform API.
437 if (gInitAPI)
438 {
439 api = gInitAPI->second;
440 }
441
442 GPUTestConfig testConfig = GPUTestConfig(api, gOptions.preRotation);
443
444 #if !defined(ANGLE_PLATFORM_ANDROID)
445 // Note: These prints mess up parsing of test list when running on Android.
446 std::cout << "Using test config with:" << std::endl;
447 for (uint32_t condition : testConfig.getConditions())
448 {
449 const char *name = GetConditionName(condition);
450 if (name != nullptr)
451 {
452 std::cout << " " << name << std::endl;
453 }
454 }
455 #endif
456
457 TestSuite *testSuite = TestSuite::GetInstance();
458
459 if (!testSuite->loadTestExpectationsFromFileWithConfig(testConfig,
460 testExpectationsPath.value()))
461 {
462 Die();
463 }
464
465 std::ifstream caseListStream(caseListPath.value());
466 if (caseListStream.fail())
467 {
468 std::cerr << "Failed to load the case list." << std::endl;
469 Die();
470 }
471
472 while (!caseListStream.eof())
473 {
474 std::string inString;
475 std::getline(caseListStream, inString);
476
477 std::string testName = TrimString(inString, kWhitespaceASCII);
478 if (testName.empty())
479 continue;
480 int expectation = testSuite->getTestExpectation(testName);
481 mCaseInfoList.push_back(CaseInfo(testName, expectation));
482 }
483 }
484
IsPassingResult(dEQPTestResult result)485 bool IsPassingResult(dEQPTestResult result)
486 {
487 // Check the global error flag for unexpected platform errors.
488 if (gGlobalError)
489 {
490 gGlobalError = false;
491 return false;
492 }
493
494 switch (result)
495 {
496 case dEQPTestResult::Fail:
497 case dEQPTestResult::Exception:
498 return false;
499
500 default:
501 return true;
502 }
503 }
504
GetTestList(size_t testModuleIndex)505 const dEQPCaseList &GetTestList(size_t testModuleIndex)
506 {
507 static dEQPCaseList sCaseList(testModuleIndex);
508 sCaseList.initialize();
509 return sCaseList;
510 }
511
GetTestCaseName(size_t testModuleIndex,size_t caseIndex)512 const std::string GetTestCaseName(size_t testModuleIndex, size_t caseIndex)
513 {
514 const auto &caseInfo = GetTestList(testModuleIndex).getCaseInfo(caseIndex);
515 return caseInfo.testName;
516 }
517
518 class dEQPTestSuiteStats
519 {
520 public:
dEQPTestSuiteStats()521 dEQPTestSuiteStats() {}
522
523 private:
524 void setUpTestStats();
525 void printTestStats();
526 void countTestResult(dEQPTestResult result);
527
528 uint32_t mTestCount;
529 uint32_t mPassedTestCount;
530 uint32_t mFailedTestCount;
531 uint32_t mTestExceptionCount;
532 uint32_t mNotSupportedTestCount;
533 uint32_t mSkippedTestCount;
534
535 std::vector<std::string> mUnexpectedFailed;
536 std::vector<std::string> mUnexpectedPasses;
537
538 friend class dEQPTest;
539 };
540
setUpTestStats()541 void dEQPTestSuiteStats::setUpTestStats()
542 {
543 mPassedTestCount = 0;
544 mFailedTestCount = 0;
545 mNotSupportedTestCount = 0;
546 mTestExceptionCount = 0;
547 mTestCount = 0;
548 mSkippedTestCount = 0;
549 mUnexpectedPasses.clear();
550 mUnexpectedFailed.clear();
551 }
552
printTestStats()553 void dEQPTestSuiteStats::printTestStats()
554 {
555 uint32_t crashedCount =
556 mTestCount - (mPassedTestCount + mFailedTestCount + mNotSupportedTestCount +
557 mTestExceptionCount + mSkippedTestCount);
558
559 std::cout << GetTestStatLine("Total", std::to_string(mTestCount));
560 std::cout << GetTestStatLine("Passed", std::to_string(mPassedTestCount));
561 std::cout << GetTestStatLine("Failed", std::to_string(mFailedTestCount));
562 std::cout << GetTestStatLine("Skipped", std::to_string(mSkippedTestCount));
563 std::cout << GetTestStatLine("Not Supported", std::to_string(mNotSupportedTestCount));
564 std::cout << GetTestStatLine("Exception", std::to_string(mTestExceptionCount));
565 std::cout << GetTestStatLine("Crashed", std::to_string(crashedCount));
566
567 if (!mUnexpectedPasses.empty())
568 {
569 std::cout << GetTestStatLine("Unexpected Passed Count",
570 std::to_string(mUnexpectedPasses.size()));
571 for (const std::string &testName : mUnexpectedPasses)
572 {
573 std::cout << GetTestStatLine("Unexpected Passed Tests", testName);
574 }
575 }
576
577 if (!mUnexpectedFailed.empty())
578 {
579 std::cout << GetTestStatLine("Unexpected Failed Count",
580 std::to_string(mUnexpectedFailed.size()));
581 for (const std::string &testName : mUnexpectedFailed)
582 {
583 std::cout << GetTestStatLine("Unexpected Failed Tests", testName);
584 }
585 }
586 }
587
countTestResult(dEQPTestResult result)588 void dEQPTestSuiteStats::countTestResult(dEQPTestResult result)
589 {
590 switch (result)
591 {
592 case dEQPTestResult::Pass:
593 mPassedTestCount++;
594 break;
595 case dEQPTestResult::Fail:
596 mFailedTestCount++;
597 break;
598 case dEQPTestResult::NotSupported:
599 mNotSupportedTestCount++;
600 break;
601 case dEQPTestResult::Exception:
602 mTestExceptionCount++;
603 break;
604 default:
605 std::cerr << "Unexpected test result code: " << static_cast<int>(result) << "\n";
606 break;
607 }
608 }
609
610 class dEQPTest : public testing::Test
611 {
612 public:
dEQPTest(size_t testModuleIndex,size_t caseIndex)613 dEQPTest(size_t testModuleIndex, size_t caseIndex)
614 : mTestModuleIndex(testModuleIndex), mTestCaseIndex(caseIndex)
615 {}
616
617 static void SetUpTestSuite();
618 static void TearDownTestSuite();
619
620 protected:
621 void TestBody() override;
622
623 private:
624 size_t mTestModuleIndex = 0;
625 size_t mTestCaseIndex = 0;
626
627 static dEQPTestSuiteStats sTestSuiteData;
628 };
629
630 dEQPTestSuiteStats dEQPTest::sTestSuiteData = dEQPTestSuiteStats();
631
632 // static function called once before running all of dEQPTest under the same test suite
SetUpTestSuite()633 void dEQPTest::SetUpTestSuite()
634 {
635 sTestSuiteData.setUpTestStats();
636
637 std::vector<const char *> argv;
638
639 // Reserve one argument for the binary name.
640 argv.push_back("");
641
642 // Add init api.
643 const char *targetApi = gInitAPI ? gInitAPI->first : GetDefaultAPIName();
644 std::string apiArgString = std::string(kdEQPEGLString) + targetApi;
645 argv.push_back(apiArgString.c_str());
646
647 std::string configNameFromCmdLineString =
648 std::string(gdEQPEGLConfigNameString) + gEGLConfigNameFromCmdLine;
649
650 // Add test config parameters
651 for (const char *configParam : gTestSuiteConfigParameters[GetTestModuleIndex()])
652 {
653 // Check if we pass --deqp-gl-config-name from the command line, if yes, use the one from
654 // command line
655 if (std::strlen(gEGLConfigNameFromCmdLine) > 0 &&
656 std::strncmp(configParam, gdEQPEGLConfigNameString, strlen(gdEQPEGLConfigNameString)) ==
657 0)
658 {
659 configParam = configNameFromCmdLineString.c_str();
660 }
661 argv.push_back(configParam);
662 }
663
664 // Hide SwiftShader window to prevent a race with Xvfb causing hangs on test bots
665 if (gInitAPI && gInitAPI->second == GPUTestConfig::kAPISwiftShader)
666 {
667 argv.push_back("--deqp-visibility=hidden");
668 }
669
670 TestSuite *testSuite = TestSuite::GetInstance();
671
672 std::stringstream logNameStream;
673 logNameStream << "TestResults";
674 if (testSuite->getBatchId() != -1)
675 {
676 logNameStream << "-Batch" << std::setfill('0') << std::setw(3) << testSuite->getBatchId();
677 }
678 logNameStream << ".qpa";
679
680 std::stringstream logArgStream;
681 logArgStream << "--deqp-log-filename="
682 << testSuite->reserveTestArtifactPath(logNameStream.str());
683
684 std::string logNameString = logArgStream.str();
685 argv.push_back(logNameString.c_str());
686
687 if (!gLogImages)
688 {
689 argv.push_back("--deqp-log-images=disable");
690 }
691
692 // Flushing during multi-process execution punishes HDDs. http://anglebug.com/42263718
693 if (testSuite->getBatchId() != -1)
694 {
695 argv.push_back("--deqp-log-flush=disable");
696 }
697
698 // Add any additional flags specified from command line to be forwarded to dEQP.
699 argv.insert(argv.end(), gdEQPForwardFlags.begin(), gdEQPForwardFlags.end());
700
701 // Init the platform.
702 if (!deqp_libtester_init_platform(static_cast<int>(argv.size()), argv.data(),
703 reinterpret_cast<void *>(&HandlePlatformError), gOptions))
704 {
705 std::cout << "Aborting test due to dEQP initialization error." << std::endl;
706 exit(1);
707 }
708 }
709
710 // static function called once after running all of dEQPTest under the same test suite
TearDownTestSuite()711 void dEQPTest::TearDownTestSuite()
712 {
713 sTestSuiteData.printTestStats();
714 deqp_libtester_shutdown_platform();
715 }
716
717 // TestBody() is called once for each dEQPTest
TestBody()718 void dEQPTest::TestBody()
719 {
720 if (sTestSuiteData.mTestExceptionCount > 1)
721 {
722 std::cout << "Too many exceptions, skipping all remaining tests." << std::endl;
723 return;
724 }
725
726 const auto &caseInfo = GetTestList(mTestModuleIndex).getCaseInfo(mTestCaseIndex);
727
728 // Tests that crash exit the harness before collecting the result. To tally the number of
729 // crashed tests we track how many tests we "tried" to run.
730 sTestSuiteData.mTestCount++;
731
732 if (caseInfo.expectation == GPUTestExpectationsParser::kGpuTestSkip)
733 {
734 sTestSuiteData.mSkippedTestCount++;
735 std::cout << "Test skipped.\n";
736 return;
737 }
738
739 TestSuite *testSuite = TestSuite::GetInstance();
740 testSuite->maybeUpdateTestTimeout(caseInfo.expectation);
741
742 gExpectError = (caseInfo.expectation != GPUTestExpectationsParser::kGpuTestPass);
743 dEQPTestResult result = deqp_libtester_run(caseInfo.testName.c_str());
744
745 bool testSucceeded = IsPassingResult(result);
746
747 if (!testSucceeded && caseInfo.expectation == GPUTestExpectationsParser::kGpuTestFlaky)
748 {
749 result = deqp_libtester_run(caseInfo.testName.c_str());
750 testSucceeded = IsPassingResult(result);
751 }
752
753 sTestSuiteData.countTestResult(result);
754
755 if (caseInfo.expectation == GPUTestExpectationsParser::kGpuTestPass ||
756 caseInfo.expectation == GPUTestExpectationsParser::kGpuTestFlaky)
757 {
758 EXPECT_TRUE(testSucceeded);
759
760 if (!testSucceeded)
761 {
762 sTestSuiteData.mUnexpectedFailed.push_back(caseInfo.testName);
763 }
764 }
765 else if (testSucceeded)
766 {
767 std::cout << "Test expected to fail but passed!" << std::endl;
768 sTestSuiteData.mUnexpectedPasses.push_back(caseInfo.testName);
769 }
770 }
771
HandleDisplayType(const char * displayTypeString)772 void HandleDisplayType(const char *displayTypeString)
773 {
774 std::stringstream argStream;
775
776 if (gInitAPI)
777 {
778 std::cout << "Cannot specify two EGL displays!" << std::endl;
779 exit(1);
780 }
781
782 argStream << displayTypeString;
783 std::string arg = argStream.str();
784 gInitAPI = FindAPIInfo(arg);
785
786 if (!gInitAPI && strncmp(displayTypeString, "angle-", strlen("angle-")) != 0)
787 {
788 std::stringstream argStream2;
789 argStream2 << "angle-" << displayTypeString;
790 std::string arg2 = argStream2.str();
791 gInitAPI = FindAPIInfo(arg2);
792 }
793
794 if (!gInitAPI)
795 {
796 std::cout << "Unknown API: " << displayTypeString << std::endl;
797 exit(1);
798 }
799 }
800
HandlePreRotation(const char * preRotationString)801 void HandlePreRotation(const char *preRotationString)
802 {
803 std::istringstream argStream(preRotationString);
804
805 uint32_t preRotation = 0;
806 argStream >> preRotation;
807
808 if (!argStream ||
809 (preRotation != 0 && preRotation != 90 && preRotation != 180 && preRotation != 270))
810 {
811 std::cout << "Invalid PreRotation '" << preRotationString
812 << "'; must be either 0, 90, 180 or 270" << std::endl;
813 exit(1);
814 }
815
816 gOptions.preRotation = preRotation;
817 }
818
HandleEGLConfigName(const char * configNameString)819 void HandleEGLConfigName(const char *configNameString)
820 {
821 gEGLConfigNameFromCmdLine = configNameString;
822 }
823
824 // The --deqp-case flag takes a case expression that is parsed into a --gtest_filter. It
825 // converts the "dEQP" style names (functional.thing.*) into "GoogleTest" style names
826 // (functional_thing_*). Currently it does not handle multiple tests and multiple filters in
827 // different arguments.
HandleFilterArg(const char * filterString,int * argc,int argIndex,char ** argv)828 void HandleFilterArg(const char *filterString, int *argc, int argIndex, char **argv)
829 {
830 std::string googleTestFilter = ReplaceDashesWithQuestionMark(filterString);
831
832 gFilterStringBuffer->resize(googleTestFilter.size() + 3 + strlen(kGTestFilter), 0);
833 std::fill(gFilterStringBuffer->begin(), gFilterStringBuffer->end(), 0);
834
835 int bytesWritten = snprintf(gFilterStringBuffer->data(), gFilterStringBuffer->size() - 1,
836 "%s*%s", kGTestFilter, googleTestFilter.c_str());
837 if (bytesWritten <= 0 || static_cast<size_t>(bytesWritten) >= gFilterStringBuffer->size() - 1)
838 {
839 std::cout << "Error parsing filter string: " << filterString;
840 exit(1);
841 }
842
843 argv[argIndex] = gFilterStringBuffer->data();
844 }
845
HandleLogImages(const char * logImagesString)846 void HandleLogImages(const char *logImagesString)
847 {
848 if (strcmp(logImagesString, "enable") == 0)
849 {
850 gLogImages = true;
851 }
852 else if (strcmp(logImagesString, "disable") == 0)
853 {
854 gLogImages = false;
855 }
856 else
857 {
858 std::cout << "Error parsing log images setting. Use enable/disable.";
859 exit(1);
860 }
861 }
862
RegisterGLCTSTests()863 void RegisterGLCTSTests()
864 {
865 size_t testModuleIndex = GetTestModuleIndex();
866
867 const dEQPCaseList &caseList = GetTestList(testModuleIndex);
868
869 for (size_t caseIndex = 0; caseIndex < caseList.numCases(); ++caseIndex)
870 {
871 auto factory = [testModuleIndex, caseIndex]() {
872 return new dEQPTest(testModuleIndex, caseIndex);
873 };
874
875 const std::string testCaseName = GetTestCaseName(testModuleIndex, caseIndex);
876 size_t pos = testCaseName.find('.');
877 ASSERT(pos != std::string::npos);
878 // testCaseName comes from one of the mustpass files in gCaseListFiles.
879 // All of the testCaseName in the same mustpass file starts with the same testSuiteName
880 // prefix. Which mustpass file to load the set of testCaseName depends on testModuleIndex
881 // compiled into the deqp test application binary. For now, only one testModuleIndex is
882 // compiled in a deqp test application binary, meaning all of the tests invoked by same deqp
883 // test application binary are under the same test suite.
884 std::string testSuiteName = testCaseName.substr(0, pos);
885 std::string testName = testCaseName.substr(pos + 1);
886 testing::RegisterTest(testSuiteName.c_str(), testName.c_str(), nullptr, nullptr, __FILE__,
887 __LINE__, factory);
888 }
889 }
890 } // anonymous namespace
891
892 // Called from main() to process command-line arguments.
RunGLCTSTests(int * argc,char ** argv)893 int RunGLCTSTests(int *argc, char **argv)
894 {
895 int argIndex = 0;
896 while (argIndex < *argc)
897 {
898 if (strncmp(argv[argIndex], kdEQPEGLString, strlen(kdEQPEGLString)) == 0)
899 {
900 HandleDisplayType(argv[argIndex] + strlen(kdEQPEGLString));
901 }
902 else if (strncmp(argv[argIndex], kANGLEEGLString, strlen(kANGLEEGLString)) == 0)
903 {
904 HandleDisplayType(argv[argIndex] + strlen(kANGLEEGLString));
905 }
906 else if (strncmp(argv[argIndex], kANGLEPreRotation, strlen(kANGLEPreRotation)) == 0)
907 {
908 HandlePreRotation(argv[argIndex] + strlen(kANGLEPreRotation));
909 }
910 else if (strncmp(argv[argIndex], gdEQPEGLConfigNameString,
911 strlen(gdEQPEGLConfigNameString)) == 0)
912 {
913 HandleEGLConfigName(argv[argIndex] + strlen(gdEQPEGLConfigNameString));
914 }
915 else if (strncmp(argv[argIndex], kdEQPCaseString, strlen(kdEQPCaseString)) == 0)
916 {
917 HandleFilterArg(argv[argIndex] + strlen(kdEQPCaseString), argc, argIndex, argv);
918 }
919 else if (strncmp(argv[argIndex], kGTestFilter, strlen(kGTestFilter)) == 0)
920 {
921 HandleFilterArg(argv[argIndex] + strlen(kGTestFilter), argc, argIndex, argv);
922 }
923 else if (strncmp(argv[argIndex], kVerboseString, strlen(kVerboseString)) == 0 ||
924 strcmp(argv[argIndex], "-v") == 0)
925 {
926 gVerbose = true;
927 }
928 else if (strncmp(argv[argIndex], gdEQPLogImagesString, strlen(gdEQPLogImagesString)) == 0)
929 {
930 HandleLogImages(argv[argIndex] + strlen(gdEQPLogImagesString));
931 }
932 else if (strncmp(argv[argIndex], kRenderDocString, strlen(kRenderDocString)) == 0)
933 {
934 gOptions.enableRenderDocCapture = true;
935 }
936 else if (strncmp(argv[argIndex], kNoRenderDocString, strlen(kNoRenderDocString)) == 0)
937 {
938 gOptions.enableRenderDocCapture = false;
939 }
940 else if (strncmp(argv[argIndex], kdEQPFlagsPrefix, strlen(kdEQPFlagsPrefix)) == 0)
941 {
942 gdEQPForwardFlags.push_back(argv[argIndex]);
943 }
944 else if (strncmp(argv[argIndex], kdEQPSurfaceWidth, strlen(kdEQPSurfaceWidth)) == 0)
945 {
946 gdEQPForwardFlags.push_back(argv[argIndex]);
947 }
948 else if (strncmp(argv[argIndex], kdEQPSurfaceHeight, strlen(kdEQPSurfaceHeight)) == 0)
949 {
950 gdEQPForwardFlags.push_back(argv[argIndex]);
951 }
952 else if (strncmp(argv[argIndex], kdEQPBaseSeed, strlen(kdEQPBaseSeed)) == 0)
953 {
954 gdEQPForwardFlags.push_back(argv[argIndex]);
955 }
956 argIndex++;
957 }
958
959 GPUTestConfig::API api = GetDefaultAPIInfo()->second;
960 if (gInitAPI)
961 {
962 api = gInitAPI->second;
963 // On Android, if --deqp-egl-display-type=native-gles, set driverOption to NATIVE
964 // We will load egl libs from native gles driver instead of ANGLE.
965 #if defined(ANGLE_PLATFORM_ANDROID)
966 if (strcmp(gInitAPI->first, "native-gles") == 0)
967 {
968 gOptions.driverOption = dEQPDriverOption::NATIVE;
969 }
970 #endif
971 }
972 if (gOptions.preRotation != 0 && api != GPUTestConfig::kAPIVulkan &&
973 api != GPUTestConfig::kAPISwiftShader)
974 {
975 std::cout << "PreRotation is only supported on Vulkan" << std::endl;
976 exit(1);
977 }
978
979 angle::TestSuite testSuite(argc, argv, RegisterGLCTSTests);
980 return testSuite.run();
981 }
982 } // namespace angle
983