1 //
2 // Copyright 2015 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // angle_deqp_gtest:
7 // dEQP and GoogleTest integration logic. Calls through to the random
8 // order executor.
9
10 #include <stdint.h>
11 #include <array>
12 #include <fstream>
13
14 #include <gtest/gtest.h>
15
16 #include "angle_deqp_libtester.h"
17 #include "common/Optional.h"
18 #include "common/angleutils.h"
19 #include "common/base/anglebase/no_destructor.h"
20 #include "common/debug.h"
21 #include "common/platform.h"
22 #include "common/string_utils.h"
23 #include "common/system_utils.h"
24 #include "platform/PlatformMethods.h"
25 #include "tests/test_utils/runner/TestSuite.h"
26 #include "util/OSWindow.h"
27 #include "util/test_utils.h"
28
29 namespace angle
30 {
31 namespace
32 {
33 #if !defined(NDEBUG)
34 constexpr bool kIsDebug = true;
35 #else
36 constexpr bool kIsDebug = false;
37 #endif // !defined(NDEBUG)
38
39 bool gGlobalError = false;
40 bool gExpectError = false;
41 bool gVerbose = false;
42
43 // Set this to true temporarily to enable image logging in release. Useful for diagnosing errors.
44 bool gLogImages = kIsDebug;
45
46 constexpr char kInfoTag[] = "*RESULT";
47
HandlePlatformError(PlatformMethods * platform,const char * errorMessage)48 void HandlePlatformError(PlatformMethods *platform, const char *errorMessage)
49 {
50 if (!gExpectError)
51 {
52 FAIL() << errorMessage;
53 }
54 gGlobalError = true;
55 }
56
57 // Relative to the ANGLE root folder.
58 constexpr char kCTSRootPath[] = "third_party/VK-GL-CTS/src/";
59 constexpr char kSupportPath[] = "src/tests/deqp_support/";
60
61 #define GLES_CTS_DIR(PATH) "external/openglcts/data/mustpass/gles/" PATH
62 #define GL_CTS_DIR(PATH) "external/openglcts/data/mustpass/gl/" PATH
63 #define EGL_CTS_DIR(PATH) "external/openglcts/data/mustpass/egl/" PATH
64
65 const char *gCaseListFiles[] = {
66 EGL_CTS_DIR("aosp_mustpass/main/egl-master.txt"),
67 GLES_CTS_DIR("aosp_mustpass/main/gles2-master.txt"),
68 GLES_CTS_DIR("aosp_mustpass/main/gles3-master.txt"),
69 GLES_CTS_DIR("aosp_mustpass/main/gles31-master.txt"),
70 GLES_CTS_DIR("khronos_mustpass/main/gles2-khr-master.txt"),
71 GLES_CTS_DIR("khronos_mustpass/main/gles3-khr-master.txt"),
72 GLES_CTS_DIR("khronos_mustpass/main/gles31-khr-master.txt"),
73 GLES_CTS_DIR("khronos_mustpass/main/gles32-khr-master.txt"),
74 GLES_CTS_DIR("khronos_mustpass_noctx/main/gles2-khr-noctx-master.txt"),
75 GLES_CTS_DIR("khronos_mustpass_noctx/main/gles32-khr-noctx-master.txt"),
76 GLES_CTS_DIR("khronos_mustpass_single/main/gles32-khr-single.txt"),
77 GLES_CTS_DIR("aosp_mustpass/main/gles3-rotate-landscape.txt"),
78 GLES_CTS_DIR("aosp_mustpass/main/gles3-rotate-reverse-portrait.txt"),
79 GLES_CTS_DIR("aosp_mustpass/main/gles3-rotate-reverse-landscape.txt"),
80 GLES_CTS_DIR("aosp_mustpass/main/gles31-rotate-landscape.txt"),
81 GLES_CTS_DIR("aosp_mustpass/main/gles31-rotate-reverse-portrait.txt"),
82 GLES_CTS_DIR("aosp_mustpass/main/gles31-rotate-reverse-landscape.txt"),
83 GL_CTS_DIR("khronos_mustpass/main/gl46-master.txt"),
84 };
85
86 #undef GLES_CTS_DIR
87 #undef GL_CTS_DIR
88
89 const char *gTestExpectationsFiles[] = {
90 "deqp_egl_test_expectations.txt",
91 "deqp_gles2_test_expectations.txt",
92 "deqp_gles3_test_expectations.txt",
93 "deqp_gles31_test_expectations.txt",
94 "deqp_khr_gles2_test_expectations.txt",
95 "deqp_khr_gles3_test_expectations.txt",
96 "deqp_khr_gles31_test_expectations.txt",
97 "deqp_khr_gles32_test_expectations.txt",
98 "deqp_khr_noctx_gles2_test_expectations.txt",
99 "deqp_khr_noctx_gles32_test_expectations.txt",
100 "deqp_khr_single_gles32_test_expectations.txt",
101 "deqp_gles3_rotate_test_expectations.txt",
102 "deqp_gles3_rotate_test_expectations.txt",
103 "deqp_gles3_rotate_test_expectations.txt",
104 "deqp_gles31_rotate_test_expectations.txt",
105 "deqp_gles31_rotate_test_expectations.txt",
106 "deqp_gles31_rotate_test_expectations.txt",
107 "deqp_gl46_test_expectations.txt",
108 };
109
110 using APIInfo = std::pair<const char *, GPUTestConfig::API>;
111
112 constexpr APIInfo kEGLDisplayAPIs[] = {
113 {"angle-d3d9", GPUTestConfig::kAPID3D9},
114 {"angle-d3d11", GPUTestConfig::kAPID3D11},
115 {"angle-d3d11-ref", GPUTestConfig::kAPID3D11},
116 {"angle-gl", GPUTestConfig::kAPIGLDesktop},
117 {"angle-gles", GPUTestConfig::kAPIGLES},
118 {"angle-metal", GPUTestConfig::kAPIMetal},
119 {"angle-null", GPUTestConfig::kAPIUnknown},
120 {"angle-swiftshader", GPUTestConfig::kAPISwiftShader},
121 {"angle-vulkan", GPUTestConfig::kAPIVulkan},
122 {"win32", GPUTestConfig::kAPIUnknown},
123 {"x11", GPUTestConfig::kAPIUnknown},
124 };
125
126 constexpr char kdEQPEGLString[] = "--deqp-egl-display-type=";
127 constexpr char kANGLEEGLString[] = "--use-angle=";
128 constexpr char kANGLEPreRotation[] = "--emulated-pre-rotation=";
129 constexpr char kdEQPCaseString[] = "--deqp-case=";
130 constexpr char kVerboseString[] = "--verbose";
131 constexpr char kRenderDocString[] = "--renderdoc";
132 constexpr char kNoRenderDocString[] = "--no-renderdoc";
133 constexpr char kdEQPFlagsPrefix[] = "--deqp-";
134 constexpr char kGTestFilter[] = "--gtest_filter=";
135
136 angle::base::NoDestructor<std::vector<char>> gFilterStringBuffer;
137
138 // For angle_deqp_gles3*_rotateN_tests, default gOptions.preRotation to N.
139 #if defined(ANGLE_DEQP_GLES3_ROTATE90_TESTS) || defined(ANGLE_DEQP_GLES31_ROTATE90_TESTS)
140 constexpr uint32_t kDefaultPreRotation = 90;
141 #elif defined(ANGLE_DEQP_GLES3_ROTATE180_TESTS) || defined(ANGLE_DEQP_GLES31_ROTATE180_TESTS)
142 constexpr uint32_t kDefaultPreRotation = 180;
143 #elif defined(ANGLE_DEQP_GLES3_ROTATE270_TESTS) || defined(ANGLE_DEQP_GLES31_ROTATE270_TESTS)
144 constexpr uint32_t kDefaultPreRotation = 270;
145 #else
146 constexpr uint32_t kDefaultPreRotation = 0;
147 #endif
148
149 #if defined(ANGLE_TEST_ENABLE_RENDERDOC_CAPTURE)
150 constexpr bool kEnableRenderDocCapture = true;
151 #else
152 constexpr bool kEnableRenderDocCapture = false;
153 #endif
154
155 const APIInfo *gInitAPI = nullptr;
156 dEQPOptions gOptions = {
157 kDefaultPreRotation, // preRotation
158 kEnableRenderDocCapture, // enableRenderDocCapture
159 };
160
161 constexpr const char gdEQPEGLConfigNameString[] = "--deqp-gl-config-name=";
162 constexpr const char gdEQPLogImagesString[] = "--deqp-log-images=";
163
164 // Default the config to RGBA8
165 const char *gEGLConfigName = "rgba8888d24s8";
166
167 std::vector<const char *> gdEQPForwardFlags;
168
169 // Returns the default API for a platform.
GetDefaultAPIName()170 const char *GetDefaultAPIName()
171 {
172 #if defined(ANGLE_PLATFORM_ANDROID) || defined(ANGLE_PLATFORM_LINUX) || \
173 defined(ANGLE_PLATFORM_WINDOWS)
174 return "angle-vulkan";
175 #elif defined(ANGLE_PLATFORM_APPLE)
176 return "angle-gl";
177 #else
178 # error Unknown platform.
179 #endif
180 }
181
FindAPIInfo(const std::string & arg)182 const APIInfo *FindAPIInfo(const std::string &arg)
183 {
184 for (auto &displayAPI : kEGLDisplayAPIs)
185 {
186 if (arg == displayAPI.first)
187 {
188 return &displayAPI;
189 }
190 }
191 return nullptr;
192 }
193
GetDefaultAPIInfo()194 const APIInfo *GetDefaultAPIInfo()
195 {
196 const APIInfo *defaultInfo = FindAPIInfo(GetDefaultAPIName());
197 ASSERT(defaultInfo);
198 return defaultInfo;
199 }
200
GetTestStatLine(const std::string & key,const std::string & value)201 std::string GetTestStatLine(const std::string &key, const std::string &value)
202 {
203 return std::string(kInfoTag) + ": " + key + ": " + value + "\n";
204 }
205
206 // During the CaseList initialization we cannot use the GTEST FAIL macro to quit the program because
207 // the initialization is called outside of tests the first time.
Die()208 void Die()
209 {
210 exit(EXIT_FAILURE);
211 }
212
FindFileFromPath(const char * dirPath,const char * filePath)213 Optional<std::string> FindFileFromPath(const char *dirPath, const char *filePath)
214 {
215 std::stringstream strstr;
216 strstr << dirPath << filePath;
217 std::string path = strstr.str();
218
219 constexpr size_t kMaxFoundPathLen = 1000;
220 char foundPath[kMaxFoundPathLen];
221 if (angle::FindTestDataPath(path.c_str(), foundPath, kMaxFoundPathLen))
222 {
223 return std::string(foundPath);
224 }
225
226 return Optional<std::string>::Invalid();
227 }
228
FindCaseListPath(size_t testModuleIndex)229 Optional<std::string> FindCaseListPath(size_t testModuleIndex)
230 {
231 return FindFileFromPath(kCTSRootPath, gCaseListFiles[testModuleIndex]);
232 }
233
FindTestExpectationsPath(size_t testModuleIndex)234 Optional<std::string> FindTestExpectationsPath(size_t testModuleIndex)
235 {
236 return FindFileFromPath(kSupportPath, gTestExpectationsFiles[testModuleIndex]);
237 }
238
239 class dEQPCaseList
240 {
241 public:
242 dEQPCaseList(size_t testModuleIndex);
243
244 struct CaseInfo
245 {
CaseInfoangle::__anon30235a810111::dEQPCaseList::CaseInfo246 CaseInfo(const std::string &testNameIn, int expectationIn)
247 : testName(testNameIn), expectation(expectationIn)
248 {}
249
250 std::string testName;
251 int expectation;
252 };
253
254 void initialize();
255
getCaseInfo(size_t caseIndex) const256 const CaseInfo &getCaseInfo(size_t caseIndex) const
257 {
258 ASSERT(mInitialized);
259 ASSERT(caseIndex < mCaseInfoList.size());
260 return mCaseInfoList[caseIndex];
261 }
262
numCases() const263 size_t numCases() const
264 {
265 ASSERT(mInitialized);
266 return mCaseInfoList.size();
267 }
268
269 private:
270 std::vector<CaseInfo> mCaseInfoList;
271 size_t mTestModuleIndex;
272 bool mInitialized = false;
273 };
274
dEQPCaseList(size_t testModuleIndex)275 dEQPCaseList::dEQPCaseList(size_t testModuleIndex) : mTestModuleIndex(testModuleIndex) {}
276
initialize()277 void dEQPCaseList::initialize()
278 {
279 if (mInitialized)
280 {
281 return;
282 }
283
284 mInitialized = true;
285
286 Optional<std::string> caseListPath = FindCaseListPath(mTestModuleIndex);
287 if (!caseListPath.valid())
288 {
289 std::cerr << "Failed to find case list file." << std::endl;
290 Die();
291 }
292
293 Optional<std::string> testExpectationsPath = FindTestExpectationsPath(mTestModuleIndex);
294 if (!testExpectationsPath.valid())
295 {
296 std::cerr << "Failed to find test expectations file." << std::endl;
297 Die();
298 }
299
300 GPUTestConfig::API api = GetDefaultAPIInfo()->second;
301 // Set the API from the command line, or using the default platform API.
302 if (gInitAPI)
303 {
304 api = gInitAPI->second;
305 }
306
307 GPUTestConfig testConfig = GPUTestConfig(api, gOptions.preRotation);
308
309 #if !defined(ANGLE_PLATFORM_ANDROID)
310 // Note: These prints mess up parsing of test list when running on Android.
311 std::cout << "Using test config with:" << std::endl;
312 for (uint32_t condition : testConfig.getConditions())
313 {
314 const char *name = GetConditionName(condition);
315 if (name != nullptr)
316 {
317 std::cout << " " << name << std::endl;
318 }
319 }
320 #endif
321
322 TestSuite *testSuite = TestSuite::GetInstance();
323
324 if (!testSuite->loadTestExpectationsFromFileWithConfig(testConfig,
325 testExpectationsPath.value()))
326 {
327 Die();
328 }
329
330 std::ifstream caseListStream(caseListPath.value());
331 if (caseListStream.fail())
332 {
333 std::cerr << "Failed to load the case list." << std::endl;
334 Die();
335 }
336
337 while (!caseListStream.eof())
338 {
339 std::string inString;
340 std::getline(caseListStream, inString);
341
342 std::string testName = TrimString(inString, kWhitespaceASCII);
343 if (testName.empty())
344 continue;
345 int expectation = testSuite->getTestExpectation(testName);
346 mCaseInfoList.push_back(CaseInfo(testName, expectation));
347 }
348
349 if (testSuite->logAnyUnusedTestExpectations())
350 {
351 Die();
352 }
353 }
354
IsPassingResult(dEQPTestResult result)355 bool IsPassingResult(dEQPTestResult result)
356 {
357 // Check the global error flag for unexpected platform errors.
358 if (gGlobalError)
359 {
360 gGlobalError = false;
361 return false;
362 }
363
364 switch (result)
365 {
366 case dEQPTestResult::Fail:
367 case dEQPTestResult::Exception:
368 return false;
369
370 default:
371 return true;
372 }
373 }
374
375 class dEQP : public testing::Test
376 {
377 public:
GetTestingRange(size_t testModuleIndex)378 static testing::internal::ParamGenerator<size_t> GetTestingRange(size_t testModuleIndex)
379 {
380 return testing::Range<size_t>(0, GetCaseList(testModuleIndex).numCases());
381 }
382
GetTestCaseName(size_t testModuleIndex,size_t caseIndex)383 static std::string GetTestCaseName(size_t testModuleIndex, size_t caseIndex)
384 {
385 const auto &caseInfo = GetCaseList(testModuleIndex).getCaseInfo(caseIndex);
386 return caseInfo.testName;
387 }
388
GetCaseList(size_t testModuleIndex)389 static const dEQPCaseList &GetCaseList(size_t testModuleIndex)
390 {
391 static dEQPCaseList sCaseList(testModuleIndex);
392 sCaseList.initialize();
393 return sCaseList;
394 }
395
396 static void SetUpTestCase();
397 static void TearDownTestCase();
398
dEQP(size_t testModuleIndex,size_t caseIndex)399 dEQP(size_t testModuleIndex, size_t caseIndex)
400 : mTestModuleIndex(testModuleIndex), mTestCaseIndex(caseIndex)
401 {}
402
403 protected:
TestBody()404 void TestBody() override
405 {
406 if (sTestExceptionCount > 1)
407 {
408 std::cout << "Too many exceptions, skipping all remaining tests." << std::endl;
409 return;
410 }
411
412 const auto &caseInfo = GetCaseList(mTestModuleIndex).getCaseInfo(mTestCaseIndex);
413
414 // Tests that crash exit the harness before collecting the result. To tally the number of
415 // crashed tests we track how many tests we "tried" to run.
416 sTestCount++;
417
418 if (caseInfo.expectation == GPUTestExpectationsParser::kGpuTestSkip)
419 {
420 sSkippedTestCount++;
421 std::cout << "Test skipped.\n";
422 return;
423 }
424
425 TestSuite *testSuite = TestSuite::GetInstance();
426 testSuite->maybeUpdateTestTimeout(caseInfo.expectation);
427
428 gExpectError = (caseInfo.expectation != GPUTestExpectationsParser::kGpuTestPass);
429 dEQPTestResult result = deqp_libtester_run(caseInfo.testName.c_str());
430
431 bool testSucceeded = IsPassingResult(result);
432
433 if (!testSucceeded && caseInfo.expectation == GPUTestExpectationsParser::kGpuTestFlaky)
434 {
435 result = deqp_libtester_run(caseInfo.testName.c_str());
436 testSucceeded = IsPassingResult(result);
437 }
438
439 countTestResult(result);
440
441 if (caseInfo.expectation == GPUTestExpectationsParser::kGpuTestPass ||
442 caseInfo.expectation == GPUTestExpectationsParser::kGpuTestFlaky)
443 {
444 EXPECT_TRUE(testSucceeded);
445
446 if (!testSucceeded)
447 {
448 sUnexpectedFailed.push_back(caseInfo.testName);
449 }
450 }
451 else if (testSucceeded)
452 {
453 std::cout << "Test expected to fail but passed!" << std::endl;
454 sUnexpectedPasses.push_back(caseInfo.testName);
455 }
456 }
457
countTestResult(dEQPTestResult result) const458 void countTestResult(dEQPTestResult result) const
459 {
460 switch (result)
461 {
462 case dEQPTestResult::Pass:
463 sPassedTestCount++;
464 break;
465 case dEQPTestResult::Fail:
466 sFailedTestCount++;
467 break;
468 case dEQPTestResult::NotSupported:
469 sNotSupportedTestCount++;
470 break;
471 case dEQPTestResult::Exception:
472 sTestExceptionCount++;
473 break;
474 default:
475 std::cerr << "Unexpected test result code: " << static_cast<int>(result) << "\n";
476 break;
477 }
478 }
479
PrintTestStats()480 static void PrintTestStats()
481 {
482 uint32_t crashedCount =
483 sTestCount - (sPassedTestCount + sFailedTestCount + sNotSupportedTestCount +
484 sTestExceptionCount + sSkippedTestCount);
485
486 std::cout << GetTestStatLine("Total", std::to_string(sTestCount));
487 std::cout << GetTestStatLine("Passed", std::to_string(sPassedTestCount));
488 std::cout << GetTestStatLine("Failed", std::to_string(sFailedTestCount));
489 std::cout << GetTestStatLine("Skipped", std::to_string(sSkippedTestCount));
490 std::cout << GetTestStatLine("Not Supported", std::to_string(sNotSupportedTestCount));
491 std::cout << GetTestStatLine("Exception", std::to_string(sTestExceptionCount));
492 std::cout << GetTestStatLine("Crashed", std::to_string(crashedCount));
493
494 if (!sUnexpectedPasses.empty())
495 {
496 std::cout << GetTestStatLine("Unexpected Passed Count",
497 std::to_string(sUnexpectedPasses.size()));
498 for (const std::string &testName : sUnexpectedPasses)
499 {
500 std::cout << GetTestStatLine("Unexpected Passed Tests", testName);
501 }
502 }
503
504 if (!sUnexpectedFailed.empty())
505 {
506 std::cout << GetTestStatLine("Unexpected Failed Count",
507 std::to_string(sUnexpectedFailed.size()));
508 for (const std::string &testName : sUnexpectedFailed)
509 {
510 std::cout << GetTestStatLine("Unexpected Failed Tests", testName);
511 }
512 }
513 }
514
515 static uint32_t sTestCount;
516 static uint32_t sPassedTestCount;
517 static uint32_t sFailedTestCount;
518 static uint32_t sTestExceptionCount;
519 static uint32_t sNotSupportedTestCount;
520 static uint32_t sSkippedTestCount;
521
522 static std::vector<std::string> sUnexpectedFailed;
523 static std::vector<std::string> sUnexpectedPasses;
524
525 size_t mTestModuleIndex = 0;
526 size_t mTestCaseIndex = 0;
527 };
528
529 uint32_t dEQP::sTestCount = 0;
530 uint32_t dEQP::sPassedTestCount = 0;
531 uint32_t dEQP::sFailedTestCount = 0;
532 uint32_t dEQP::sTestExceptionCount = 0;
533 uint32_t dEQP::sNotSupportedTestCount = 0;
534 uint32_t dEQP::sSkippedTestCount = 0;
535 std::vector<std::string> dEQP::sUnexpectedFailed;
536 std::vector<std::string> dEQP::sUnexpectedPasses;
537
538 // static
SetUpTestCase()539 void dEQP::SetUpTestCase()
540 {
541 sPassedTestCount = 0;
542 sFailedTestCount = 0;
543 sNotSupportedTestCount = 0;
544 sTestExceptionCount = 0;
545 sTestCount = 0;
546 sSkippedTestCount = 0;
547 sUnexpectedPasses.clear();
548 sUnexpectedFailed.clear();
549
550 std::vector<const char *> argv;
551
552 // Reserve one argument for the binary name.
553 argv.push_back("");
554
555 // Add init api.
556 const char *targetApi = gInitAPI ? gInitAPI->first : GetDefaultAPIName();
557 std::string apiArgString = std::string(kdEQPEGLString) + targetApi;
558 argv.push_back(apiArgString.c_str());
559
560 // Add config name
561 const char *targetConfigName = gEGLConfigName;
562 std::string configArgString = std::string(gdEQPEGLConfigNameString) + targetConfigName;
563 argv.push_back(configArgString.c_str());
564
565 // Hide SwiftShader window to prevent a race with Xvfb causing hangs on test bots
566 if (gInitAPI && gInitAPI->second == GPUTestConfig::kAPISwiftShader)
567 {
568 argv.push_back("--deqp-visibility=hidden");
569 }
570
571 TestSuite *testSuite = TestSuite::GetInstance();
572
573 std::stringstream logNameStream;
574 logNameStream << "TestResults";
575 if (testSuite->getBatchId() != -1)
576 {
577 logNameStream << "-Batch" << std::setfill('0') << std::setw(3) << testSuite->getBatchId();
578 }
579 logNameStream << ".qpa";
580
581 std::stringstream logArgStream;
582 logArgStream << "--deqp-log-filename="
583 << testSuite->reserveTestArtifactPath(logNameStream.str());
584
585 std::string logNameString = logArgStream.str();
586 argv.push_back(logNameString.c_str());
587
588 if (!gLogImages)
589 {
590 argv.push_back("--deqp-log-images=disable");
591 }
592
593 // Flushing during multi-process execution punishes HDDs. http://anglebug.com/5157
594 if (testSuite->getBatchId() != -1)
595 {
596 argv.push_back("--deqp-log-flush=disable");
597 }
598
599 // Add any additional flags specified from command line to be forwarded to dEQP.
600 argv.insert(argv.end(), gdEQPForwardFlags.begin(), gdEQPForwardFlags.end());
601
602 // Init the platform.
603 if (!deqp_libtester_init_platform(static_cast<int>(argv.size()), argv.data(),
604 reinterpret_cast<void *>(&HandlePlatformError), gOptions))
605 {
606 std::cout << "Aborting test due to dEQP initialization error." << std::endl;
607 exit(1);
608 }
609 }
610
611 // static
TearDownTestCase()612 void dEQP::TearDownTestCase()
613 {
614 PrintTestStats();
615 deqp_libtester_shutdown_platform();
616 }
617
HandleDisplayType(const char * displayTypeString)618 void HandleDisplayType(const char *displayTypeString)
619 {
620 std::stringstream argStream;
621
622 if (gInitAPI)
623 {
624 std::cout << "Cannot specify two EGL displays!" << std::endl;
625 exit(1);
626 }
627
628 argStream << displayTypeString;
629 std::string arg = argStream.str();
630 gInitAPI = FindAPIInfo(arg);
631
632 if (!gInitAPI && strncmp(displayTypeString, "angle-", strlen("angle-")) != 0)
633 {
634 std::stringstream argStream2;
635 argStream2 << "angle-" << displayTypeString;
636 std::string arg2 = argStream2.str();
637 gInitAPI = FindAPIInfo(arg2);
638
639 if (!gInitAPI)
640 {
641 std::cout << "Unknown API: " << displayTypeString << std::endl;
642 exit(1);
643 }
644 }
645 }
646
HandlePreRotation(const char * preRotationString)647 void HandlePreRotation(const char *preRotationString)
648 {
649 std::istringstream argStream(preRotationString);
650
651 uint32_t preRotation = 0;
652 argStream >> preRotation;
653
654 if (!argStream ||
655 (preRotation != 0 && preRotation != 90 && preRotation != 180 && preRotation != 270))
656 {
657 std::cout << "Invalid PreRotation '" << preRotationString
658 << "'; must be either 0, 90, 180 or 270" << std::endl;
659 exit(1);
660 }
661
662 gOptions.preRotation = preRotation;
663 }
664
HandleEGLConfigName(const char * configNameString)665 void HandleEGLConfigName(const char *configNameString)
666 {
667 gEGLConfigName = configNameString;
668 }
669
670 // The --deqp-case flag takes a case expression that is parsed into a --gtest_filter. It converts
671 // the "dEQP" style names (functional.thing.*) into "GoogleTest" style names (functional_thing_*).
672 // Currently it does not handle multiple tests and multiple filters in different arguments.
HandleFilterArg(const char * filterString,int * argc,int argIndex,char ** argv)673 void HandleFilterArg(const char *filterString, int *argc, int argIndex, char **argv)
674 {
675 std::string googleTestFilter = ReplaceDashesWithQuestionMark(filterString);
676
677 gFilterStringBuffer->resize(googleTestFilter.size() + 3 + strlen(kGTestFilter), 0);
678 std::fill(gFilterStringBuffer->begin(), gFilterStringBuffer->end(), 0);
679
680 int bytesWritten = snprintf(gFilterStringBuffer->data(), gFilterStringBuffer->size() - 1,
681 "%s*%s", kGTestFilter, googleTestFilter.c_str());
682 if (bytesWritten <= 0 || static_cast<size_t>(bytesWritten) >= gFilterStringBuffer->size() - 1)
683 {
684 std::cout << "Error parsing filter string: " << filterString;
685 exit(1);
686 }
687
688 argv[argIndex] = gFilterStringBuffer->data();
689 }
690
HandleLogImages(const char * logImagesString)691 void HandleLogImages(const char *logImagesString)
692 {
693 if (strcmp(logImagesString, "enable") == 0)
694 {
695 gLogImages = true;
696 }
697 else if (strcmp(logImagesString, "disable") == 0)
698 {
699 gLogImages = false;
700 }
701 else
702 {
703 std::cout << "Error parsing log images setting. Use enable/disable.";
704 exit(1);
705 }
706 }
707
GetTestModuleIndex()708 size_t GetTestModuleIndex()
709 {
710 #ifdef ANGLE_DEQP_EGL_TESTS
711 return 0;
712 #endif
713
714 #ifdef ANGLE_DEQP_GLES2_TESTS
715 return 1;
716 #endif
717
718 #ifdef ANGLE_DEQP_GLES3_TESTS
719 return 2;
720 #endif
721
722 #ifdef ANGLE_DEQP_GLES31_TESTS
723 return 3;
724 #endif
725
726 #ifdef ANGLE_DEQP_KHR_GLES2_TESTS
727 return 4;
728 #endif
729
730 #ifdef ANGLE_DEQP_KHR_GLES3_TESTS
731 return 5;
732 #endif
733
734 #ifdef ANGLE_DEQP_KHR_GLES31_TESTS
735 return 6;
736 #endif
737
738 #ifdef ANGLE_DEQP_KHR_GLES32_TESTS
739 return 7;
740 #endif
741
742 #ifdef ANGLE_DEQP_KHR_NOCTX_GLES2_TESTS
743 return 8;
744 #endif
745
746 #ifdef ANGLE_DEQP_KHR_NOCTX_GLES32_TESTS
747 return 9;
748 #endif
749
750 #ifdef ANGLE_DEQP_KHR_SINGLE_GLES32_TESTS
751 return 10;
752 #endif
753
754 #ifdef ANGLE_DEQP_GLES3_ROTATE90_TESTS
755 return 11;
756 #endif
757
758 #ifdef ANGLE_DEQP_GLES3_ROTATE180_TESTS
759 return 12;
760 #endif
761
762 #ifdef ANGLE_DEQP_GLES3_ROTATE270_TESTS
763 return 13;
764 #endif
765
766 #ifdef ANGLE_DEQP_GLES31_ROTATE90_TESTS
767 return 14;
768 #endif
769
770 #ifdef ANGLE_DEQP_GLES31_ROTATE180_TESTS
771 return 15;
772 #endif
773
774 #ifdef ANGLE_DEQP_GLES31_ROTATE270_TESTS
775 return 16;
776 #endif
777
778 #ifdef ANGLE_DEQP_GL_TESTS
779 return 17;
780 #endif
781 }
782
RegisterGLCTSTests()783 void RegisterGLCTSTests()
784 {
785 size_t testModuleIndex = GetTestModuleIndex();
786
787 const dEQPCaseList &caseList = dEQP::GetCaseList(testModuleIndex);
788
789 for (size_t caseIndex = 0; caseIndex < caseList.numCases(); ++caseIndex)
790 {
791 auto factory = [testModuleIndex, caseIndex]() {
792 return new dEQP(testModuleIndex, caseIndex);
793 };
794
795 std::string testCaseName = dEQP::GetTestCaseName(testModuleIndex, caseIndex);
796 size_t pos = testCaseName.find('.');
797 ASSERT(pos != std::string::npos);
798 std::string moduleName = testCaseName.substr(0, pos);
799 std::string testName = testCaseName.substr(pos + 1);
800 testing::RegisterTest(moduleName.c_str(), testName.c_str(), nullptr, nullptr, __FILE__,
801 __LINE__, factory);
802 }
803 }
804 } // anonymous namespace
805
806 // Called from main() to process command-line arguments.
RunGLCTSTests(int * argc,char ** argv)807 int RunGLCTSTests(int *argc, char **argv)
808 {
809 int argIndex = 0;
810 while (argIndex < *argc)
811 {
812 if (strncmp(argv[argIndex], kdEQPEGLString, strlen(kdEQPEGLString)) == 0)
813 {
814 HandleDisplayType(argv[argIndex] + strlen(kdEQPEGLString));
815 }
816 else if (strncmp(argv[argIndex], kANGLEEGLString, strlen(kANGLEEGLString)) == 0)
817 {
818 HandleDisplayType(argv[argIndex] + strlen(kANGLEEGLString));
819 }
820 else if (strncmp(argv[argIndex], kANGLEPreRotation, strlen(kANGLEPreRotation)) == 0)
821 {
822 HandlePreRotation(argv[argIndex] + strlen(kANGLEPreRotation));
823 }
824 else if (strncmp(argv[argIndex], gdEQPEGLConfigNameString,
825 strlen(gdEQPEGLConfigNameString)) == 0)
826 {
827 HandleEGLConfigName(argv[argIndex] + strlen(gdEQPEGLConfigNameString));
828 }
829 else if (strncmp(argv[argIndex], kdEQPCaseString, strlen(kdEQPCaseString)) == 0)
830 {
831 HandleFilterArg(argv[argIndex] + strlen(kdEQPCaseString), argc, argIndex, argv);
832 }
833 else if (strncmp(argv[argIndex], kGTestFilter, strlen(kGTestFilter)) == 0)
834 {
835 HandleFilterArg(argv[argIndex] + strlen(kGTestFilter), argc, argIndex, argv);
836 }
837 else if (strncmp(argv[argIndex], kVerboseString, strlen(kVerboseString)) == 0 ||
838 strcmp(argv[argIndex], "-v") == 0)
839 {
840 gVerbose = true;
841 }
842 else if (strncmp(argv[argIndex], gdEQPLogImagesString, strlen(gdEQPLogImagesString)) == 0)
843 {
844 HandleLogImages(argv[argIndex] + strlen(gdEQPLogImagesString));
845 }
846 else if (strncmp(argv[argIndex], kRenderDocString, strlen(kRenderDocString)) == 0)
847 {
848 gOptions.enableRenderDocCapture = true;
849 }
850 else if (strncmp(argv[argIndex], kNoRenderDocString, strlen(kNoRenderDocString)) == 0)
851 {
852 gOptions.enableRenderDocCapture = false;
853 }
854 else if (strncmp(argv[argIndex], kdEQPFlagsPrefix, strlen(kdEQPFlagsPrefix)) == 0)
855 {
856 gdEQPForwardFlags.push_back(argv[argIndex]);
857 }
858 argIndex++;
859 }
860
861 GPUTestConfig::API api = GetDefaultAPIInfo()->second;
862 if (gInitAPI)
863 {
864 api = gInitAPI->second;
865 }
866 if (gOptions.preRotation != 0 && api != GPUTestConfig::kAPIVulkan &&
867 api != GPUTestConfig::kAPISwiftShader)
868 {
869 std::cout << "PreRotation is only supported on Vulkan" << std::endl;
870 exit(1);
871 }
872
873 angle::TestSuite testSuite(argc, argv, RegisterGLCTSTests);
874 return testSuite.run();
875 }
876 } // namespace angle
877