1 //
2 // Copyright 2019 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6
7 #include "GPUTestExpectationsParser.h"
8
9 #include <stddef.h>
10 #include <stdint.h>
11 #include <string.h>
12
13 #include "common/angleutils.h"
14 #include "common/debug.h"
15 #include "common/string_utils.h"
16
17 #include "GPUTestConfig.h"
18
19 namespace angle
20 {
21
22 namespace
23 {
24
25 enum LineParserStage
26 {
27 kLineParserBegin = 0,
28 kLineParserBugID,
29 kLineParserConfigs,
30 kLineParserColon,
31 kLineParserTestName,
32 kLineParserEqual,
33 kLineParserExpectations,
34 };
35
36 enum Token
37 {
38 // os
39 kConfigWinXP = 0,
40 kConfigWinVista,
41 kConfigWin7,
42 kConfigWin8,
43 kConfigWin10,
44 kConfigWin,
45 kConfigMacLeopard,
46 kConfigMacSnowLeopard,
47 kConfigMacLion,
48 kConfigMacMountainLion,
49 kConfigMacMavericks,
50 kConfigMacYosemite,
51 kConfigMacElCapitan,
52 kConfigMacSierra,
53 kConfigMacHighSierra,
54 kConfigMacMojave,
55 kConfigMac,
56 kConfigLinux,
57 kConfigChromeOS,
58 kConfigAndroid,
59 // gpu vendor
60 kConfigNVIDIA,
61 kConfigAMD,
62 kConfigIntel,
63 kConfigVMWare,
64 // build type
65 kConfigRelease,
66 kConfigDebug,
67 // ANGLE renderer
68 kConfigD3D9,
69 kConfigD3D11,
70 kConfigGLDesktop,
71 kConfigGLES,
72 kConfigVulkan,
73 kConfigSwiftShader,
74 kConfigMetal,
75 // Android devices
76 kConfigNexus5X,
77 kConfigPixel2,
78 // GPU devices
79 kConfigNVIDIAQuadroP400,
80 // expectation
81 kExpectationPass,
82 kExpectationFail,
83 kExpectationFlaky,
84 kExpectationTimeout,
85 kExpectationSkip,
86 // separator
87 kSeparatorColon,
88 kSeparatorEqual,
89
90 kNumberOfExactMatchTokens,
91
92 // others
93 kTokenComment,
94 kTokenWord,
95
96 kNumberOfTokens,
97 };
98
99 enum ErrorType
100 {
101 kErrorFileIO = 0,
102 kErrorIllegalEntry,
103 kErrorInvalidEntry,
104 kErrorEntryWithExpectationConflicts,
105 kErrorEntriesOverlap,
106
107 kNumberOfErrors,
108 };
109
110 struct TokenInfo
111 {
TokenInfoangle::__anon8cee1eeb0111::TokenInfo112 constexpr TokenInfo()
113 : name(nullptr),
114 condition(GPUTestConfig::kConditionNone),
115 expectation(GPUTestExpectationsParser::kGpuTestPass)
116 {}
117
TokenInfoangle::__anon8cee1eeb0111::TokenInfo118 constexpr TokenInfo(const char *nameIn,
119 GPUTestConfig::Condition conditionIn,
120 GPUTestExpectationsParser::GPUTestExpectation expectationIn)
121 : name(nameIn), condition(conditionIn), expectation(expectationIn)
122 {}
123
TokenInfoangle::__anon8cee1eeb0111::TokenInfo124 constexpr TokenInfo(const char *nameIn, GPUTestConfig::Condition conditionIn)
125 : TokenInfo(nameIn, conditionIn, GPUTestExpectationsParser::kGpuTestPass)
126 {}
127
128 const char *name;
129 GPUTestConfig::Condition condition;
130 GPUTestExpectationsParser::GPUTestExpectation expectation;
131 };
132
133 constexpr TokenInfo kTokenData[kNumberOfTokens] = {
134 {"xp", GPUTestConfig::kConditionWinXP},
135 {"vista", GPUTestConfig::kConditionWinVista},
136 {"win7", GPUTestConfig::kConditionWin7},
137 {"win8", GPUTestConfig::kConditionWin8},
138 {"win10", GPUTestConfig::kConditionWin10},
139 {"win", GPUTestConfig::kConditionWin},
140 {"leopard", GPUTestConfig::kConditionMacLeopard},
141 {"snowleopard", GPUTestConfig::kConditionMacSnowLeopard},
142 {"lion", GPUTestConfig::kConditionMacLion},
143 {"mountainlion", GPUTestConfig::kConditionMacMountainLion},
144 {"mavericks", GPUTestConfig::kConditionMacMavericks},
145 {"yosemite", GPUTestConfig::kConditionMacYosemite},
146 {"elcapitan", GPUTestConfig::kConditionMacElCapitan},
147 {"sierra", GPUTestConfig::kConditionMacSierra},
148 {"highsierra", GPUTestConfig::kConditionMacHighSierra},
149 {"mojave", GPUTestConfig::kConditionMacMojave},
150 {"mac", GPUTestConfig::kConditionMac},
151 {"linux", GPUTestConfig::kConditionLinux},
152 {"chromeos", GPUTestConfig::kConditionNone}, // https://anglebug.com/3363 CrOS not supported
153 {"android", GPUTestConfig::kConditionAndroid},
154 {"nvidia", GPUTestConfig::kConditionNVIDIA},
155 {"amd", GPUTestConfig::kConditionAMD},
156 {"intel", GPUTestConfig::kConditionIntel},
157 {"vmware", GPUTestConfig::kConditionVMWare},
158 {"release", GPUTestConfig::kConditionRelease},
159 {"debug", GPUTestConfig::kConditionDebug},
160 {"d3d9", GPUTestConfig::kConditionD3D9},
161 {"d3d11", GPUTestConfig::kConditionD3D11},
162 {"opengl", GPUTestConfig::kConditionGLDesktop},
163 {"gles", GPUTestConfig::kConditionGLES},
164 {"vulkan", GPUTestConfig::kConditionVulkan},
165 {"swiftshader", GPUTestConfig::kConditionSwiftShader},
166 {"metal", GPUTestConfig::kConditionMetal},
167 {"nexus5x", GPUTestConfig::kConditionNexus5X},
168 {"pixel2orxl", GPUTestConfig::kConditionPixel2OrXL},
169 {"quadrop400", GPUTestConfig::kConditionNVIDIAQuadroP400},
170 {"pass", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestPass},
171 {"fail", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestFail},
172 {"flaky", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestFlaky},
173 {"timeout", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestTimeout},
174 {"skip", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestSkip},
175 {":", GPUTestConfig::kConditionNone}, // kSeparatorColon
176 {"=", GPUTestConfig::kConditionNone}, // kSeparatorEqual
177 {}, // kNumberOfExactMatchTokens
178 {}, // kTokenComment
179 {}, // kTokenWord
180 };
181
182 const char *kErrorMessage[kNumberOfErrors] = {
183 "file IO failed",
184 "entry with wrong format",
185 "entry invalid, likely unimplemented modifiers",
186 "entry with expectation modifier conflicts",
187 "two entries' configs overlap",
188 };
189
StartsWithASCII(const std::string & str,const std::string & search,bool caseSensitive)190 inline bool StartsWithASCII(const std::string &str, const std::string &search, bool caseSensitive)
191 {
192 ASSERT(!caseSensitive);
193 return str.compare(0, search.length(), search) == 0;
194 }
195
196 template <class Char>
ToLowerASCII(Char c)197 inline Char ToLowerASCII(Char c)
198 {
199 return (c >= 'A' && c <= 'Z') ? (c + ('a' - 'A')) : c;
200 }
201
202 template <typename Iter>
DoLowerCaseEqualsASCII(Iter a_begin,Iter a_end,const char * b)203 inline bool DoLowerCaseEqualsASCII(Iter a_begin, Iter a_end, const char *b)
204 {
205 for (Iter it = a_begin; it != a_end; ++it, ++b)
206 {
207 if (!*b || ToLowerASCII(*it) != *b)
208 return false;
209 }
210 return *b == 0;
211 }
212
LowerCaseEqualsASCII(const std::string & a,const char * b)213 inline bool LowerCaseEqualsASCII(const std::string &a, const char *b)
214 {
215 return DoLowerCaseEqualsASCII(a.begin(), a.end(), b);
216 }
217
ParseToken(const std::string & word)218 inline Token ParseToken(const std::string &word)
219 {
220 if (StartsWithASCII(word, "//", false))
221 return kTokenComment;
222
223 for (int32_t i = 0; i < kNumberOfExactMatchTokens; ++i)
224 {
225 if (LowerCaseEqualsASCII(word, kTokenData[i].name))
226 return static_cast<Token>(i);
227 }
228 return kTokenWord;
229 }
230
231 // reference name can have *.
NamesMatching(const char * ref,const char * testName)232 inline bool NamesMatching(const char *ref, const char *testName)
233 {
234 // Find the first * in ref.
235 const char *firstWildcard = strchr(ref, '*');
236
237 // If there are no wildcards, match the strings precisely.
238 if (firstWildcard == nullptr)
239 {
240 return strcmp(ref, testName) == 0;
241 }
242
243 // Otherwise, match up to the wildcard first.
244 size_t preWildcardLen = firstWildcard - ref;
245 if (strncmp(ref, testName, preWildcardLen) != 0)
246 {
247 return false;
248 }
249
250 const char *postWildcardRef = ref + preWildcardLen + 1;
251
252 // As a small optimization, if the wildcard is the last character in ref, accept the match
253 // already.
254 if (postWildcardRef[0] == '\0')
255 {
256 return true;
257 }
258
259 // Try to match the wildcard with a number of characters.
260 for (size_t matchSize = 0; testName[matchSize] != '\0'; ++matchSize)
261 {
262 if (NamesMatching(postWildcardRef, testName + matchSize))
263 {
264 return true;
265 }
266 }
267
268 return false;
269 }
270
271 } // anonymous namespace
272
GetConditionName(uint32_t condition)273 const char *GetConditionName(uint32_t condition)
274 {
275 if (condition == GPUTestConfig::kConditionNone)
276 {
277 return nullptr;
278 }
279
280 for (const TokenInfo &info : kTokenData)
281 {
282 if (info.condition == condition)
283 {
284 // kConditionNone is used to tag tokens that aren't conditions, but this case has been
285 // handled above.
286 ASSERT(info.condition != GPUTestConfig::kConditionNone);
287 return info.name;
288 }
289 }
290
291 return nullptr;
292 }
293
GPUTestExpectationsParser()294 GPUTestExpectationsParser::GPUTestExpectationsParser()
295 {
296 // Some sanity check.
297 ASSERT((static_cast<unsigned int>(kNumberOfTokens)) ==
298 (sizeof(kTokenData) / sizeof(kTokenData[0])));
299 ASSERT((static_cast<unsigned int>(kNumberOfErrors)) ==
300 (sizeof(kErrorMessage) / sizeof(kErrorMessage[0])));
301 }
302
303 GPUTestExpectationsParser::~GPUTestExpectationsParser() = default;
304
loadTestExpectations(const GPUTestConfig & config,const std::string & data)305 bool GPUTestExpectationsParser::loadTestExpectations(const GPUTestConfig &config,
306 const std::string &data)
307 {
308 mEntries.clear();
309 mErrorMessages.clear();
310
311 std::vector<std::string> lines = SplitString(data, "\n", TRIM_WHITESPACE, SPLIT_WANT_ALL);
312 bool rt = true;
313 for (size_t i = 0; i < lines.size(); ++i)
314 {
315 if (!parseLine(config, lines[i], i + 1))
316 rt = false;
317 }
318 if (detectConflictsBetweenEntries())
319 {
320 mEntries.clear();
321 rt = false;
322 }
323
324 return rt;
325 }
326
loadTestExpectationsFromFile(const GPUTestConfig & config,const std::string & path)327 bool GPUTestExpectationsParser::loadTestExpectationsFromFile(const GPUTestConfig &config,
328 const std::string &path)
329 {
330 mEntries.clear();
331 mErrorMessages.clear();
332
333 std::string data;
334 if (!ReadFileToString(path, &data))
335 {
336 mErrorMessages.push_back(kErrorMessage[kErrorFileIO]);
337 return false;
338 }
339 return loadTestExpectations(config, data);
340 }
341
getTestExpectation(const std::string & testName)342 int32_t GPUTestExpectationsParser::getTestExpectation(const std::string &testName)
343 {
344 size_t maxExpectationLen = 0;
345 GPUTestExpectationEntry *foundEntry = nullptr;
346 for (size_t i = 0; i < mEntries.size(); ++i)
347 {
348 if (NamesMatching(mEntries[i].testName.c_str(), testName.c_str()))
349 {
350 size_t expectationLen = mEntries[i].testName.length();
351 // The longest/most specific matching expectation overrides any others.
352 if (expectationLen > maxExpectationLen)
353 {
354 maxExpectationLen = expectationLen;
355 foundEntry = &mEntries[i];
356 }
357 }
358 }
359 if (foundEntry != nullptr)
360 {
361 foundEntry->used = true;
362 return foundEntry->testExpectation;
363 }
364 return kGpuTestPass;
365 }
366
getErrorMessages() const367 const std::vector<std::string> &GPUTestExpectationsParser::getErrorMessages() const
368 {
369 return mErrorMessages;
370 }
371
getUnusedExpectationsMessages() const372 std::vector<std::string> GPUTestExpectationsParser::getUnusedExpectationsMessages() const
373 {
374 std::vector<std::string> messages;
375 std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry> unusedExpectations =
376 getUnusedExpectations();
377 for (size_t i = 0; i < unusedExpectations.size(); ++i)
378 {
379 std::string message =
380 "Line " + ToString(unusedExpectations[i].lineNumber) + ": expectation was unused.";
381 messages.push_back(message);
382 }
383 return messages;
384 }
385
parseLine(const GPUTestConfig & config,const std::string & lineData,size_t lineNumber)386 bool GPUTestExpectationsParser::parseLine(const GPUTestConfig &config,
387 const std::string &lineData,
388 size_t lineNumber)
389 {
390 std::vector<std::string> tokens =
391 SplitString(lineData, kWhitespaceASCII, KEEP_WHITESPACE, SPLIT_WANT_NONEMPTY);
392 int32_t stage = kLineParserBegin;
393 GPUTestExpectationEntry entry;
394 entry.lineNumber = lineNumber;
395 entry.used = false;
396 bool skipLine = false;
397 for (size_t i = 0; i < tokens.size() && !skipLine; ++i)
398 {
399 Token token = ParseToken(tokens[i]);
400 switch (token)
401 {
402 case kTokenComment:
403 skipLine = true;
404 break;
405 case kConfigWinXP:
406 case kConfigWinVista:
407 case kConfigWin7:
408 case kConfigWin8:
409 case kConfigWin10:
410 case kConfigWin:
411 case kConfigMacLeopard:
412 case kConfigMacSnowLeopard:
413 case kConfigMacLion:
414 case kConfigMacMountainLion:
415 case kConfigMacMavericks:
416 case kConfigMacYosemite:
417 case kConfigMacElCapitan:
418 case kConfigMacSierra:
419 case kConfigMacHighSierra:
420 case kConfigMacMojave:
421 case kConfigMac:
422 case kConfigLinux:
423 case kConfigChromeOS:
424 case kConfigAndroid:
425 case kConfigNVIDIA:
426 case kConfigAMD:
427 case kConfigIntel:
428 case kConfigVMWare:
429 case kConfigRelease:
430 case kConfigDebug:
431 case kConfigD3D9:
432 case kConfigD3D11:
433 case kConfigGLDesktop:
434 case kConfigGLES:
435 case kConfigVulkan:
436 case kConfigSwiftShader:
437 case kConfigMetal:
438 case kConfigNexus5X:
439 case kConfigPixel2:
440 case kConfigNVIDIAQuadroP400:
441 // MODIFIERS, check each condition and add accordingly.
442 if (stage != kLineParserConfigs && stage != kLineParserBugID)
443 {
444 pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
445 return false;
446 }
447 {
448 bool err = false;
449 if (!checkTokenCondition(config, err, token, lineNumber))
450 {
451 skipLine = true; // Move to the next line without adding this one.
452 }
453 if (err)
454 {
455 return false;
456 }
457 }
458 if (stage == kLineParserBugID)
459 {
460 stage++;
461 }
462 break;
463 case kSeparatorColon:
464 // :
465 // If there are no modifiers, move straight to separator colon
466 if (stage == kLineParserBugID)
467 {
468 stage++;
469 }
470 if (stage != kLineParserConfigs)
471 {
472 pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
473 return false;
474 }
475 stage++;
476 break;
477 case kSeparatorEqual:
478 // =
479 if (stage != kLineParserTestName)
480 {
481 pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
482 return false;
483 }
484 stage++;
485 break;
486 case kTokenWord:
487 // BUG_ID or TEST_NAME
488 if (stage == kLineParserBegin)
489 {
490 // Bug ID is not used for anything; ignore it.
491 }
492 else if (stage == kLineParserColon)
493 {
494 entry.testName = tokens[i];
495 }
496 else
497 {
498 pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
499 return false;
500 }
501 stage++;
502 break;
503 case kExpectationPass:
504 case kExpectationFail:
505 case kExpectationFlaky:
506 case kExpectationTimeout:
507 case kExpectationSkip:
508 // TEST_EXPECTATIONS
509 if (stage != kLineParserEqual && stage != kLineParserExpectations)
510 {
511 pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
512 return false;
513 }
514 if (entry.testExpectation != 0)
515 {
516 pushErrorMessage(kErrorMessage[kErrorEntryWithExpectationConflicts],
517 lineNumber);
518 return false;
519 }
520 entry.testExpectation = kTokenData[token].expectation;
521 if (stage == kLineParserEqual)
522 stage++;
523 break;
524 default:
525 ASSERT(false);
526 break;
527 }
528 }
529 if (stage == kLineParserBegin || skipLine)
530 {
531 // The whole line is empty or all comments, or has been skipped to to a condition token.
532 return true;
533 }
534 if (stage == kLineParserExpectations)
535 {
536 mEntries.push_back(entry);
537 return true;
538 }
539 pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
540 return false;
541 }
542
checkTokenCondition(const GPUTestConfig & config,bool & err,int32_t token,size_t lineNumber)543 bool GPUTestExpectationsParser::checkTokenCondition(const GPUTestConfig &config,
544 bool &err,
545 int32_t token,
546 size_t lineNumber)
547 {
548 if (token >= kNumberOfTokens)
549 {
550 pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber);
551 err = true;
552 return false;
553 }
554
555 if (kTokenData[token].condition == GPUTestConfig::kConditionNone ||
556 kTokenData[token].condition >= GPUTestConfig::kNumberOfConditions)
557 {
558 pushErrorMessage(kErrorMessage[kErrorInvalidEntry], lineNumber);
559 // error on any unsupported conditions
560 err = true;
561 return false;
562 }
563 err = false;
564 return config.getConditions()[kTokenData[token].condition];
565 }
566
detectConflictsBetweenEntries()567 bool GPUTestExpectationsParser::detectConflictsBetweenEntries()
568 {
569 bool rt = false;
570 for (size_t i = 0; i < mEntries.size(); ++i)
571 {
572 for (size_t j = i + 1; j < mEntries.size(); ++j)
573 {
574 if (mEntries[i].testName == mEntries[j].testName)
575 {
576 pushErrorMessage(kErrorMessage[kErrorEntriesOverlap], mEntries[i].lineNumber,
577 mEntries[j].lineNumber);
578 rt = true;
579 }
580 }
581 }
582 return rt;
583 }
584
585 std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry>
getUnusedExpectations() const586 GPUTestExpectationsParser::getUnusedExpectations() const
587 {
588 std::vector<GPUTestExpectationsParser::GPUTestExpectationEntry> unusedExpectations;
589 for (size_t i = 0; i < mEntries.size(); ++i)
590 {
591 if (!mEntries[i].used)
592 {
593 unusedExpectations.push_back(mEntries[i]);
594 }
595 }
596 return unusedExpectations;
597 }
598
pushErrorMessage(const std::string & message,size_t lineNumber)599 void GPUTestExpectationsParser::pushErrorMessage(const std::string &message, size_t lineNumber)
600 {
601 mErrorMessages.push_back("Line " + ToString(lineNumber) + " : " + message.c_str());
602 }
603
pushErrorMessage(const std::string & message,size_t entry1LineNumber,size_t entry2LineNumber)604 void GPUTestExpectationsParser::pushErrorMessage(const std::string &message,
605 size_t entry1LineNumber,
606 size_t entry2LineNumber)
607 {
608 mErrorMessages.push_back("Line " + ToString(entry1LineNumber) + " and " +
609 ToString(entry2LineNumber) + " : " + message.c_str());
610 }
611
GPUTestExpectationEntry()612 GPUTestExpectationsParser::GPUTestExpectationEntry::GPUTestExpectationEntry()
613 : testExpectation(0), lineNumber(0)
614 {}
615
616 } // namespace angle
617