// // Copyright 2019 The ANGLE Project Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. // #include "GPUTestExpectationsParser.h" #include #include #include #include "common/angleutils.h" #include "common/debug.h" #include "common/string_utils.h" namespace angle { namespace { enum LineParserStage { kLineParserBegin = 0, kLineParserBugID, kLineParserConfigs, kLineParserColon, kLineParserTestName, kLineParserEqual, kLineParserExpectations, }; enum Token { // os kConfigWinXP = 0, kConfigWinVista, kConfigWin7, kConfigWin8, kConfigWin10, kConfigWin, kConfigMacLeopard, kConfigMacSnowLeopard, kConfigMacLion, kConfigMacMountainLion, kConfigMacMavericks, kConfigMacYosemite, kConfigMacElCapitan, kConfigMacSierra, kConfigMacHighSierra, kConfigMacMojave, kConfigMac, kConfigIOS, kConfigLinux, kConfigChromeOS, kConfigAndroid, // gpu vendor kConfigNVIDIA, kConfigAMD, kConfigIntel, kConfigVMWare, // build type kConfigRelease, kConfigDebug, // ANGLE renderer kConfigD3D9, kConfigD3D11, kConfigGLDesktop, kConfigGLES, kConfigVulkan, kConfigSwiftShader, kConfigMetal, // Android devices kConfigNexus5X, kConfigPixel2, kConfigPixel4, // GPU devices kConfigNVIDIAQuadroP400, // PreRotation kConfigPreRotation, kConfigPreRotation90, kConfigPreRotation180, kConfigPreRotation270, // expectation kExpectationPass, kExpectationFail, kExpectationFlaky, kExpectationTimeout, kExpectationSkip, // separator kSeparatorColon, kSeparatorEqual, kNumberOfExactMatchTokens, // others kTokenComment, kTokenWord, kNumberOfTokens, }; enum ErrorType { kErrorFileIO = 0, kErrorIllegalEntry, kErrorInvalidEntry, kErrorEntryWithExpectationConflicts, kErrorEntryWithDisallowedExpectation, kErrorEntriesOverlap, kNumberOfErrors, }; struct TokenInfo { constexpr TokenInfo() : name(nullptr), condition(GPUTestConfig::kConditionNone), expectation(GPUTestExpectationsParser::kGpuTestPass) {} constexpr TokenInfo(const char *nameIn, GPUTestConfig::Condition conditionIn, GPUTestExpectationsParser::GPUTestExpectation expectationIn) : name(nameIn), condition(conditionIn), expectation(expectationIn) {} constexpr TokenInfo(const char *nameIn, GPUTestConfig::Condition conditionIn) : TokenInfo(nameIn, conditionIn, GPUTestExpectationsParser::kGpuTestPass) {} const char *name; GPUTestConfig::Condition condition; GPUTestExpectationsParser::GPUTestExpectation expectation; }; constexpr TokenInfo kTokenData[kNumberOfTokens] = { {"xp", GPUTestConfig::kConditionWinXP}, {"vista", GPUTestConfig::kConditionWinVista}, {"win7", GPUTestConfig::kConditionWin7}, {"win8", GPUTestConfig::kConditionWin8}, {"win10", GPUTestConfig::kConditionWin10}, {"win", GPUTestConfig::kConditionWin}, {"leopard", GPUTestConfig::kConditionMacLeopard}, {"snowleopard", GPUTestConfig::kConditionMacSnowLeopard}, {"lion", GPUTestConfig::kConditionMacLion}, {"mountainlion", GPUTestConfig::kConditionMacMountainLion}, {"mavericks", GPUTestConfig::kConditionMacMavericks}, {"yosemite", GPUTestConfig::kConditionMacYosemite}, {"elcapitan", GPUTestConfig::kConditionMacElCapitan}, {"sierra", GPUTestConfig::kConditionMacSierra}, {"highsierra", GPUTestConfig::kConditionMacHighSierra}, {"mojave", GPUTestConfig::kConditionMacMojave}, {"mac", GPUTestConfig::kConditionMac}, {"ios", GPUTestConfig::kConditionIOS}, {"linux", GPUTestConfig::kConditionLinux}, {"chromeos", GPUTestConfig::kConditionNone}, // https://anglebug.com/3363 CrOS not supported {"android", GPUTestConfig::kConditionAndroid}, {"nvidia", GPUTestConfig::kConditionNVIDIA}, {"amd", GPUTestConfig::kConditionAMD}, {"intel", GPUTestConfig::kConditionIntel}, {"vmware", GPUTestConfig::kConditionVMWare}, {"release", GPUTestConfig::kConditionRelease}, {"debug", GPUTestConfig::kConditionDebug}, {"d3d9", GPUTestConfig::kConditionD3D9}, {"d3d11", GPUTestConfig::kConditionD3D11}, {"opengl", GPUTestConfig::kConditionGLDesktop}, {"gles", GPUTestConfig::kConditionGLES}, {"vulkan", GPUTestConfig::kConditionVulkan}, {"swiftshader", GPUTestConfig::kConditionSwiftShader}, {"metal", GPUTestConfig::kConditionMetal}, {"nexus5x", GPUTestConfig::kConditionNexus5X}, {"pixel2orxl", GPUTestConfig::kConditionPixel2OrXL}, {"pixel4orxl", GPUTestConfig::kConditionPixel4OrXL}, {"quadrop400", GPUTestConfig::kConditionNVIDIAQuadroP400}, {"prerotation", GPUTestConfig::kConditionPreRotation}, {"prerotation90", GPUTestConfig::kConditionPreRotation90}, {"prerotation180", GPUTestConfig::kConditionPreRotation180}, {"prerotation270", GPUTestConfig::kConditionPreRotation270}, {"pass", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestPass}, {"fail", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestFail}, {"flaky", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestFlaky}, {"timeout", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestTimeout}, {"skip", GPUTestConfig::kConditionNone, GPUTestExpectationsParser::kGpuTestSkip}, {":", GPUTestConfig::kConditionNone}, // kSeparatorColon {"=", GPUTestConfig::kConditionNone}, // kSeparatorEqual {}, // kNumberOfExactMatchTokens {}, // kTokenComment {}, // kTokenWord }; const char *kErrorMessage[kNumberOfErrors] = { "file IO failed", "entry with wrong format", "entry invalid, likely unimplemented modifiers", "entry with expectation modifier conflicts", "entry with unsupported expectation", "two entries' configs overlap", }; inline bool StartsWithASCII(const std::string &str, const std::string &search, bool caseSensitive) { ASSERT(!caseSensitive); return str.compare(0, search.length(), search) == 0; } template inline Char ToLowerASCII(Char c) { return (c >= 'A' && c <= 'Z') ? (c + ('a' - 'A')) : c; } template inline bool DoLowerCaseEqualsASCII(Iter a_begin, Iter a_end, const char *b) { for (Iter it = a_begin; it != a_end; ++it, ++b) { if (!*b || ToLowerASCII(*it) != *b) return false; } return *b == 0; } inline bool LowerCaseEqualsASCII(const std::string &a, const char *b) { return DoLowerCaseEqualsASCII(a.begin(), a.end(), b); } inline Token ParseToken(const std::string &word) { if (StartsWithASCII(word, "//", false)) return kTokenComment; for (int32_t i = 0; i < kNumberOfExactMatchTokens; ++i) { if (LowerCaseEqualsASCII(word, kTokenData[i].name)) return static_cast(i); } return kTokenWord; } bool ConditionArrayIsSubset(const GPUTestConfig::ConditionArray &subset, const GPUTestConfig::ConditionArray &superset) { for (size_t subsetCondition : subset) { bool foundCondition = false; for (size_t supersetCondition : superset) { if (subsetCondition == supersetCondition) { foundCondition = true; break; } } if (!foundCondition) { return false; } } return true; } // If one array is completely contained within the other, then we say the conditions overlap. bool ConditionsOverlap(const GPUTestConfig::ConditionArray &conditionsI, const GPUTestConfig::ConditionArray &conditionsJ) { return ConditionArrayIsSubset(conditionsI, conditionsJ) || ConditionArrayIsSubset(conditionsJ, conditionsI); } } // anonymous namespace const char *GetConditionName(uint32_t condition) { if (condition == GPUTestConfig::kConditionNone) { return nullptr; } for (const TokenInfo &info : kTokenData) { if (info.condition == condition) { // kConditionNone is used to tag tokens that aren't conditions, but this case has been // handled above. ASSERT(info.condition != GPUTestConfig::kConditionNone); return info.name; } } return nullptr; } GPUTestExpectationsParser::GPUTestExpectationsParser() : mExpectationsAllowMask( GPUTestExpectationsParser::kGpuTestPass | GPUTestExpectationsParser::kGpuTestFail | GPUTestExpectationsParser::kGpuTestFlaky | GPUTestExpectationsParser::kGpuTestTimeout | GPUTestExpectationsParser::kGpuTestSkip) { // Some initial checks. ASSERT((static_cast(kNumberOfTokens)) == (sizeof(kTokenData) / sizeof(kTokenData[0]))); ASSERT((static_cast(kNumberOfErrors)) == (sizeof(kErrorMessage) / sizeof(kErrorMessage[0]))); } GPUTestExpectationsParser::~GPUTestExpectationsParser() = default; bool GPUTestExpectationsParser::loadTestExpectationsImpl(const GPUTestConfig *config, const std::string &data) { mEntries.clear(); mErrorMessages.clear(); std::vector lines = SplitString(data, "\n", TRIM_WHITESPACE, SPLIT_WANT_ALL); bool rt = true; for (size_t i = 0; i < lines.size(); ++i) { if (!parseLine(config, lines[i], i + 1)) rt = false; } if (detectConflictsBetweenEntries()) { mEntries.clear(); rt = false; } return rt; } bool GPUTestExpectationsParser::loadTestExpectations(const GPUTestConfig &config, const std::string &data) { return loadTestExpectationsImpl(&config, data); } bool GPUTestExpectationsParser::loadAllTestExpectations(const std::string &data) { return loadTestExpectationsImpl(nullptr, data); } bool GPUTestExpectationsParser::loadTestExpectationsFromFileImpl(const GPUTestConfig *config, const std::string &path) { mEntries.clear(); mErrorMessages.clear(); std::string data; if (!ReadFileToString(path, &data)) { mErrorMessages.push_back(kErrorMessage[kErrorFileIO]); return false; } return loadTestExpectationsImpl(config, data); } bool GPUTestExpectationsParser::loadTestExpectationsFromFile(const GPUTestConfig &config, const std::string &path) { return loadTestExpectationsFromFileImpl(&config, path); } bool GPUTestExpectationsParser::loadAllTestExpectationsFromFile(const std::string &path) { return loadTestExpectationsFromFileImpl(nullptr, path); } int32_t GPUTestExpectationsParser::getTestExpectationImpl(const GPUTestConfig *config, const std::string &testName) { size_t maxExpectationLen = 0; GPUTestExpectationEntry *foundEntry = nullptr; for (GPUTestExpectationEntry &entry : mEntries) { if (NamesMatchWithWildcard(entry.testName.c_str(), testName.c_str())) { size_t expectationLen = entry.testName.length(); // Filter by condition first. bool satisfiesConditions = true; if (config) { for (size_t condition : entry.conditions) { if (!config->getConditions()[condition]) { satisfiesConditions = false; break; } } } // The longest/most specific matching expectation overrides any others. if (satisfiesConditions && expectationLen > maxExpectationLen) { maxExpectationLen = expectationLen; foundEntry = &entry; } } } if (foundEntry != nullptr) { foundEntry->used = true; return foundEntry->testExpectation; } return kGpuTestPass; } int32_t GPUTestExpectationsParser::getTestExpectation(const std::string &testName) { return getTestExpectationImpl(nullptr, testName); } int32_t GPUTestExpectationsParser::getTestExpectationWithConfig(const GPUTestConfig &config, const std::string &testName) { return getTestExpectationImpl(&config, testName); } const std::vector &GPUTestExpectationsParser::getErrorMessages() const { return mErrorMessages; } std::vector GPUTestExpectationsParser::getUnusedExpectationsMessages() const { std::vector messages; std::vector unusedExpectations = getUnusedExpectations(); for (size_t i = 0; i < unusedExpectations.size(); ++i) { std::string message = "Line " + ToString(unusedExpectations[i].lineNumber) + ": expectation was unused."; messages.push_back(message); } return messages; } bool GPUTestExpectationsParser::parseLine(const GPUTestConfig *config, const std::string &lineData, size_t lineNumber) { std::vector tokens = SplitString(lineData, kWhitespaceASCII, KEEP_WHITESPACE, SPLIT_WANT_NONEMPTY); int32_t stage = kLineParserBegin; GPUTestExpectationEntry entry; entry.lineNumber = lineNumber; entry.used = false; bool skipLine = false; for (size_t i = 0; i < tokens.size() && !skipLine; ++i) { Token token = ParseToken(tokens[i]); switch (token) { case kTokenComment: skipLine = true; break; case kConfigWinXP: case kConfigWinVista: case kConfigWin7: case kConfigWin8: case kConfigWin10: case kConfigWin: case kConfigMacLeopard: case kConfigMacSnowLeopard: case kConfigMacLion: case kConfigMacMountainLion: case kConfigMacMavericks: case kConfigMacYosemite: case kConfigMacElCapitan: case kConfigMacSierra: case kConfigMacHighSierra: case kConfigMacMojave: case kConfigMac: case kConfigIOS: case kConfigLinux: case kConfigChromeOS: case kConfigAndroid: case kConfigNVIDIA: case kConfigAMD: case kConfigIntel: case kConfigVMWare: case kConfigRelease: case kConfigDebug: case kConfigD3D9: case kConfigD3D11: case kConfigGLDesktop: case kConfigGLES: case kConfigVulkan: case kConfigSwiftShader: case kConfigMetal: case kConfigNexus5X: case kConfigPixel2: case kConfigPixel4: case kConfigNVIDIAQuadroP400: case kConfigPreRotation: case kConfigPreRotation90: case kConfigPreRotation180: case kConfigPreRotation270: // MODIFIERS, check each condition and add accordingly. if (stage != kLineParserConfigs && stage != kLineParserBugID) { pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber); return false; } { bool err = false; if (config) { if (!checkTokenCondition(*config, err, token, lineNumber)) { skipLine = true; // Move to the next line without adding this one. } } else { // Store the conditions for later comparison if we don't have a config. entry.conditions[kTokenData[token].condition] = true; } if (err) { return false; } } if (stage == kLineParserBugID) { stage++; } break; case kSeparatorColon: // : // If there are no modifiers, move straight to separator colon if (stage == kLineParserBugID) { stage++; } if (stage != kLineParserConfigs) { pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber); return false; } stage++; break; case kSeparatorEqual: // = if (stage != kLineParserTestName) { pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber); return false; } stage++; break; case kTokenWord: // BUG_ID or TEST_NAME if (stage == kLineParserBegin) { // Bug ID is not used for anything; ignore it. } else if (stage == kLineParserColon) { entry.testName = tokens[i]; } else { pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber); return false; } stage++; break; case kExpectationPass: case kExpectationFail: case kExpectationFlaky: case kExpectationTimeout: case kExpectationSkip: // TEST_EXPECTATIONS if (stage != kLineParserEqual && stage != kLineParserExpectations) { pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber); return false; } if (entry.testExpectation != 0) { pushErrorMessage(kErrorMessage[kErrorEntryWithExpectationConflicts], lineNumber); return false; } if ((mExpectationsAllowMask & kTokenData[token].expectation) == 0) { pushErrorMessage(kErrorMessage[kErrorEntryWithDisallowedExpectation], lineNumber); return false; } entry.testExpectation = kTokenData[token].expectation; if (stage == kLineParserEqual) stage++; break; default: ASSERT(false); break; } } if (stage == kLineParserBegin || skipLine) { // The whole line is empty or all comments, or has been skipped to to a condition token. return true; } if (stage == kLineParserExpectations) { mEntries.push_back(entry); return true; } pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber); return false; } bool GPUTestExpectationsParser::checkTokenCondition(const GPUTestConfig &config, bool &err, int32_t token, size_t lineNumber) { if (token >= kNumberOfTokens) { pushErrorMessage(kErrorMessage[kErrorIllegalEntry], lineNumber); err = true; return false; } if (kTokenData[token].condition == GPUTestConfig::kConditionNone || kTokenData[token].condition >= GPUTestConfig::kNumberOfConditions) { pushErrorMessage(kErrorMessage[kErrorInvalidEntry], lineNumber); // error on any unsupported conditions err = true; return false; } err = false; return config.getConditions()[kTokenData[token].condition]; } bool GPUTestExpectationsParser::detectConflictsBetweenEntries() { bool rt = false; for (size_t i = 0; i < mEntries.size(); ++i) { for (size_t j = i + 1; j < mEntries.size(); ++j) { const GPUTestExpectationEntry &entryI = mEntries[i]; const GPUTestExpectationEntry &entryJ = mEntries[j]; if (entryI.testName == entryJ.testName && ConditionsOverlap(entryI.conditions, entryJ.conditions)) { pushErrorMessage(kErrorMessage[kErrorEntriesOverlap], entryI.lineNumber, entryJ.lineNumber); rt = true; } } } return rt; } std::vector GPUTestExpectationsParser::getUnusedExpectations() const { std::vector unusedExpectations; for (size_t i = 0; i < mEntries.size(); ++i) { if (!mEntries[i].used) { unusedExpectations.push_back(mEntries[i]); } } return unusedExpectations; } void GPUTestExpectationsParser::pushErrorMessage(const std::string &message, size_t lineNumber) { mErrorMessages.push_back("Line " + ToString(lineNumber) + " : " + message.c_str()); } void GPUTestExpectationsParser::pushErrorMessage(const std::string &message, size_t entry1LineNumber, size_t entry2LineNumber) { mErrorMessages.push_back("Line " + ToString(entry1LineNumber) + " and " + ToString(entry2LineNumber) + " : " + message.c_str()); } GPUTestExpectationsParser::GPUTestExpectationEntry::GPUTestExpectationEntry() : testExpectation(0), lineNumber(0) {} } // namespace angle