1 // 2 // Copyright 2019 The ANGLE Project Authors. All rights reserved. 3 // Use of this source code is governed by a BSD-style license that can be 4 // found in the LICENSE file. 5 // 6 // TestSuite: 7 // Basic implementation of a test harness in ANGLE. 8 9 #ifndef ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_ 10 #define ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_ 11 12 #include <map> 13 #include <memory> 14 #include <mutex> 15 #include <queue> 16 #include <string> 17 #include <thread> 18 19 #include "HistogramWriter.h" 20 #include "tests/test_expectations/GPUTestExpectationsParser.h" 21 #include "util/test_utils.h" 22 23 namespace angle 24 { 25 struct TestIdentifier 26 { 27 TestIdentifier(); 28 TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn); 29 TestIdentifier(const TestIdentifier &other); 30 ~TestIdentifier(); 31 32 TestIdentifier &operator=(const TestIdentifier &other); 33 34 static bool ParseFromString(const std::string &str, TestIdentifier *idOut); 35 validTestIdentifier36 bool valid() const { return !testName.empty(); } 37 void sprintfName(char *outBuffer) const; 38 39 std::string testSuiteName; 40 std::string testName; 41 }; 42 43 inline bool operator<(const TestIdentifier &a, const TestIdentifier &b) 44 { 45 return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName); 46 } 47 48 inline bool operator==(const TestIdentifier &a, const TestIdentifier &b) 49 { 50 return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName); 51 } 52 53 inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id) 54 { 55 return os << id.testSuiteName << "." << id.testName; 56 } 57 58 enum class TestResultType 59 { 60 Crash, 61 Fail, 62 NoResult, 63 Pass, 64 Skip, 65 Timeout, 66 Unknown, 67 }; 68 69 const char *TestResultTypeToString(TestResultType type); 70 71 struct TestResult 72 { 73 TestResultType type = TestResultType::NoResult; 74 std::vector<double> elapsedTimeSeconds = std::vector<double>({0.0}); 75 uint32_t flakyFailures = 0; 76 }; 77 78 inline bool operator==(const TestResult &a, const TestResult &b) 79 { 80 return a.type == b.type; 81 } 82 83 inline std::ostream &operator<<(std::ostream &os, const TestResult &result) 84 { 85 return os << TestResultTypeToString(result.type); 86 } 87 88 struct TestResults 89 { 90 TestResults(); 91 ~TestResults(); 92 93 std::map<TestIdentifier, TestResult> results; 94 std::mutex currentTestMutex; 95 TestIdentifier currentTest; 96 Timer currentTestTimer; 97 double currentTestTimeout = 0.0; 98 bool allDone = false; 99 std::string testArtifactsFakeTestName; 100 std::vector<std::string> testArtifactPaths; 101 }; 102 103 struct FileLine 104 { 105 const char *file; 106 int line; 107 }; 108 109 struct ProcessInfo : angle::NonCopyable 110 { 111 ProcessInfo(); 112 ~ProcessInfo(); 113 ProcessInfo(ProcessInfo &&other); 114 ProcessInfo &operator=(ProcessInfo &&rhs); 115 116 ProcessHandle process; 117 std::vector<TestIdentifier> testsInBatch; 118 std::string resultsFileName; 119 std::string filterFileName; 120 std::string commandLine; 121 std::string filterString; 122 }; 123 124 using TestQueue = std::queue<std::vector<TestIdentifier>>; 125 126 class TestSuite 127 { 128 public: 129 TestSuite(int *argc, char **argv); 130 ~TestSuite(); 131 132 int run(); 133 void onCrashOrTimeout(TestResultType crashOrTimeout); 134 void addHistogramSample(const std::string &measurement, 135 const std::string &story, 136 double value, 137 const std::string &units); 138 GetInstance()139 static TestSuite *GetInstance() { return mInstance; } 140 141 // Returns the path to the artifact in the output directory. 142 bool hasTestArtifactsDirectory() const; 143 std::string reserveTestArtifactPath(const std::string &artifactName); 144 getShardIndex()145 int getShardIndex() const { return mShardIndex; } getBatchId()146 int getBatchId() const { return mBatchId; } 147 148 // Test expectation processing. 149 bool loadTestExpectationsFromFileWithConfig(const GPUTestConfig &config, 150 const std::string &fileName); 151 bool loadAllTestExpectationsFromFile(const std::string &fileName); 152 int32_t getTestExpectation(const std::string &testName); 153 void maybeUpdateTestTimeout(uint32_t testExpectation); 154 int32_t getTestExpectationWithConfigAndUpdateTimeout(const GPUTestConfig &config, 155 const std::string &testName); 156 bool logAnyUnusedTestExpectations(); setTestExpectationsAllowMask(uint32_t mask)157 void setTestExpectationsAllowMask(uint32_t mask) 158 { 159 mTestExpectationsParser.setTestExpectationsAllowMask(mask); 160 } 161 162 private: 163 bool parseSingleArg(const char *argument); 164 bool launchChildTestProcess(uint32_t batchId, const std::vector<TestIdentifier> &testsInBatch); 165 bool finishProcess(ProcessInfo *processInfo); 166 int printFailuresAndReturnCount() const; 167 void startWatchdog(); 168 void dumpTestExpectationsErrorMessages(); 169 int getSlowTestTimeout() const; 170 171 static TestSuite *mInstance; 172 173 std::string mTestExecutableName; 174 std::string mTestSuiteName; 175 TestQueue mTestQueue; 176 std::string mFilterString; 177 std::string mFilterFile; 178 std::string mResultsDirectory; 179 std::string mResultsFile; 180 std::string mHistogramJsonFile; 181 int mShardCount; 182 int mShardIndex; 183 angle::CrashCallback mCrashCallback; 184 TestResults mTestResults; 185 bool mBotMode; 186 bool mDebugTestGroups; 187 bool mGTestListTests; 188 bool mListTests; 189 bool mPrintTestStdout; 190 bool mDisableCrashHandler; 191 int mBatchSize; 192 int mCurrentResultCount; 193 int mTotalResultCount; 194 int mMaxProcesses; 195 int mTestTimeout; 196 int mBatchTimeout; 197 int mBatchId; 198 int mFlakyRetries; 199 int mMaxFailures; 200 int mFailureCount; 201 bool mModifiedPreferredDevice; 202 std::vector<std::string> mChildProcessArgs; 203 std::map<TestIdentifier, FileLine> mTestFileLines; 204 std::vector<ProcessInfo> mCurrentProcesses; 205 std::thread mWatchdogThread; 206 HistogramWriter mHistogramWriter; 207 std::string mTestArtifactDirectory; 208 GPUTestExpectationsParser mTestExpectationsParser; 209 }; 210 211 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut); 212 } // namespace angle 213 214 #endif // ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_ 215