1 // 2 // Copyright 2019 The ANGLE Project Authors. All rights reserved. 3 // Use of this source code is governed by a BSD-style license that can be 4 // found in the LICENSE file. 5 // 6 // TestSuite: 7 // Basic implementation of a test harness in ANGLE. 8 9 #ifndef ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_ 10 #define ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_ 11 12 #include <map> 13 #include <memory> 14 #include <mutex> 15 #include <queue> 16 #include <string> 17 #include <thread> 18 19 #include "HistogramWriter.h" 20 #include "tests/test_expectations/GPUTestExpectationsParser.h" 21 #include "util/test_utils.h" 22 23 namespace angle 24 { 25 struct TestIdentifier 26 { 27 TestIdentifier(); 28 TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn); 29 TestIdentifier(const TestIdentifier &other); 30 ~TestIdentifier(); 31 32 TestIdentifier &operator=(const TestIdentifier &other); 33 34 static bool ParseFromString(const std::string &str, TestIdentifier *idOut); 35 validTestIdentifier36 bool valid() const { return !testName.empty(); } 37 void sprintfName(char *outBuffer) const; 38 39 std::string testSuiteName; 40 std::string testName; 41 }; 42 43 inline bool operator<(const TestIdentifier &a, const TestIdentifier &b) 44 { 45 return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName); 46 } 47 48 inline bool operator==(const TestIdentifier &a, const TestIdentifier &b) 49 { 50 return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName); 51 } 52 53 inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id) 54 { 55 return os << id.testSuiteName << "." << id.testName; 56 } 57 58 enum class TestResultType 59 { 60 Crash, 61 Fail, 62 NoResult, 63 Pass, 64 Skip, 65 Timeout, 66 Unknown, 67 }; 68 69 const char *TestResultTypeToString(TestResultType type); 70 71 struct TestResult 72 { 73 TestResultType type = TestResultType::NoResult; 74 double elapsedTimeSeconds = 0.0; 75 uint32_t flakyFailures = 0; 76 }; 77 78 inline bool operator==(const TestResult &a, const TestResult &b) 79 { 80 return a.type == b.type; 81 } 82 83 inline std::ostream &operator<<(std::ostream &os, const TestResult &result) 84 { 85 return os << TestResultTypeToString(result.type); 86 } 87 88 struct TestResults 89 { 90 TestResults(); 91 ~TestResults(); 92 93 std::map<TestIdentifier, TestResult> results; 94 std::mutex currentTestMutex; 95 TestIdentifier currentTest; 96 Timer currentTestTimer; 97 double currentTestTimeout = 0.0; 98 bool allDone = false; 99 std::string testArtifactsFakeTestName; 100 std::vector<std::string> testArtifactPaths; 101 }; 102 103 struct FileLine 104 { 105 const char *file; 106 int line; 107 }; 108 109 struct ProcessInfo : angle::NonCopyable 110 { 111 ProcessInfo(); 112 ~ProcessInfo(); 113 ProcessInfo(ProcessInfo &&other); 114 ProcessInfo &operator=(ProcessInfo &&rhs); 115 116 ProcessHandle process; 117 std::vector<TestIdentifier> testsInBatch; 118 std::string resultsFileName; 119 std::string filterFileName; 120 std::string commandLine; 121 std::string filterString; 122 }; 123 124 using TestQueue = std::queue<std::vector<TestIdentifier>>; 125 126 class TestSuite 127 { 128 public: 129 TestSuite(int *argc, char **argv); 130 ~TestSuite(); 131 132 int run(); 133 void onCrashOrTimeout(TestResultType crashOrTimeout); 134 void addHistogramSample(const std::string &measurement, 135 const std::string &story, 136 double value, 137 const std::string &units); 138 void registerSlowTests(const char *slowTests[], size_t numSlowTests); 139 GetInstance()140 static TestSuite *GetInstance() { return mInstance; } 141 142 // Returns the path to the artifact in the output directory. 143 std::string addTestArtifact(const std::string &artifactName); 144 getShardIndex()145 int getShardIndex() const { return mShardIndex; } getBatchId()146 int getBatchId() const { return mBatchId; } 147 148 // Test expectation processing. 149 bool loadTestExpectationsFromFileWithConfig(const GPUTestConfig &config, 150 const std::string &fileName); 151 bool loadAllTestExpectationsFromFile(const std::string &fileName); 152 int32_t getTestExpectation(const std::string &testName); 153 int32_t getTestExpectationWithConfig(const GPUTestConfig &config, const std::string &testName); 154 bool logAnyUnusedTestExpectations(); setTestExpectationsAllowMask(uint32_t mask)155 void setTestExpectationsAllowMask(uint32_t mask) 156 { 157 mTestExpectationsParser.setTestExpectationsAllowMask(mask); 158 } 159 160 private: 161 bool parseSingleArg(const char *argument); 162 bool launchChildTestProcess(uint32_t batchId, const std::vector<TestIdentifier> &testsInBatch); 163 bool finishProcess(ProcessInfo *processInfo); 164 int printFailuresAndReturnCount() const; 165 void startWatchdog(); 166 void dumpTestExpectationsErrorMessages(); 167 168 static TestSuite *mInstance; 169 170 std::string mTestExecutableName; 171 std::string mTestSuiteName; 172 TestQueue mTestQueue; 173 std::string mFilterString; 174 std::string mFilterFile; 175 std::string mResultsDirectory; 176 std::string mResultsFile; 177 std::string mHistogramJsonFile; 178 int mShardCount; 179 int mShardIndex; 180 angle::CrashCallback mCrashCallback; 181 TestResults mTestResults; 182 bool mBotMode; 183 bool mDebugTestGroups; 184 bool mGTestListTests; 185 bool mListTests; 186 bool mPrintTestStdout; 187 bool mDisableCrashHandler; 188 int mBatchSize; 189 int mCurrentResultCount; 190 int mTotalResultCount; 191 int mMaxProcesses; 192 int mTestTimeout; 193 int mBatchTimeout; 194 int mBatchId; 195 int mFlakyRetries; 196 int mMaxFailures; 197 int mFailureCount; 198 std::vector<std::string> mChildProcessArgs; 199 std::map<TestIdentifier, FileLine> mTestFileLines; 200 std::vector<ProcessInfo> mCurrentProcesses; 201 std::thread mWatchdogThread; 202 HistogramWriter mHistogramWriter; 203 std::vector<std::string> mSlowTests; 204 std::string mTestArtifactDirectory; 205 GPUTestExpectationsParser mTestExpectationsParser; 206 }; 207 208 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut); 209 } // namespace angle 210 211 #endif // ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_ 212