1 // 2 // Copyright 2019 The ANGLE Project Authors. All rights reserved. 3 // Use of this source code is governed by a BSD-style license that can be 4 // found in the LICENSE file. 5 // 6 // TestSuite: 7 // Basic implementation of a test harness in ANGLE. 8 9 #include <map> 10 #include <memory> 11 #include <mutex> 12 #include <string> 13 #include <thread> 14 15 #include "util/test_utils.h" 16 17 namespace angle 18 { 19 struct TestIdentifier 20 { 21 TestIdentifier(); 22 TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn); 23 TestIdentifier(const TestIdentifier &other); 24 ~TestIdentifier(); 25 26 TestIdentifier &operator=(const TestIdentifier &other); 27 28 static bool ParseFromString(const std::string &str, TestIdentifier *idOut); 29 validTestIdentifier30 bool valid() const { return !testName.empty(); } 31 void sprintfName(char *outBuffer) const; 32 33 std::string testSuiteName; 34 std::string testName; 35 }; 36 37 inline bool operator<(const TestIdentifier &a, const TestIdentifier &b) 38 { 39 return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName); 40 } 41 42 inline bool operator==(const TestIdentifier &a, const TestIdentifier &b) 43 { 44 return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName); 45 } 46 47 inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id) 48 { 49 return os << id.testSuiteName << "." << id.testName; 50 } 51 52 enum class TestResultType 53 { 54 Crash, 55 Fail, 56 Skip, 57 Pass, 58 Timeout, 59 Unknown, 60 }; 61 62 const char *TestResultTypeToString(TestResultType type); 63 64 struct TestResult 65 { 66 TestResultType type = TestResultType::Skip; 67 double elapsedTimeSeconds = 0.0; 68 }; 69 70 inline bool operator==(const TestResult &a, const TestResult &b) 71 { 72 return a.type == b.type; 73 } 74 75 inline std::ostream &operator<<(std::ostream &os, const TestResult &result) 76 { 77 return os << TestResultTypeToString(result.type); 78 } 79 80 struct TestResults 81 { 82 TestResults(); 83 ~TestResults(); 84 85 std::map<TestIdentifier, TestResult> results; 86 std::mutex currentTestMutex; 87 TestIdentifier currentTest; 88 Timer currentTestTimer; 89 bool allDone = false; 90 }; 91 92 struct FileLine 93 { 94 const char *file; 95 int line; 96 }; 97 98 struct ProcessInfo : angle::NonCopyable 99 { 100 ProcessInfo(); 101 ~ProcessInfo(); 102 ProcessInfo(ProcessInfo &&other); 103 ProcessInfo &operator=(ProcessInfo &&rhs); 104 105 ProcessHandle process; 106 std::vector<TestIdentifier> testsInBatch; 107 std::string resultsFileName; 108 std::string filterFileName; 109 std::string commandLine; 110 }; 111 112 class TestSuite 113 { 114 public: 115 TestSuite(int *argc, char **argv); 116 ~TestSuite(); 117 118 int run(); 119 void onCrashOrTimeout(TestResultType crashOrTimeout); 120 121 private: 122 bool parseSingleArg(const char *argument); 123 bool launchChildTestProcess(const std::vector<TestIdentifier> &testsInBatch); 124 bool finishProcess(ProcessInfo *processInfo); 125 int printFailuresAndReturnCount() const; 126 void startWatchdog(); 127 128 std::string mTestExecutableName; 129 std::string mTestSuiteName; 130 std::vector<TestIdentifier> mTestQueue; 131 std::string mFilterString; 132 std::string mFilterFile; 133 std::string mResultsDirectory; 134 std::string mResultsFile; 135 int mShardCount; 136 int mShardIndex; 137 angle::CrashCallback mCrashCallback; 138 TestResults mTestResults; 139 bool mBotMode; 140 int mBatchSize; 141 int mCurrentResultCount; 142 int mTotalResultCount; 143 int mMaxProcesses; 144 int mTestTimeout; 145 int mBatchTimeout; 146 std::vector<std::string> mGoogleTestCommandLineArgs; 147 std::map<TestIdentifier, FileLine> mTestFileLines; 148 std::vector<ProcessInfo> mCurrentProcesses; 149 std::thread mWatchdogThread; 150 }; 151 152 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut); 153 } // namespace angle 154