• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //
2 // Copyright 2019 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // TestSuite:
7 //   Basic implementation of a test harness in ANGLE.
8 
9 #ifndef ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
10 #define ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
11 
12 #include <map>
13 #include <memory>
14 #include <mutex>
15 #include <queue>
16 #include <string>
17 #include <thread>
18 
19 #include "HistogramWriter.h"
20 #include "tests/test_expectations/GPUTestExpectationsParser.h"
21 #include "util/test_utils.h"
22 
23 namespace angle
24 {
25 struct TestIdentifier
26 {
27     TestIdentifier();
28     TestIdentifier(const std::string &suiteNameIn, const std::string &nameIn);
29     TestIdentifier(const TestIdentifier &other);
30     ~TestIdentifier();
31 
32     TestIdentifier &operator=(const TestIdentifier &other);
33 
34     static bool ParseFromString(const std::string &str, TestIdentifier *idOut);
35 
validTestIdentifier36     bool valid() const { return !testName.empty(); }
37     void snprintfName(char *outBuffer, size_t maxLen) const;
38 
39     std::string testSuiteName;
40     std::string testName;
41 };
42 
43 inline bool operator<(const TestIdentifier &a, const TestIdentifier &b)
44 {
45     return std::tie(a.testSuiteName, a.testName) < std::tie(b.testSuiteName, b.testName);
46 }
47 
48 inline bool operator==(const TestIdentifier &a, const TestIdentifier &b)
49 {
50     return std::tie(a.testSuiteName, a.testName) == std::tie(b.testSuiteName, b.testName);
51 }
52 
53 inline std::ostream &operator<<(std::ostream &os, const TestIdentifier &id)
54 {
55     return os << id.testSuiteName << "." << id.testName;
56 }
57 
58 enum class TestResultType
59 {
60     Crash,
61     Fail,
62     NoResult,
63     Pass,
64     Skip,
65     Timeout,
66     Unknown,
67 };
68 
69 const char *TestResultTypeToString(TestResultType type);
70 
71 struct TestResult
72 {
73     TestResultType type                    = TestResultType::NoResult;
74     std::vector<double> elapsedTimeSeconds = std::vector<double>({0.0});
75     uint32_t flakyFailures                 = 0;
76 };
77 
78 inline bool operator==(const TestResult &a, const TestResult &b)
79 {
80     return a.type == b.type;
81 }
82 
83 inline std::ostream &operator<<(std::ostream &os, const TestResult &result)
84 {
85     return os << TestResultTypeToString(result.type);
86 }
87 
88 struct TestResults
89 {
90     TestResults();
91     ~TestResults();
92 
93     std::map<TestIdentifier, TestResult> results;
94     std::mutex currentTestMutex;
95     TestIdentifier currentTest;
96     Timer currentTestTimer;
97     double currentTestTimeout = 0.0;
98     bool allDone              = false;
99     std::string testArtifactsFakeTestName;
100     std::vector<std::string> testArtifactPaths;
101 };
102 
103 struct FileLine
104 {
105     const char *file;
106     int line;
107 };
108 
109 struct ProcessInfo : angle::NonCopyable
110 {
111     ProcessInfo();
112     ~ProcessInfo();
113     ProcessInfo(ProcessInfo &&other);
114     ProcessInfo &operator=(ProcessInfo &&rhs);
115 
116     ProcessHandle process;
117     std::vector<TestIdentifier> testsInBatch;
118     std::string resultsFileName;
119     std::string filterFileName;
120     std::string commandLine;
121     std::string filterString;
122 };
123 
124 using TestQueue = std::queue<std::vector<TestIdentifier>>;
125 
126 class MetricWriter
127 {
128   public:
MetricWriter()129     MetricWriter() {}
130 
131     void enable(const std::string &testArtifactDirectory);
132 
133     void writeInfo(const std::string &name,
134                    const std::string &backend,
135                    const std::string &story,
136                    const std::string &metric,
137                    const std::string &units);
138 
139     void writeDoubleValue(double value);
140     void writeIntegerValue(size_t value);
141 
142     void close();
143 
144   private:
145     std::string mPath;
146     FILE *mFile = nullptr;
147 };
148 
149 class TestSuite
150 {
151   public:
152     TestSuite(int *argc, char **argv);
153     TestSuite(int *argc, char **argv, std::function<void()> registerTestsCallback);
154     ~TestSuite();
155 
156     int run();
157     void onCrashOrTimeout(TestResultType crashOrTimeout);
158     void addHistogramSample(const std::string &measurement,
159                             const std::string &story,
160                             double value,
161                             const std::string &units);
162 
GetInstance()163     static TestSuite *GetInstance() { return mInstance; }
GetMetricWriter()164     static MetricWriter &GetMetricWriter() { return GetInstance()->mMetricWriter; }
165 
166     // Returns the path to the artifact in the output directory.
167     bool hasTestArtifactsDirectory() const;
168     std::string reserveTestArtifactPath(const std::string &artifactName);
169 
getShardIndex()170     int getShardIndex() const { return mShardIndex; }
getBatchId()171     int getBatchId() const { return mBatchId; }
172 
173     // Test expectation processing.
174     bool loadTestExpectationsFromFileWithConfig(const GPUTestConfig &config,
175                                                 const std::string &fileName);
176     bool loadAllTestExpectationsFromFile(const std::string &fileName);
177     int32_t getTestExpectation(const std::string &testName);
178     void maybeUpdateTestTimeout(uint32_t testExpectation);
179     int32_t getTestExpectationWithConfigAndUpdateTimeout(const GPUTestConfig &config,
180                                                          const std::string &testName);
181     bool logAnyUnusedTestExpectations();
setTestExpectationsAllowMask(uint32_t mask)182     void setTestExpectationsAllowMask(uint32_t mask)
183     {
184         mTestExpectationsParser.setTestExpectationsAllowMask(mask);
185     }
186 
getTestExecutableName()187     const std::string &getTestExecutableName() const { return mTestExecutableName; }
188 
189   private:
190     bool parseSingleArg(int *argc, char **argv, int argIndex);
191     bool launchChildTestProcess(uint32_t batchId, const std::vector<TestIdentifier> &testsInBatch);
192     bool finishProcess(ProcessInfo *processInfo);
193     int printFailuresAndReturnCount() const;
194     void startWatchdog();
195     void dumpTestExpectationsErrorMessages();
196     int getSlowTestTimeout() const;
197     void writeOutputFiles(bool interrupted);
198 
199     static TestSuite *mInstance;
200 
201     std::string mTestExecutableName;
202     TestQueue mTestQueue;
203     std::string mFilterString;
204     std::string mFilterFile;
205     std::string mResultsDirectory;
206     std::string mResultsFile;
207     std::string mHistogramJsonFile;
208     int mShardCount;
209     int mShardIndex;
210     angle::CrashCallback mCrashCallback;
211     TestResults mTestResults;
212     bool mBotMode;
213     bool mDebugTestGroups;
214     bool mGTestListTests;
215     bool mListTests;
216     bool mPrintTestStdout;
217     bool mDisableCrashHandler;
218     int mBatchSize;
219     int mCurrentResultCount;
220     int mTotalResultCount;
221     int mMaxProcesses;
222     int mTestTimeout;
223     int mBatchTimeout;
224     int mBatchId;
225     int mFlakyRetries;
226     int mMaxFailures;
227     int mFailureCount;
228     bool mModifiedPreferredDevice;
229     std::vector<std::string> mChildProcessArgs;
230     std::map<TestIdentifier, FileLine> mTestFileLines;
231     std::vector<ProcessInfo> mCurrentProcesses;
232     std::thread mWatchdogThread;
233     HistogramWriter mHistogramWriter;
234     MetricWriter mMetricWriter;
235     std::string mTestArtifactDirectory;
236     GPUTestExpectationsParser mTestExpectationsParser;
237 
238     class TestEventListener;
239 };
240 
241 std::string ReplaceDashesWithQuestionMark(std::string dashesString);
242 bool GetTestResultsFromFile(const char *fileName, TestResults *resultsOut);
243 }  // namespace angle
244 
245 #endif  // ANGLE_TESTS_TEST_UTILS_TEST_SUITE_H_
246