1 //
2 // Copyright 2019 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // TestSuite_unittest.cpp: Unit tests for ANGLE's test harness.
7 //
8
9 #include <gtest/gtest.h>
10
11 #include "../angle_test_instantiate.h"
12 #include "TestSuite.h"
13 #include "common/debug.h"
14 #include "common/system_utils.h"
15 #include "util/test_utils.h"
16 #include "util/test_utils_unittest_helper.h"
17
18 #include <rapidjson/document.h>
19
20 using namespace angle;
21
22 namespace js = rapidjson;
23
24 // This file is included in both angle_unittests and test_utils_unittest_helper. This variable is
25 // defined separately in each test target's main file.
26 extern bool gVerbose;
27
28 namespace
29 {
30 constexpr char kTestHelperExecutable[] = "test_utils_unittest_helper";
31 constexpr int kFlakyRetries = 3;
32
33 class TestSuiteTest : public testing::Test
34 {
35 protected:
TearDown()36 void TearDown() override
37 {
38 if (!mTempFileName.empty())
39 {
40 angle::DeleteSystemFile(mTempFileName.c_str());
41 }
42 }
43
runTestSuite(const std::vector<std::string> & extraArgs,TestResults * actualResults,bool validateStderr)44 bool runTestSuite(const std::vector<std::string> &extraArgs,
45 TestResults *actualResults,
46 bool validateStderr)
47 {
48 std::string executablePath = GetExecutableDirectory();
49 EXPECT_NE(executablePath, "");
50 executablePath += std::string("/") + kTestHelperExecutable + GetExecutableExtension();
51
52 constexpr uint32_t kMaxTempDirLen = 100;
53 char tempDirName[kMaxTempDirLen * 2];
54
55 if (!GetTempDir(tempDirName, kMaxTempDirLen))
56 {
57 return false;
58 }
59
60 std::stringstream tempFNameStream;
61 tempFNameStream << tempDirName << GetPathSeparator() << "test_temp_" << rand() << ".json";
62 mTempFileName = tempFNameStream.str();
63
64 std::string resultsFileName = "--results-file=" + mTempFileName;
65
66 std::vector<const char *> args = {
67 executablePath.c_str(), kRunTestSuite, "--gtest_also_run_disabled_tests",
68 "--bot-mode", "--test-timeout=5", resultsFileName.c_str()};
69
70 for (const std::string &arg : extraArgs)
71 {
72 args.push_back(arg.c_str());
73 }
74
75 if (gVerbose)
76 {
77 printf("Test arguments:\n");
78 for (const char *arg : args)
79 {
80 printf("%s ", arg);
81 }
82 printf("\n");
83 }
84
85 ProcessHandle process(args, ProcessOutputCapture::StdoutAndStderrSeparately);
86 EXPECT_TRUE(process->started());
87 EXPECT_TRUE(process->finish());
88 EXPECT_TRUE(process->finished());
89
90 if (validateStderr)
91 {
92 EXPECT_EQ(process->getStderr(), "");
93 }
94
95 if (gVerbose)
96 {
97 printf("stdout:\n%s\n", process->getStdout().c_str());
98 }
99
100 return GetTestResultsFromFile(mTempFileName.c_str(), actualResults);
101 }
102
103 std::string mTempFileName;
104 };
105
106 // Tests the ANGLE standalone testing harness. Runs four tests with different ending conditions.
107 // Verifies that Pass, Fail, Crash and Timeout are all handled correctly.
TEST_F(TestSuiteTest,RunMockTests)108 TEST_F(TestSuiteTest, RunMockTests)
109 {
110 std::vector<std::string> extraArgs = {"--gtest_filter=MockTestSuiteTest.DISABLED_*"};
111
112 TestResults actual;
113 ASSERT_TRUE(runTestSuite(extraArgs, &actual, true));
114
115 std::map<TestIdentifier, TestResult> expectedResults = {
116 {{"MockTestSuiteTest", "DISABLED_Pass"},
117 {TestResultType::Pass, std::vector<double>({0.0})}},
118 {{"MockTestSuiteTest", "DISABLED_Fail"},
119 {TestResultType::Fail, std::vector<double>({0.0})}},
120 {{"MockTestSuiteTest", "DISABLED_Skip"},
121 {TestResultType::Skip, std::vector<double>({0.0})}},
122 {{"MockTestSuiteTest", "DISABLED_Timeout"},
123 {TestResultType::Timeout, std::vector<double>({0.0})}},
124 };
125
126 EXPECT_EQ(expectedResults, actual.results);
127 }
128
129 // Verifies the flaky retry feature works as expected.
TEST_F(TestSuiteTest,RunFlakyTests)130 TEST_F(TestSuiteTest, RunFlakyTests)
131 {
132 std::vector<std::string> extraArgs = {"--gtest_filter=MockFlakyTestSuiteTest.DISABLED_Flaky",
133 "--flaky-retries=" + std::to_string(kFlakyRetries)};
134
135 TestResults actual;
136 ASSERT_TRUE(runTestSuite(extraArgs, &actual, true));
137
138 std::vector<double> times;
139 for (int i = 0; i < kFlakyRetries; i++)
140 {
141 times.push_back(0.0);
142 }
143 std::map<TestIdentifier, TestResult> expectedResults = {
144 {{"MockFlakyTestSuiteTest", "DISABLED_Flaky"},
145 {TestResultType::Pass, times, kFlakyRetries - 1}}};
146
147 EXPECT_EQ(expectedResults, actual.results);
148 }
149
150 // Verifies that crashes are handled even without the crash handler.
TEST_F(TestSuiteTest,RunCrashingTests)151 TEST_F(TestSuiteTest, RunCrashingTests)
152 {
153 std::vector<std::string> extraArgs = {
154 "--gtest_filter=MockTestSuiteTest.DISABLED_Pass:MockTestSuiteTest.DISABLED_Fail:"
155 "MockTestSuiteTest.DISABLED_Skip:"
156 "MockCrashTestSuiteTest.DISABLED_*",
157 "--disable-crash-handler"};
158
159 TestResults actual;
160 ASSERT_TRUE(runTestSuite(extraArgs, &actual, false));
161
162 std::map<TestIdentifier, TestResult> expectedResults = {
163 {{"MockTestSuiteTest", "DISABLED_Pass"},
164 {TestResultType::Pass, std::vector<double>({0.0})}},
165 {{"MockTestSuiteTest", "DISABLED_Fail"},
166 {TestResultType::Fail, std::vector<double>({0.0})}},
167 {{"MockTestSuiteTest", "DISABLED_Skip"},
168 {TestResultType::Skip, std::vector<double>({0.0})}},
169 {{"MockCrashTestSuiteTest", "DISABLED_Crash"},
170 {TestResultType::Crash, std::vector<double>({0.0})}},
171 {{"MockCrashTestSuiteTest", "DISABLED_PassAfterCrash"},
172 {TestResultType::Pass, std::vector<double>({0.0})}},
173 {{"MockCrashTestSuiteTest", "DISABLED_SkipAfterCrash"},
174 {TestResultType::Skip, std::vector<double>({0.0})}},
175 };
176
177 EXPECT_EQ(expectedResults, actual.results);
178 }
179
180 // Normal passing test.
TEST(MockTestSuiteTest,DISABLED_Pass)181 TEST(MockTestSuiteTest, DISABLED_Pass)
182 {
183 EXPECT_TRUE(true);
184 }
185
186 // Normal failing test.
TEST(MockTestSuiteTest,DISABLED_Fail)187 TEST(MockTestSuiteTest, DISABLED_Fail)
188 {
189 EXPECT_TRUE(false);
190 }
191
192 // Trigger a test timeout.
TEST(MockTestSuiteTest,DISABLED_Timeout)193 TEST(MockTestSuiteTest, DISABLED_Timeout)
194 {
195 angle::Sleep(20000);
196 }
197
198 // Trigger a test skip.
TEST(MockTestSuiteTest,DISABLED_Skip)199 TEST(MockTestSuiteTest, DISABLED_Skip)
200 {
201 GTEST_SKIP() << "Test skipped.";
202 }
203
204 // Trigger a flaky test failure.
TEST(MockFlakyTestSuiteTest,DISABLED_Flaky)205 TEST(MockFlakyTestSuiteTest, DISABLED_Flaky)
206 {
207 constexpr uint32_t kMaxTempDirLen = 100;
208 char tempDirName[kMaxTempDirLen * 2];
209 ASSERT_TRUE(GetTempDir(tempDirName, kMaxTempDirLen));
210
211 std::stringstream tempFNameStream;
212 tempFNameStream << tempDirName << GetPathSeparator() << "flaky_temp.txt";
213 std::string tempFileName = tempFNameStream.str();
214
215 int fails = 0;
216 {
217 FILE *fp = fopen(tempFileName.c_str(), "r");
218 if (fp)
219 {
220 ASSERT_EQ(fscanf(fp, "%d", &fails), 1);
221 fclose(fp);
222 }
223 }
224
225 if (fails >= kFlakyRetries - 1)
226 {
227 angle::DeleteSystemFile(tempFileName.c_str());
228 }
229 else
230 {
231 EXPECT_TRUE(false);
232
233 FILE *fp = fopen(tempFileName.c_str(), "w");
234 ASSERT_NE(fp, nullptr);
235
236 fprintf(fp, "%d", fails + 1);
237 fclose(fp);
238 }
239 }
240
241 // Trigger a test crash.
TEST(MockCrashTestSuiteTest,DISABLED_Crash)242 TEST(MockCrashTestSuiteTest, DISABLED_Crash)
243 {
244 ANGLE_CRASH();
245 }
246
247 // This test runs after the crash test.
TEST(MockCrashTestSuiteTest,DISABLED_PassAfterCrash)248 TEST(MockCrashTestSuiteTest, DISABLED_PassAfterCrash)
249 {
250 EXPECT_TRUE(true);
251 }
252
253 // This test runs after the crash test.
TEST(MockCrashTestSuiteTest,DISABLED_SkipAfterCrash)254 TEST(MockCrashTestSuiteTest, DISABLED_SkipAfterCrash)
255 {
256 GTEST_SKIP() << "Test skipped.";
257 }
258 } // namespace
259