1 //
2 // Copyright 2019 The ANGLE Project Authors. All rights reserved.
3 // Use of this source code is governed by a BSD-style license that can be
4 // found in the LICENSE file.
5 //
6 // TestSuite_unittest.cpp: Unit tests for ANGLE's test harness.
7 //
8
9 #include <gtest/gtest.h>
10
11 #include "../angle_test_instantiate.h"
12 #include "TestSuite.h"
13 #include "common/debug.h"
14 #include "common/system_utils.h"
15 #include "util/test_utils.h"
16 #include "util/test_utils_unittest_helper.h"
17
18 #include <rapidjson/document.h>
19
20 using namespace angle;
21
22 namespace js = rapidjson;
23
24 // This file is included in both angle_unittests and test_utils_unittest_helper. This variable is
25 // defined separately in each test target's main file.
26 extern bool gVerbose;
27
28 namespace
29 {
30 constexpr char kTestHelperExecutable[] = "test_utils_unittest_helper";
31 constexpr int kFlakyRetries = 3;
32
33 class TestSuiteTest : public testing::Test
34 {
35 protected:
TearDown()36 void TearDown() override
37 {
38 if (!mTempFileName.empty())
39 {
40 angle::DeleteSystemFile(mTempFileName.c_str());
41 }
42 }
43
runTestSuite(const std::vector<std::string> & extraArgs,TestResults * actualResults,bool validateStderr)44 bool runTestSuite(const std::vector<std::string> &extraArgs,
45 TestResults *actualResults,
46 bool validateStderr)
47 {
48 std::string executablePath = GetExecutableDirectory();
49 EXPECT_NE(executablePath, "");
50 executablePath += std::string("/") + kTestHelperExecutable + GetExecutableExtension();
51
52 const Optional<std::string> tempDirName = GetTempDirectory();
53 if (!tempDirName.valid())
54 {
55 return false;
56 }
57
58 Optional<std::string> tempFile = CreateTemporaryFileInDirectory(tempDirName.value());
59 if (!tempFile.valid())
60 {
61 return false;
62 }
63
64 mTempFileName = tempFile.value();
65 std::string resultsFileName = "--results-file=" + mTempFileName;
66
67 std::vector<const char *> args = {
68 executablePath.c_str(), kRunTestSuite, "--gtest_also_run_disabled_tests",
69 "--bot-mode", "--test-timeout=5", resultsFileName.c_str()};
70
71 for (const std::string &arg : extraArgs)
72 {
73 args.push_back(arg.c_str());
74 }
75
76 if (gVerbose)
77 {
78 printf("Test arguments:\n");
79 for (const char *arg : args)
80 {
81 printf("%s ", arg);
82 }
83 printf("\n");
84 }
85
86 ProcessHandle process(args, ProcessOutputCapture::StdoutAndStderrSeparately);
87 EXPECT_TRUE(process->started());
88 EXPECT_TRUE(process->finish());
89 EXPECT_TRUE(process->finished());
90
91 if (validateStderr)
92 {
93 EXPECT_EQ(process->getStderr(), "");
94 }
95
96 if (gVerbose)
97 {
98 printf("stdout:\n%s\n", process->getStdout().c_str());
99 }
100
101 return GetTestResultsFromFile(mTempFileName.c_str(), actualResults);
102 }
103
104 std::string mTempFileName;
105 };
106
107 // Tests the ANGLE standalone testing harness. Runs four tests with different ending conditions.
108 // Verifies that Pass, Fail, Crash and Timeout are all handled correctly.
TEST_F(TestSuiteTest,RunMockTests)109 TEST_F(TestSuiteTest, RunMockTests)
110 {
111 std::vector<std::string> extraArgs = {"--gtest_filter=MockTestSuiteTest.DISABLED_*"};
112
113 TestResults actual;
114 ASSERT_TRUE(runTestSuite(extraArgs, &actual, true));
115
116 std::map<TestIdentifier, TestResult> expectedResults = {
117 {{"MockTestSuiteTest", "DISABLED_Pass"},
118 {TestResultType::Pass, std::vector<double>({0.0})}},
119 {{"MockTestSuiteTest", "DISABLED_Fail"},
120 {TestResultType::Fail, std::vector<double>({0.0})}},
121 {{"MockTestSuiteTest", "DISABLED_Skip"},
122 {TestResultType::Skip, std::vector<double>({0.0})}},
123 {{"MockTestSuiteTest", "DISABLED_Timeout"},
124 {TestResultType::Timeout, std::vector<double>({0.0})}},
125 };
126
127 EXPECT_EQ(expectedResults, actual.results);
128 }
129
130 // Verifies the flaky retry feature works as expected.
TEST_F(TestSuiteTest,RunFlakyTests)131 TEST_F(TestSuiteTest, RunFlakyTests)
132 {
133 std::vector<std::string> extraArgs = {"--gtest_filter=MockFlakyTestSuiteTest.DISABLED_Flaky",
134 "--flaky-retries=" + std::to_string(kFlakyRetries)};
135
136 TestResults actual;
137 ASSERT_TRUE(runTestSuite(extraArgs, &actual, true));
138
139 std::vector<double> times;
140 for (int i = 0; i < kFlakyRetries; i++)
141 {
142 times.push_back(0.0);
143 }
144 std::map<TestIdentifier, TestResult> expectedResults = {
145 {{"MockFlakyTestSuiteTest", "DISABLED_Flaky"},
146 {TestResultType::Pass, times, kFlakyRetries - 1}}};
147
148 EXPECT_EQ(expectedResults, actual.results);
149 }
150
151 // Verifies that crashes are handled even without the crash handler.
TEST_F(TestSuiteTest,RunCrashingTests)152 TEST_F(TestSuiteTest, RunCrashingTests)
153 {
154 std::vector<std::string> extraArgs = {
155 "--gtest_filter=MockTestSuiteTest.DISABLED_Pass:MockTestSuiteTest.DISABLED_Fail:"
156 "MockTestSuiteTest.DISABLED_Skip:"
157 "MockCrashTestSuiteTest.DISABLED_*",
158 "--disable-crash-handler"};
159
160 TestResults actual;
161 ASSERT_TRUE(runTestSuite(extraArgs, &actual, false));
162
163 std::map<TestIdentifier, TestResult> expectedResults = {
164 {{"MockTestSuiteTest", "DISABLED_Pass"},
165 {TestResultType::Pass, std::vector<double>({0.0})}},
166 {{"MockTestSuiteTest", "DISABLED_Fail"},
167 {TestResultType::Fail, std::vector<double>({0.0})}},
168 {{"MockTestSuiteTest", "DISABLED_Skip"},
169 {TestResultType::Skip, std::vector<double>({0.0})}},
170 {{"MockCrashTestSuiteTest", "DISABLED_Crash"},
171 {TestResultType::Crash, std::vector<double>({0.0})}},
172 {{"MockCrashTestSuiteTest", "DISABLED_PassAfterCrash"},
173 {TestResultType::Pass, std::vector<double>({0.0})}},
174 {{"MockCrashTestSuiteTest", "DISABLED_SkipAfterCrash"},
175 {TestResultType::Skip, std::vector<double>({0.0})}},
176 };
177
178 EXPECT_EQ(expectedResults, actual.results);
179 }
180
181 // Normal passing test.
TEST(MockTestSuiteTest,DISABLED_Pass)182 TEST(MockTestSuiteTest, DISABLED_Pass)
183 {
184 EXPECT_TRUE(true);
185 }
186
187 // Normal failing test.
TEST(MockTestSuiteTest,DISABLED_Fail)188 TEST(MockTestSuiteTest, DISABLED_Fail)
189 {
190 EXPECT_TRUE(false);
191 }
192
193 // Trigger a test timeout.
TEST(MockTestSuiteTest,DISABLED_Timeout)194 TEST(MockTestSuiteTest, DISABLED_Timeout)
195 {
196 angle::Sleep(20000);
197 }
198
199 // Trigger a test skip.
TEST(MockTestSuiteTest,DISABLED_Skip)200 TEST(MockTestSuiteTest, DISABLED_Skip)
201 {
202 GTEST_SKIP() << "Test skipped.";
203 }
204
205 // Trigger a flaky test failure.
TEST(MockFlakyTestSuiteTest,DISABLED_Flaky)206 TEST(MockFlakyTestSuiteTest, DISABLED_Flaky)
207 {
208 const Optional<std::string> tempDirName = GetTempDirectory();
209 ASSERT_TRUE(tempDirName.valid());
210
211 std::stringstream tempFNameStream;
212 tempFNameStream << tempDirName.value() << GetPathSeparator() << "flaky_temp.txt";
213 std::string tempFileName = tempFNameStream.str();
214
215 int fails = 0;
216 {
217 FILE *fp = fopen(tempFileName.c_str(), "r");
218 if (fp)
219 {
220 ASSERT_EQ(fscanf(fp, "%d", &fails), 1);
221 fclose(fp);
222 }
223 }
224
225 if (fails >= kFlakyRetries - 1)
226 {
227 angle::DeleteSystemFile(tempFileName.c_str());
228 }
229 else
230 {
231 EXPECT_TRUE(false);
232
233 FILE *fp = fopen(tempFileName.c_str(), "w");
234 ASSERT_NE(fp, nullptr);
235
236 fprintf(fp, "%d", fails + 1);
237 fclose(fp);
238 }
239 }
240
241 // Trigger a test crash.
TEST(MockCrashTestSuiteTest,DISABLED_Crash)242 TEST(MockCrashTestSuiteTest, DISABLED_Crash)
243 {
244 ANGLE_CRASH();
245 }
246
247 // This test runs after the crash test.
TEST(MockCrashTestSuiteTest,DISABLED_PassAfterCrash)248 TEST(MockCrashTestSuiteTest, DISABLED_PassAfterCrash)
249 {
250 EXPECT_TRUE(true);
251 }
252
253 // This test runs after the crash test.
TEST(MockCrashTestSuiteTest,DISABLED_SkipAfterCrash)254 TEST(MockCrashTestSuiteTest, DISABLED_SkipAfterCrash)
255 {
256 GTEST_SKIP() << "Test skipped.";
257 }
258 } // namespace
259