1 /*
2 * Copyright (C) 2014 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <fcntl.h>
18 #if !defined(__APPLE__)
19 #include <malloc.h>
20 #endif
21 #include <signal.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <sys/types.h>
25 #include <sys/wait.h>
26 #include <unistd.h>
27
28 #include <map>
29 #include <regex>
30 #include <sstream>
31 #include <string>
32 #include <vector>
33
34 #include <android-base/file.h>
35 #include <android-base/strings.h>
36 #include <android-base/test_utils.h>
37 #include <gtest/gtest.h>
38
39 #include "NanoTime.h"
40
41 #if defined(__APPLE__)
42 extern char** environ;
43
clearenv()44 int clearenv() {
45 *environ = nullptr;
46 return 0;
47 }
48 #endif
49
50 // Change the slow threshold for these tests since a few can take around
51 // 20 seconds.
GetInitialArgs(const char *** args,size_t * num_args)52 extern "C" bool GetInitialArgs(const char*** args, size_t* num_args) {
53 static const char* initial_args[] = {"--slow_threshold_ms=25000"};
54 *args = initial_args;
55 *num_args = 1;
56 return true;
57 }
58
59 namespace android {
60 namespace gtest_extras {
61
62 class SystemTests : public ::testing::Test {
63 protected:
SetUp()64 void SetUp() override {
65 raw_output_ = "";
66 sanitized_output_ = "";
67 exitcode_ = 0;
68
69 // Clear all environment variables to make sure the test isn't affected
70 // by GTEST_XXX ones.
71 clearenv();
72 }
73
74 void SanitizeOutput();
75
76 void Exec(std::vector<const char*> args);
77 void ExecAndCapture(std::vector<const char*> args);
78 void RunTest(const std::string& test_name, std::vector<const char*> extra_args = {});
79 void RunTestCaptureFooter(const std::string& test_name, std::string* footer,
80 std::vector<const char*> extra_args = {});
81 void Verify(const std::string& test_name, const std::string& expected_output,
82 int expected_exitcode, std::vector<const char*> extra_args = {});
83 void VerifySortedOutput(const std::string& test_name, const std::string& expected_output,
84 int expected_exitcode, std::vector<const char*> extra_args = {});
85
86 std::string raw_output_;
87 std::string sanitized_output_;
88 int exitcode_;
89 pid_t pid_;
90 int fd_;
91 };
92
SortTestOutput(const std::string & output)93 static std::string SortTestOutput(const std::string& output) {
94 std::map<std::string, std::string> tests;
95
96 std::vector<std::string> lines(android::base::Split(output, "\n"));
97 std::string prefix;
98 std::string suffix;
99 bool capture_prefix = true;
100 for (auto iter = lines.begin(); iter != lines.end(); ++iter) {
101 const char* line = iter->c_str();
102 if (line[0] == '\x1B') {
103 // Skip the escape sequence.
104 line = &line[7];
105 }
106 if (strncmp(line, "[ RUN ]", 12) == 0) {
107 std::string test_name(iter->substr(12));
108 std::string test_body(*iter + '\n');
109 bool ended = false;
110 for (++iter; iter != lines.end(); ++iter) {
111 test_body += *iter + '\n';
112 line = iter->c_str();
113 if (line[0] == '\x1B') {
114 // Skip the escape sequence.
115 line = &line[7];
116 }
117 if (strncmp(line, "[ ", 2) == 0) {
118 ended = true;
119 break;
120 }
121 }
122 if (!ended) {
123 return output;
124 }
125 tests[test_name] = test_body;
126 capture_prefix = false;
127 } else if (capture_prefix) {
128 prefix += *iter + '\n';
129 } else {
130 suffix += *iter + '\n';
131 }
132 }
133
134 std::string new_output("Output Sorted\n" + prefix);
135 for (auto entry : tests) {
136 new_output += entry.second;
137 }
138 new_output += suffix;
139 if (android::base::EndsWith(new_output, "\n\n")) {
140 new_output.resize(new_output.size() - 1);
141 }
142 return new_output;
143 }
144
SanitizeOutput()145 void SystemTests::SanitizeOutput() {
146 // Change (100 ms to (XX ms
147 sanitized_output_ =
148 std::regex_replace(raw_output_, std::regex("\\(\\d+ ms(\\)|\\s|,)"), "(XX ms$1");
149
150 // Change (elapsed time 100 ms to (elapsed time XX ms
151 sanitized_output_ = std::regex_replace(
152 sanitized_output_, std::regex("\\(elapsed time \\d+ ms(\\)|\\s|,)"), "(elapsed time XX ms$1");
153
154 // Change stopped|timeout at 100 ms to stopped|timeout at XX ms
155 sanitized_output_ = std::regex_replace(sanitized_output_,
156 std::regex("(stopped|timeout) at \\d+ ms"), "$1 at XX ms");
157
158 // Change any error message like .../file.cc:(200) to file:(XX)
159 sanitized_output_ = std::regex_replace(
160 sanitized_output_, std::regex("\\b([^/\\s]+/)*[^/\\s]+:\\(\\d+\\)\\s"), "file:(XX) ");
161
162 // Change any terminated by signal message to ignore the actual signal name.
163 sanitized_output_ =
164 std::regex_replace(sanitized_output_, std::regex("( terminated by signal:) .*"), "$1 XXX");
165 }
166
Exec(std::vector<const char * > args)167 void SystemTests::Exec(std::vector<const char*> args) {
168 int fds[2];
169 #if !defined(__APPLE__)
170 ASSERT_NE(-1, pipe2(fds, O_NONBLOCK));
171 #else
172 ASSERT_NE(-1, pipe(fds));
173 ASSERT_NE(-1, fcntl(fds[0], F_SETFL, O_NONBLOCK));
174 #endif
175
176 if ((pid_ = fork()) == 0) {
177 // Run the test.
178 close(fds[0]);
179 close(STDIN_FILENO);
180 close(STDOUT_FILENO);
181 close(STDERR_FILENO);
182 ASSERT_NE(0, dup2(fds[1], STDOUT_FILENO));
183 ASSERT_NE(0, dup2(fds[1], STDERR_FILENO));
184 close(fds[1]);
185
186 std::string exe_name = android::base::GetExecutablePath();
187 args.insert(args.begin(), exe_name.c_str());
188 args.push_back(nullptr);
189 execv(args[0], reinterpret_cast<char* const*>(const_cast<char**>(args.data())));
190 exit(1);
191 }
192 ASSERT_NE(-1, pid_);
193
194 close(fds[1]);
195 fd_ = fds[0];
196 }
197
ExecAndCapture(std::vector<const char * > args)198 void SystemTests::ExecAndCapture(std::vector<const char*> args) {
199 Exec(args);
200
201 int flags = fcntl(fd_, F_GETFL);
202 ASSERT_NE(-1, flags);
203 flags &= ~O_NONBLOCK;
204 ASSERT_NE(-1, fcntl(fd_, F_SETFL, flags));
205 ASSERT_TRUE(android::base::ReadFdToString(fd_, &raw_output_));
206 close(fd_);
207
208 int status;
209 ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << raw_output_;
210 exitcode_ = WEXITSTATUS(status);
211 SanitizeOutput();
212 }
213
RunTest(const std::string & test_name,std::vector<const char * > extra_args)214 void SystemTests::RunTest(const std::string& test_name, std::vector<const char*> extra_args) {
215 std::vector<const char*> args;
216 bool job_count = false;
217 for (const auto& arg : extra_args) {
218 if (strncmp(arg, "-j", 2) == 0) {
219 job_count = true;
220 }
221 args.push_back(arg);
222 }
223 if (!job_count) {
224 // Always set to only 20 jobs if no job count option is set.
225 args.push_back("-j20");
226 }
227 args.push_back("--gtest_also_run_disabled_tests");
228 std::string filter_arg("--gtest_filter=" + test_name);
229 args.push_back(filter_arg.c_str());
230
231 ExecAndCapture(args);
232 }
233
RunTestCaptureFooter(const std::string & test_name,std::string * footer,std::vector<const char * > extra_args)234 void SystemTests::RunTestCaptureFooter(const std::string& test_name, std::string* footer,
235 std::vector<const char*> extra_args) {
236 ASSERT_NO_FATAL_FAILURE(RunTest(test_name, extra_args));
237
238 // Verify the order of the output messages.
239 std::stringstream stream(sanitized_output_);
240 std::string line;
241 footer->clear();
242 while (std::getline(stream, line, '\n')) {
243 if (!footer->empty()) {
244 *footer += line + '\n';
245 } else if (android::base::StartsWith(line, "[ PASSED ] ")) {
246 *footer = line + '\n';
247 }
248 }
249 ASSERT_FALSE(footer->empty()) << "Test output:\n" << raw_output_;
250 }
251
Verify(const std::string & test_name,const std::string & expected_output,int expected_exitcode,std::vector<const char * > extra_args)252 void SystemTests::Verify(const std::string& test_name, const std::string& expected_output,
253 int expected_exitcode, std::vector<const char*> extra_args) {
254 ASSERT_NO_FATAL_FAILURE(RunTest(test_name, extra_args));
255 ASSERT_EQ(expected_exitcode, exitcode_) << "Test output:\n" << raw_output_;
256 if (!expected_output.empty()) {
257 ASSERT_EQ(expected_output, sanitized_output_);
258 }
259 }
260
VerifySortedOutput(const std::string & test_name,const std::string & expected_output,int expected_exitcode,std::vector<const char * > extra_args)261 void SystemTests::VerifySortedOutput(const std::string& test_name,
262 const std::string& expected_output, int expected_exitcode,
263 std::vector<const char*> extra_args) {
264 ASSERT_NO_FATAL_FAILURE(RunTest(test_name, extra_args));
265 ASSERT_EQ(expected_exitcode, exitcode_) << "Test output:\n" << raw_output_;
266 if (!expected_output.empty()) {
267 std::string sorted_output = SortTestOutput(sanitized_output_);
268 ASSERT_EQ(expected_output, sorted_output);
269 }
270 }
271
TEST_F(SystemTests,verify_pass)272 TEST_F(SystemTests, verify_pass) {
273 std::string expected =
274 "Note: Google Test filter = *.DISABLED_pass\n"
275 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
276 "[ RUN ] SystemTests.DISABLED_pass\n"
277 "[ OK ] SystemTests.DISABLED_pass (XX ms)\n"
278 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
279 "[ PASSED ] 1 test.\n";
280 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_pass", expected, 0));
281 }
282
TEST_F(SystemTests,verify_pass_no_print_time)283 TEST_F(SystemTests, verify_pass_no_print_time) {
284 std::string expected =
285 "Note: Google Test filter = *.DISABLED_pass\n"
286 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
287 "[ RUN ] SystemTests.DISABLED_pass\n"
288 "[ OK ] SystemTests.DISABLED_pass\n"
289 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
290 "[ PASSED ] 1 test.\n";
291 ASSERT_NO_FATAL_FAILURE(
292 Verify("*.DISABLED_pass", expected, 0, std::vector<const char*>{"--gtest_print_time=0"}));
293 }
294
TEST_F(SystemTests,verify_pass_color)295 TEST_F(SystemTests, verify_pass_color) {
296 std::string expected =
297 "\x1B[0;33mNote: Google Test filter = *.DISABLED_pass\x1B[m\n"
298 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
299 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_pass\n"
300 "\x1B[0;32m[ OK ]\x1B[m SystemTests.DISABLED_pass (XX ms)\n"
301 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
302 "\x1B[0;32m[ PASSED ]\x1B[m 1 test.\n";
303 ASSERT_NO_FATAL_FAILURE(
304 Verify("*.DISABLED_pass", expected, 0, std::vector<const char*>{"--gtest_color=yes"}));
305 }
306
TEST_F(SystemTests,verify_skip)307 TEST_F(SystemTests, verify_skip) {
308 std::string expected =
309 "Note: Google Test filter = *.DISABLED_skip_no_message\n"
310 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
311 "[ RUN ] SystemTests.DISABLED_skip_no_message\n"
312 "file:(XX) Skipped\n"
313 "[ SKIPPED ] SystemTests.DISABLED_skip_no_message (XX ms)\n"
314 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
315 "[ PASSED ] 0 tests.\n"
316 "[ SKIPPED ] 1 test, listed below:\n"
317 "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n";
318 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0));
319 }
320
TEST_F(SystemTests,verify_skip_with_message)321 TEST_F(SystemTests, verify_skip_with_message) {
322 std::string expected =
323 "Note: Google Test filter = *.DISABLED_skip_with_message\n"
324 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
325 "[ RUN ] SystemTests.DISABLED_skip_with_message\n"
326 "file:(XX) Skipped\n"
327 "This is a skip message\n"
328 "[ SKIPPED ] SystemTests.DISABLED_skip_with_message (XX ms)\n"
329 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
330 "[ PASSED ] 0 tests.\n"
331 "[ SKIPPED ] 1 test, listed below:\n"
332 "[ SKIPPED ] SystemTests.DISABLED_skip_with_message\n";
333 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_message", expected, 0));
334 }
335
TEST_F(SystemTests,verify_skip_with_output_before_message)336 TEST_F(SystemTests, verify_skip_with_output_before_message) {
337 std::string expected =
338 "Note: Google Test filter = *.DISABLED_skip_with_output_before\n"
339 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
340 "[ RUN ] SystemTests.DISABLED_skip_with_output_before\n"
341 "This is the message before the skip message\n"
342 "file:(XX) Skipped\n"
343 "This is the skip message\n"
344 "[ SKIPPED ] SystemTests.DISABLED_skip_with_output_before (XX ms)\n"
345 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
346 "[ PASSED ] 0 tests.\n"
347 "[ SKIPPED ] 1 test, listed below:\n"
348 "[ SKIPPED ] SystemTests.DISABLED_skip_with_output_before\n";
349 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_output_before", expected, 0));
350 }
351
TEST_F(SystemTests,verify_skip_with_output_after_message)352 TEST_F(SystemTests, verify_skip_with_output_after_message) {
353 std::string expected =
354 "Note: Google Test filter = *.DISABLED_skip_with_output_after\n"
355 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
356 "[ RUN ] SystemTests.DISABLED_skip_with_output_after\n"
357 "file:(XX) Skipped\n"
358 "This is the skip message\n"
359 "This is the message after the skip message\n"
360 "[ SKIPPED ] SystemTests.DISABLED_skip_with_output_after (XX ms)\n"
361 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
362 "[ PASSED ] 0 tests.\n"
363 "[ SKIPPED ] 1 test, listed below:\n"
364 "[ SKIPPED ] SystemTests.DISABLED_skip_with_output_after\n";
365 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_output_after", expected, 0));
366 }
367
TEST_F(SystemTests,verify_skip_with_skipped_line)368 TEST_F(SystemTests, verify_skip_with_skipped_line) {
369 std::string expected =
370 "Note: Google Test filter = *.DISABLED_skip_with_skipped_line\n"
371 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
372 "[ RUN ] SystemTests.DISABLED_skip_with_skipped_line\n"
373 "\n"
374 "Skipped\n"
375 "file:(XX) Skipped\n"
376 "This is the skip message 1\n"
377 "Skipped\n"
378 "file:(XX) Skipped\n"
379 "This is the skip message 2\n"
380 "Skipped\n"
381 "[ SKIPPED ] SystemTests.DISABLED_skip_with_skipped_line (XX ms)\n"
382 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
383 "[ PASSED ] 0 tests.\n"
384 "[ SKIPPED ] 1 test, listed below:\n"
385 "[ SKIPPED ] SystemTests.DISABLED_skip_with_skipped_line\n";
386 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_with_skipped_line", expected, 0));
387 }
388
TEST_F(SystemTests,verify_skip_multiple)389 TEST_F(SystemTests, verify_skip_multiple) {
390 std::string expected =
391 "Note: Google Test filter = *.DISABLED_skip_multiple\n"
392 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
393 "[ RUN ] SystemTests.DISABLED_skip_multiple\n"
394 "This is not a skip message 1\n"
395 "file:(XX) Skipped\n"
396 "This is the skip message 1\n"
397 "This is not a skip message 2\n"
398 "file:(XX) Skipped\n"
399 "This is the skip message 2\n"
400 "file:(XX) Skipped\n"
401 "This is the skip message 3\n"
402 "This is not a skip message 4\n"
403 "[ SKIPPED ] SystemTests.DISABLED_skip_multiple (XX ms)\n"
404 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
405 "[ PASSED ] 0 tests.\n"
406 "[ SKIPPED ] 1 test, listed below:\n"
407 "[ SKIPPED ] SystemTests.DISABLED_skip_multiple\n";
408 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_multiple", expected, 0));
409 }
410
TEST_F(SystemTests,verify_skip_no_print_time)411 TEST_F(SystemTests, verify_skip_no_print_time) {
412 std::string expected =
413 "Note: Google Test filter = *.DISABLED_skip_no_message\n"
414 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
415 "[ RUN ] SystemTests.DISABLED_skip_no_message\n"
416 "file:(XX) Skipped\n"
417 "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n"
418 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
419 "[ PASSED ] 0 tests.\n"
420 "[ SKIPPED ] 1 test, listed below:\n"
421 "[ SKIPPED ] SystemTests.DISABLED_skip_no_message\n";
422 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0,
423 std::vector<const char*>{"--gtest_print_time=0"}));
424 }
425
TEST_F(SystemTests,verify_skip_color)426 TEST_F(SystemTests, verify_skip_color) {
427 std::string expected =
428 "\x1B[0;33mNote: Google Test filter = *.DISABLED_skip_no_message\x1B[m\n"
429 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
430 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_skip_no_message\n"
431 "file:(XX) Skipped\n"
432 "\x1B[0;32m[ SKIPPED ]\x1B[m SystemTests.DISABLED_skip_no_message (XX ms)\n"
433 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
434 "\x1B[0;32m[ PASSED ]\x1B[m 0 tests.\n"
435 "\x1B[0;32m[ SKIPPED ]\x1B[m 1 test, listed below:\n"
436 "\x1B[0;32m[ SKIPPED ]\x1B[m SystemTests.DISABLED_skip_no_message\n";
437 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_skip_no_message", expected, 0,
438 std::vector<const char*>{"--gtest_color=yes"}));
439 }
440
TEST_F(SystemTests,verify_xfail_fail_expect_to_fail)441 TEST_F(SystemTests, verify_xfail_fail_expect_to_fail) {
442 std::string expected =
443 "Note: Google Test filter = *.xfail_fail\n"
444 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
445 "[ RUN ] DISABLED_SystemTestsXfail.xfail_fail\n"
446 "file:(XX) Failure in test DISABLED_SystemTestsXfail.xfail_fail\n"
447 "Expected equality of these values:\n"
448 " 1\n"
449 " 0\n"
450 "DISABLED_SystemTestsXfail.xfail_fail exited with exitcode 1.\n"
451 "[ OK ] DISABLED_SystemTestsXfail.xfail_fail (XX ms)\n"
452 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
453 "[ PASSED ] 1 test. (1 expected failure)\n";
454 ASSERT_NO_FATAL_FAILURE(Verify("*.xfail_fail", expected, 0));
455 }
456
TEST_F(SystemTests,verify_xfail_pass_expect_to_fail)457 TEST_F(SystemTests, verify_xfail_pass_expect_to_fail) {
458 std::string expected =
459 "Note: Google Test filter = *.xfail_pass\n"
460 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
461 "[ RUN ] DISABLED_SystemTestsXfail.xfail_pass\n"
462 "[ FAILED ] DISABLED_SystemTestsXfail.xfail_pass (XX ms)\n"
463 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
464 "[ PASSED ] 0 tests.\n"
465 "[ FAILED ] 1 test should have failed, listed below:\n"
466 "[ FAILED ] DISABLED_SystemTestsXfail.xfail_pass\n"
467 "\n"
468 " 1 SHOULD HAVE FAILED TEST\n";
469 ASSERT_NO_FATAL_FAILURE(Verify("*.xfail_pass", expected, 1));
470 }
471
TEST_F(SystemTests,verify_xfail_pass_expect_to_fail_color)472 TEST_F(SystemTests, verify_xfail_pass_expect_to_fail_color) {
473 std::string expected =
474 "\x1B[0;33mNote: Google Test filter = *.xfail_pass\x1B[m\n"
475 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
476 "\x1B[0;32m[ RUN ]\x1B[m DISABLED_SystemTestsXfail.xfail_pass\n"
477 "\x1B[0;31m[ FAILED ]\x1B[m DISABLED_SystemTestsXfail.xfail_pass (XX ms)\n"
478 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
479 "\x1B[0;32m[ PASSED ]\x1B[m 0 tests.\n"
480 "\x1B[0;31m[ FAILED ]\x1B[m 1 test should have failed, listed below:\n"
481 "\x1B[0;31m[ FAILED ]\x1B[m DISABLED_SystemTestsXfail.xfail_pass\n"
482 "\n"
483 " 1 SHOULD HAVE FAILED TEST\n";
484 ASSERT_NO_FATAL_FAILURE(
485 Verify("*.xfail_pass", expected, 1, std::vector<const char*>{"--gtest_color=yes"}));
486 }
487
TEST_F(SystemTests,verify_deathtest_pass)488 TEST_F(SystemTests, verify_deathtest_pass) {
489 std::string expected =
490 "Note: Google Test filter = *.DISABLED_death_pass\n"
491 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
492 "[ RUN ] SystemTestsDeathTest.DISABLED_death_pass\n"
493 "[ OK ] SystemTestsDeathTest.DISABLED_death_pass (XX ms)\n"
494 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
495 "[ PASSED ] 1 test.\n";
496 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_death_pass", expected, 0));
497 }
498
TEST_F(SystemTests,verify_fail)499 TEST_F(SystemTests, verify_fail) {
500 std::string expected =
501 "Note: Google Test filter = *.DISABLED_fail\n"
502 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
503 "[ RUN ] SystemTests.DISABLED_fail\n"
504 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
505 "Expected equality of these values:\n"
506 " 1\n"
507 " 0\n"
508 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
509 "[ FAILED ] SystemTests.DISABLED_fail (XX ms)\n"
510 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
511 "[ PASSED ] 0 tests.\n"
512 "[ FAILED ] 1 test, listed below:\n"
513 "[ FAILED ] SystemTests.DISABLED_fail\n"
514 "\n"
515 " 1 FAILED TEST\n";
516 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_fail", expected, 1));
517 }
518
TEST_F(SystemTests,verify_fail_color)519 TEST_F(SystemTests, verify_fail_color) {
520 std::string expected =
521 "\x1B[0;33mNote: Google Test filter = *.DISABLED_fail\x1B[m\n"
522 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
523 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_fail\n"
524 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
525 "Expected equality of these values:\n"
526 " 1\n"
527 " 0\n"
528 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
529 "\x1B[0;31m[ FAILED ]\x1B[m SystemTests.DISABLED_fail (XX ms)\n"
530 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
531 "\x1B[0;32m[ PASSED ]\x1B[m 0 tests.\n"
532 "\x1B[0;31m[ FAILED ]\x1B[m 1 test, listed below:\n"
533 "\x1B[0;31m[ FAILED ]\x1B[m SystemTests.DISABLED_fail\n"
534 "\n"
535 " 1 FAILED TEST\n";
536 ASSERT_NO_FATAL_FAILURE(
537 Verify("*.DISABLED_fail", expected, 1, std::vector<const char*>{"--gtest_color=yes"}));
538 }
539
TEST_F(SystemTests,verify_deathtest_fail)540 TEST_F(SystemTests, verify_deathtest_fail) {
541 std::string expected =
542 "Note: Google Test filter = *.DISABLED_death_fail\n"
543 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
544 "[ RUN ] SystemTestsDeathTest.DISABLED_death_fail\n"
545 "file:(XX) Failure in test SystemTestsDeathTest.DISABLED_death_fail\n"
546 "Death test: DeathTestHelperFail()\n"
547 " Result: failed to die.\n"
548 " Error msg:\n"
549 "[ DEATH ] \n"
550 "SystemTestsDeathTest.DISABLED_death_fail exited with exitcode 1.\n"
551 "[ FAILED ] SystemTestsDeathTest.DISABLED_death_fail (XX ms)\n"
552 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
553 "[ PASSED ] 0 tests.\n"
554 "[ FAILED ] 1 test, listed below:\n"
555 "[ FAILED ] SystemTestsDeathTest.DISABLED_death_fail\n"
556 "\n"
557 " 1 FAILED TEST\n";
558 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_death_fail", expected, 1));
559 }
560
TEST_F(SystemTests,verify_crash)561 TEST_F(SystemTests, verify_crash) {
562 std::string expected =
563 "Note: Google Test filter = *.DISABLED_crash\n"
564 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
565 "[ RUN ] SystemTests.DISABLED_crash\n"
566 "SystemTests.DISABLED_crash terminated by signal: XXX\n"
567 "[ FAILED ] SystemTests.DISABLED_crash (XX ms)\n"
568 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
569 "[ PASSED ] 0 tests.\n"
570 "[ FAILED ] 1 test, listed below:\n"
571 "[ FAILED ] SystemTests.DISABLED_crash\n"
572 "\n"
573 " 1 FAILED TEST\n";
574 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_crash", expected, 1));
575 }
576
TEST_F(SystemTests,verify_warning_slow)577 TEST_F(SystemTests, verify_warning_slow) {
578 std::string expected =
579 "Note: Google Test filter = *.DISABLED_sleep5\n"
580 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
581 "[ RUN ] SystemTests.DISABLED_sleep5\n"
582 "[ OK ] SystemTests.DISABLED_sleep5 (XX ms)\n"
583 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
584 "[ PASSED ] 1 test.\n"
585 "[ SLOW ] 1 test, listed below:\n"
586 "[ SLOW ] SystemTests.DISABLED_sleep5 (XX ms, exceeded 3000 ms)\n"
587 "\n"
588 " 1 SLOW TEST\n";
589 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_sleep5", expected, 0,
590 std::vector<const char*>{"--slow_threshold_ms=3000"}));
591 }
592
TEST_F(SystemTests,verify_warning_slow_color)593 TEST_F(SystemTests, verify_warning_slow_color) {
594 std::string expected =
595 "\x1B[0;33mNote: Google Test filter = *.DISABLED_sleep5\x1B[m\n"
596 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
597 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_sleep5\n"
598 "\x1B[0;32m[ OK ]\x1B[m SystemTests.DISABLED_sleep5 (XX ms)\n"
599 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
600 "\x1B[0;32m[ PASSED ]\x1B[m 1 test.\n"
601 "\x1B[0;33m[ SLOW ]\x1B[m 1 test, listed below:\n"
602 "\x1B[0;33m[ SLOW ]\x1B[m SystemTests.DISABLED_sleep5 (XX ms, exceeded 3000 ms)\n"
603 "\n"
604 " 1 SLOW TEST\n";
605 ASSERT_NO_FATAL_FAILURE(
606 Verify("*.DISABLED_sleep5", expected, 0,
607 std::vector<const char*>{"--slow_threshold_ms=3000", "--gtest_color=yes"}));
608 }
609
TEST_F(SystemTests,verify_timeout)610 TEST_F(SystemTests, verify_timeout) {
611 std::string expected =
612 "Note: Google Test filter = *.DISABLED_sleep_forever\n"
613 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
614 "[ RUN ] SystemTests.DISABLED_sleep_forever\n"
615 "SystemTests.DISABLED_sleep_forever killed because of timeout at XX ms.\n"
616 "[ FAILED ] SystemTests.DISABLED_sleep_forever (XX ms)\n"
617 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
618 "[ PASSED ] 0 tests.\n"
619 "[ TIMEOUT ] 1 test, listed below:\n"
620 "[ TIMEOUT ] SystemTests.DISABLED_sleep_forever (stopped at XX ms)\n"
621 "\n"
622 " 1 TIMEOUT TEST\n";
623 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_sleep_forever", expected, 1,
624 std::vector<const char*>{"--deadline_threshold_ms=3000"}));
625 }
626
627 // Verify that tests that timeout do not get marked as slow too when
628 // another test is marked as slow.
TEST_F(SystemTests,verify_timeout_not_slow)629 TEST_F(SystemTests, verify_timeout_not_slow) {
630 std::string expected =
631 "Note: Google Test filter = *.DISABLED_sleep*\n"
632 "[==========] Running 2 tests from 1 test suite (20 jobs).\n"
633 "[ RUN ] SystemTests.DISABLED_sleep5\n"
634 "[ OK ] SystemTests.DISABLED_sleep5 (XX ms)\n"
635 "[ RUN ] SystemTests.DISABLED_sleep_forever\n"
636 "SystemTests.DISABLED_sleep_forever killed because of timeout at XX ms.\n"
637 "[ FAILED ] SystemTests.DISABLED_sleep_forever (XX ms)\n"
638 "[==========] 2 tests from 1 test suite ran. (XX ms total)\n"
639 "[ PASSED ] 1 test.\n"
640 "[ SLOW ] 1 test, listed below:\n"
641 "[ SLOW ] SystemTests.DISABLED_sleep5 (XX ms, exceeded 1000 ms)\n"
642 "[ TIMEOUT ] 1 test, listed below:\n"
643 "[ TIMEOUT ] SystemTests.DISABLED_sleep_forever (stopped at XX ms)\n"
644 "\n"
645 " 1 SLOW TEST\n"
646 " 1 TIMEOUT TEST\n";
647 ASSERT_NO_FATAL_FAILURE(Verify(
648 "*.DISABLED_sleep*", expected, 1,
649 std::vector<const char*>{"--slow_threshold_ms=1000", "--deadline_threshold_ms=10000"}));
650 }
651
TEST_F(SystemTests,verify_timeout_color)652 TEST_F(SystemTests, verify_timeout_color) {
653 std::string expected =
654 "\x1B[0;33mNote: Google Test filter = *.DISABLED_sleep_forever\x1B[m\n"
655 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (20 jobs).\n"
656 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.DISABLED_sleep_forever\n"
657 "SystemTests.DISABLED_sleep_forever killed because of timeout at XX ms.\n"
658 "\x1B[0;31m[ FAILED ]\x1B[m SystemTests.DISABLED_sleep_forever (XX ms)\n"
659 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
660 "\x1B[0;32m[ PASSED ]\x1B[m 0 tests.\n"
661 "\x1B[0;31m[ TIMEOUT ]\x1B[m 1 test, listed below:\n"
662 "\x1B[0;31m[ TIMEOUT ]\x1B[m SystemTests.DISABLED_sleep_forever (stopped at XX ms)\n"
663 "\n"
664 " 1 TIMEOUT TEST\n";
665 ASSERT_NO_FATAL_FAILURE(
666 Verify("*.DISABLED_sleep_forever", expected, 1,
667 std::vector<const char*>{"--deadline_threshold_ms=3000", "--gtest_color=yes"}));
668 }
669
TEST_F(SystemTests,verify_order_isolated)670 TEST_F(SystemTests, verify_order_isolated) {
671 std::string expected =
672 "Note: Google Test filter = *.DISABLED_order_*\n"
673 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
674 "[ RUN ] SystemTests.DISABLED_order_3\n"
675 "[ OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
676 "[ RUN ] SystemTests.DISABLED_order_2\n"
677 "[ OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
678 "[ RUN ] SystemTests.DISABLED_order_1\n"
679 "[ OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
680 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
681 "[ PASSED ] 3 tests.\n";
682 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_order_*", expected, 0));
683 }
684
TEST_F(SystemTests,verify_order_not_isolated)685 TEST_F(SystemTests, verify_order_not_isolated) {
686 std::string expected =
687 "Note: Google Test filter = *.DISABLED_order_*\n"
688 "[==========] Running 3 tests from 1 test suite.\n"
689 "[----------] Global test environment set-up.\n"
690 "[----------] 3 tests from SystemTests\n"
691 "[ RUN ] SystemTests.DISABLED_order_1\n"
692 "[ OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
693 "[ RUN ] SystemTests.DISABLED_order_2\n"
694 "[ OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
695 "[ RUN ] SystemTests.DISABLED_order_3\n"
696 "[ OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
697 "[----------] 3 tests from SystemTests (XX ms total)\n"
698 "\n"
699 "[----------] Global test environment tear-down\n"
700 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
701 "[ PASSED ] 3 tests.\n";
702 ASSERT_NO_FATAL_FAILURE(
703 Verify("*.DISABLED_order_*", expected, 0, std::vector<const char*>{"--no_isolate"}));
704 }
705
TEST_F(SystemTests,verify_fail_ge10)706 TEST_F(SystemTests, verify_fail_ge10) {
707 ASSERT_NO_FATAL_FAILURE(RunTest("*.DISABLED_fail_*"));
708 // Verify the failed output at the end has no space in front.
709 std::regex regex("\\n.*\\d+ FAILED TESTS\\n");
710 std::cmatch match;
711 ASSERT_TRUE(std::regex_search(sanitized_output_.c_str(), match, regex)) << "Test Output:\n"
712 << raw_output_;
713 ASSERT_EQ("\n10 FAILED TESTS\n", match[0]);
714 ASSERT_NE(0, exitcode_);
715 }
716
TEST_F(SystemTests,verify_title_order)717 TEST_F(SystemTests, verify_title_order) {
718 std::string footer;
719 ASSERT_NO_FATAL_FAILURE(RunTestCaptureFooter(
720 "*.DISABLED_all_*", &footer,
721 std::vector<const char*>{"--slow_threshold_ms=2000", "--deadline_threshold_ms=4000"}));
722
723 ASSERT_EQ(
724 "[ PASSED ] 4 tests.\n"
725 "[ SKIPPED ] 2 tests, listed below:\n"
726 "[ SKIPPED ] SystemTests.DISABLED_all_skip_1\n"
727 "[ SKIPPED ] SystemTests.DISABLED_all_skip_2\n"
728 "[ SLOW ] 2 tests, listed below:\n"
729 "[ SLOW ] SystemTests.DISABLED_all_slow_1 (XX ms, exceeded 2000 ms)\n"
730 "[ SLOW ] SystemTests.DISABLED_all_slow_2 (XX ms, exceeded 2000 ms)\n"
731 "[ TIMEOUT ] 2 tests, listed below:\n"
732 "[ TIMEOUT ] SystemTests.DISABLED_all_timeout_1 (stopped at XX ms)\n"
733 "[ TIMEOUT ] SystemTests.DISABLED_all_timeout_2 (stopped at XX ms)\n"
734 "[ FAILED ] 2 tests, listed below:\n"
735 "[ FAILED ] SystemTests.DISABLED_all_fail_1\n"
736 "[ FAILED ] SystemTests.DISABLED_all_fail_2\n"
737 "\n"
738 " 2 SLOW TESTS\n"
739 " 2 TIMEOUT TESTS\n"
740 " 2 FAILED TESTS\n",
741 footer);
742 }
743
TEST_F(SystemTests,verify_job_count_single)744 TEST_F(SystemTests, verify_job_count_single) {
745 std::string expected =
746 "Note: Google Test filter = *.DISABLED_job_*\n"
747 "[==========] Running 3 tests from 1 test suite (1 job).\n"
748 "[ RUN ] SystemTests.DISABLED_job_1\n"
749 "[ OK ] SystemTests.DISABLED_job_1 (XX ms)\n"
750 "[ RUN ] SystemTests.DISABLED_job_2\n"
751 "[ OK ] SystemTests.DISABLED_job_2 (XX ms)\n"
752 "[ RUN ] SystemTests.DISABLED_job_3\n"
753 "[ OK ] SystemTests.DISABLED_job_3 (XX ms)\n"
754 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
755 "[ PASSED ] 3 tests.\n";
756 ASSERT_NO_FATAL_FAILURE(Verify("*.DISABLED_job_*", expected, 0, std::vector<const char*>{"-j1"}));
757 }
758
TEST_F(SystemTests,verify_job_count_multiple)759 TEST_F(SystemTests, verify_job_count_multiple) {
760 std::string expected =
761 "Note: Google Test filter = *.DISABLED_job_*\n"
762 "[==========] Running 3 tests from 1 test suite (2 jobs).\n"
763 "[ RUN ] SystemTests.DISABLED_job_2\n"
764 "[ OK ] SystemTests.DISABLED_job_2 (XX ms)\n"
765 "[ RUN ] SystemTests.DISABLED_job_1\n"
766 "[ OK ] SystemTests.DISABLED_job_1 (XX ms)\n"
767 "[ RUN ] SystemTests.DISABLED_job_3\n"
768 "[ OK ] SystemTests.DISABLED_job_3 (XX ms)\n"
769 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
770 "[ PASSED ] 3 tests.\n";
771 ASSERT_NO_FATAL_FAILURE(
772 Verify("*.DISABLED_job_*", expected, 0, std::vector<const char*>{"-j", "2"}));
773 }
774
TEST_F(SystemTests,verify_help)775 TEST_F(SystemTests, verify_help) {
776 // This tests verifies that the help options display the help for
777 // the isolated test run, and for the gtest data.
778 std::vector<const char*> help_args{"-h", "--help"};
779 for (auto arg : help_args) {
780 ASSERT_NO_FATAL_FAILURE(RunTest("*.DISABLED_pass", std::vector<const char*>{arg}));
781 ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
782 // First find something from the isolation help.
783 std::size_t isolation_help = sanitized_output_.find("In isolation mode,");
784 ASSERT_NE(std::string::npos, isolation_help) << "Cannot find isolation help:\n" << raw_output_;
785 std::size_t gtest_help = sanitized_output_.find("Assertion Behavior:");
786 ASSERT_NE(std::string::npos, gtest_help) << "Cannot find gtest help:\n" << raw_output_;
787
788 ASSERT_GT(gtest_help, isolation_help) << "Gtest help before isolation help:\n" << raw_output_;
789 }
790 }
791
TEST_F(SystemTests,verify_help_color)792 TEST_F(SystemTests, verify_help_color) {
793 // Verify that the color option does change the help display.
794 std::vector<const char*> help_args{"-h", "--help"};
795 for (auto arg : help_args) {
796 ASSERT_NO_FATAL_FAILURE(
797 RunTest("*.DISABLED_pass", std::vector<const char*>{arg, "--gtest_color=yes"}));
798 ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
799 // First find something from the isolation help that is in color.
800 std::size_t isolation_help =
801 sanitized_output_.find("Unit Test Options:\n\x1B[0;32m -j \x1B[m");
802 ASSERT_NE(std::string::npos, isolation_help) << "Cannot find isolation help:\n" << raw_output_;
803 std::size_t gtest_help = sanitized_output_.find("\x1B[0;32m--gtest_list_tests\x1B[m");
804 ASSERT_NE(std::string::npos, gtest_help) << "Cannot find gtest help:\n" << raw_output_;
805
806 ASSERT_GT(gtest_help, isolation_help) << "Gtest help before isolation help:\n" << raw_output_;
807 }
808 }
809
TEST_F(SystemTests,verify_repeat)810 TEST_F(SystemTests, verify_repeat) {
811 std::string expected =
812 "Note: Google Test filter = *.DISABLED_order_*\n"
813 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
814 "[ RUN ] SystemTests.DISABLED_order_3\n"
815 "[ OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
816 "[ RUN ] SystemTests.DISABLED_order_2\n"
817 "[ OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
818 "[ RUN ] SystemTests.DISABLED_order_1\n"
819 "[ OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
820 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
821 "[ PASSED ] 3 tests.\n"
822 "\n"
823 "Repeating all tests (iteration 2) . . .\n"
824 "\n"
825 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
826 "[ RUN ] SystemTests.DISABLED_order_3\n"
827 "[ OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
828 "[ RUN ] SystemTests.DISABLED_order_2\n"
829 "[ OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
830 "[ RUN ] SystemTests.DISABLED_order_1\n"
831 "[ OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
832 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
833 "[ PASSED ] 3 tests.\n"
834 "\n"
835 "Repeating all tests (iteration 3) . . .\n"
836 "\n"
837 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
838 "[ RUN ] SystemTests.DISABLED_order_3\n"
839 "[ OK ] SystemTests.DISABLED_order_3 (XX ms)\n"
840 "[ RUN ] SystemTests.DISABLED_order_2\n"
841 "[ OK ] SystemTests.DISABLED_order_2 (XX ms)\n"
842 "[ RUN ] SystemTests.DISABLED_order_1\n"
843 "[ OK ] SystemTests.DISABLED_order_1 (XX ms)\n"
844 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
845 "[ PASSED ] 3 tests.\n";
846 uint64_t time_ns = NanoTime();
847 ASSERT_NO_FATAL_FAILURE(
848 Verify("*.DISABLED_order_*", expected, 0, std::vector<const char*>{"--gtest_repeat=3"}));
849 time_ns = NanoTime() - time_ns;
850 // Make sure that the total test time is about 18 seconds.
851 double seconds = double(time_ns) / 1000000000;
852 ASSERT_LE(18.0, seconds) << "Repeat test should take at least 18 seconds.\n"
853 << "Test output:\n"
854 << raw_output_;
855 ASSERT_GT(20.0, seconds) << "Repeat test should take about 18 seconds.\n"
856 << "Test output:\n"
857 << raw_output_;
858 }
859
TEST_F(SystemTests,verify_results_as_tests_finish)860 TEST_F(SystemTests, verify_results_as_tests_finish) {
861 // This test verifies that test output comes out as the test finishes.
862 Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_order_*",
863 "--gtest_also_run_disabled_tests", "-j20"});
864
865 std::string output;
866 std::vector<char> buffer(4096);
867 uint64_t time_ns = NanoTime();
868 while (true) {
869 ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
870 if (bytes == -1 && errno == EAGAIN) {
871 continue;
872 }
873 ASSERT_NE(-1, bytes);
874 ASSERT_NE(0, bytes) << "Did not find test output before test finished:\n" << output;
875 buffer[bytes] = '\0';
876 output += buffer.data();
877 // See if the output has come out now.
878 if (output.find("[ OK ] SystemTests.DISABLED_order_2") != std::string::npos) {
879 uint64_t test_ns = NanoTime() - time_ns;
880 double test_sec = double(test_ns) / 1000000000;
881 // This should happen after 3 seconds, but before 4.5 seconds.
882 ASSERT_LE(3.0, test_sec) << "Test output:\n" << output;
883 ASSERT_GT(4.5, test_sec) << "Test output:\n" << output;
884 break;
885 }
886 }
887
888 // Read the rest of the output.
889 while (true) {
890 ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
891 if (bytes == -1 && errno == EAGAIN) {
892 continue;
893 }
894 ASSERT_NE(-1, bytes);
895 if (bytes == 0) {
896 break;
897 }
898 buffer[bytes] = '\0';
899 output += buffer.data();
900 }
901 close(fd_);
902 time_ns = NanoTime() - time_ns;
903 ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, nullptr, 0))) << "Test output:\n" << output;
904 // Verify that the total test time is > 6 seconds.
905 ASSERT_LE(6.0, double(time_ns) / 1000000000) << "Test output:\n" << output;
906 }
907
TEST_F(SystemTests,verify_xml)908 TEST_F(SystemTests, verify_xml) {
909 std::string tmp_arg("--gtest_output=xml:");
910 TemporaryFile tf;
911 ASSERT_TRUE(tf.fd != -1);
912 close(tf.fd);
913 tmp_arg += tf.path;
914
915 ASSERT_NO_FATAL_FAILURE(RunTest("*.DISABLED_xml_*", std::vector<const char*>{tmp_arg.c_str()}));
916 ASSERT_EQ(1, exitcode_) << "Test output:\n" << raw_output_;
917
918 // Check that the xml file exists.
919 FILE* xml_file = fopen(tf.path, "r");
920 ASSERT_TRUE(xml_file != nullptr) << "Failed to find xml file:\n" << raw_output_;
921 // Read the entire file in.
922 std::string xml_output;
923 std::vector<char> buffer(4096);
924 size_t bytes;
925 while ((bytes = fread(buffer.data(), 1, buffer.size(), xml_file)) > 0) {
926 xml_output += std::string(buffer.data(), bytes);
927 }
928 fclose(xml_file);
929 unlink(tf.path);
930
931 // Change time|timestamp="" to time|timestamp="XX"
932 xml_output =
933 std::regex_replace(xml_output, std::regex("(time|timestamp)=\"[^\"]+\""), "$1=\"XX\"");
934 // Change ".*.cc:(XX) to "file:(XX)
935 xml_output = std::regex_replace(xml_output, std::regex("\"([^/\\s]+/)*[^/\\s]+:\\(\\d+\\)\\s"),
936 "\"file:(XX) ");
937
938 std::string expected =
939 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
940 "<testsuites tests=\"6\" failures=\"3\" disabled=\"0\" errors=\"0\" timestamp=\"XX\" "
941 "time=\"XX\" name=\"AllTests\">\n"
942 " <testsuite name=\"SystemTestsXml1\" tests=\"2\" failures=\"1\" disabled=\"0\" "
943 "errors=\"0\" time=\"XX\">\n"
944 " <testcase name=\"DISABLED_xml_1\" status=\"run\" time=\"XX\" "
945 "classname=\"SystemTestsXml1\" />\n"
946 " <testcase name=\"DISABLED_xml_2\" status=\"run\" time=\"XX\" "
947 "classname=\"SystemTestsXml1\">\n"
948 " <failure message=\"file:(XX) Failure in test SystemTestsXml1.DISABLED_xml_2\n"
949 "Expected equality of these values:\n"
950 " 1\n"
951 " 0\n"
952 "SystemTestsXml1.DISABLED_xml_2 exited with exitcode 1.\n"
953 "\" type=\"\">\n"
954 " </failure>\n"
955 " </testcase>\n"
956 " </testsuite>\n"
957 " <testsuite name=\"SystemTestsXml2\" tests=\"2\" failures=\"1\" disabled=\"0\" "
958 "errors=\"0\" time=\"XX\">\n"
959 " <testcase name=\"DISABLED_xml_1\" status=\"run\" time=\"XX\" "
960 "classname=\"SystemTestsXml2\">\n"
961 " <failure message=\"file:(XX) Failure in test SystemTestsXml2.DISABLED_xml_1\n"
962 "Expected equality of these values:\n"
963 " 1\n"
964 " 0\n"
965 "SystemTestsXml2.DISABLED_xml_1 exited with exitcode 1.\n"
966 "\" type=\"\">\n"
967 " </failure>\n"
968 " </testcase>\n"
969 " <testcase name=\"DISABLED_xml_2\" status=\"run\" time=\"XX\" "
970 "classname=\"SystemTestsXml2\" />\n"
971 " </testsuite>\n"
972 " <testsuite name=\"SystemTestsXml3\" tests=\"2\" failures=\"1\" disabled=\"0\" "
973 "errors=\"0\" time=\"XX\">\n"
974 " <testcase name=\"DISABLED_xml_1\" status=\"run\" time=\"XX\" "
975 "classname=\"SystemTestsXml3\" />\n"
976 " <testcase name=\"DISABLED_xml_2\" status=\"run\" time=\"XX\" "
977 "classname=\"SystemTestsXml3\">\n"
978 " <failure message=\"file:(XX) Failure in test SystemTestsXml3.DISABLED_xml_2\n"
979 "Expected equality of these values:\n"
980 " 1\n"
981 " 0\n"
982 "SystemTestsXml3.DISABLED_xml_2 exited with exitcode 1.\n"
983 "\" type=\"\">\n"
984 " </failure>\n"
985 " </testcase>\n"
986 " </testsuite>\n"
987 "</testsuites>\n";
988 ASSERT_EQ(expected, xml_output);
989 }
990
TEST_F(SystemTests,verify_disabled_not_displayed_with_no_tests)991 TEST_F(SystemTests, verify_disabled_not_displayed_with_no_tests) {
992 std::vector<const char*> args{"--gtest_filter=NO_TEST_FILTER_MATCH", "-j2"};
993
994 ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
995 ASSERT_EQ(0, exitcode_);
996 std::string expected =
997 "Note: Google Test filter = NO_TEST_FILTER_MATCH\n"
998 "[==========] Running 0 tests from 0 test suites (2 jobs).\n"
999 "[==========] 0 tests from 0 test suites ran. (XX ms total)\n"
1000 "[ PASSED ] 0 tests.\n";
1001 ASSERT_EQ(expected, sanitized_output_) << "Test output:\n" << raw_output_;
1002 }
1003
TEST_F(SystemTests,verify_disabled)1004 TEST_F(SystemTests, verify_disabled) {
1005 std::vector<const char*> args{"--gtest_filter=*always_pass", "-j2"};
1006
1007 ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
1008 ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
1009 std::string expected =
1010 "Note: Google Test filter = *always_pass\n"
1011 "[==========] Running 1 test from 1 test suite (2 jobs).\n"
1012 "[ RUN ] SystemTests.always_pass\n"
1013 "[ OK ] SystemTests.always_pass (XX ms)\n"
1014 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1015 "[ PASSED ] 1 test.\n"
1016 "\n"
1017 " YOU HAVE 1 DISABLED TEST\n"
1018 "\n";
1019 ASSERT_EQ(expected, sanitized_output_);
1020 }
1021
TEST_F(SystemTests,verify_disabled_color)1022 TEST_F(SystemTests, verify_disabled_color) {
1023 std::vector<const char*> args{"--gtest_filter=*always_pass", "-j2", "--gtest_color=yes"};
1024
1025 ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
1026 ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
1027 std::string expected =
1028 "\x1B[0;33mNote: Google Test filter = *always_pass\x1B[m\n"
1029 "\x1B[0;32m[==========]\x1B[m Running 1 test from 1 test suite (2 jobs).\n"
1030 "\x1B[0;32m[ RUN ]\x1B[m SystemTests.always_pass\n"
1031 "\x1B[0;32m[ OK ]\x1B[m SystemTests.always_pass (XX ms)\n"
1032 "\x1B[0;32m[==========]\x1B[m 1 test from 1 test suite ran. (XX ms total)\n"
1033 "\x1B[0;32m[ PASSED ]\x1B[m 1 test.\n"
1034 "\n"
1035 "\x1B[0;33m YOU HAVE 1 DISABLED TEST\n"
1036 "\n\x1B[m";
1037 ASSERT_EQ(expected, sanitized_output_);
1038 }
1039
TEST_F(SystemTests,verify_SIGINT)1040 TEST_F(SystemTests, verify_SIGINT) {
1041 // Verify that SIGINT kills all of the tests.
1042 Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_job*", "--gtest_also_run_disabled_tests",
1043 "-j20"});
1044 // It is expected that all of the tests will be sleeping so nothing will
1045 // complete by the time the signal is sent.
1046 sleep(1);
1047 ASSERT_NE(-1, kill(pid_, SIGINT));
1048
1049 std::string output;
1050 std::vector<char> buffer(4096);
1051 while (true) {
1052 ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
1053 if (bytes == -1 && errno == EAGAIN) {
1054 continue;
1055 }
1056 ASSERT_NE(-1, bytes);
1057 if (bytes == 0) {
1058 break;
1059 }
1060 buffer[bytes] = '\0';
1061 output += buffer.data();
1062 }
1063 close(fd_);
1064 int status;
1065 ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << output;
1066 ASSERT_EQ(
1067 "Note: Google Test filter = *.DISABLED_job*\n"
1068 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
1069 "Terminating due to signal...\n",
1070 output);
1071 ASSERT_EQ(1, WEXITSTATUS(status));
1072 }
1073
TEST_F(SystemTests,verify_SIGQUIT)1074 TEST_F(SystemTests, verify_SIGQUIT) {
1075 // Verify that SIGQUIT prints all of the running tests.
1076 Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_job*", "--gtest_also_run_disabled_tests",
1077 "-j20"});
1078 // It is expected that all of the tests will be sleeping so nothing will
1079 // complete by the time the signal is sent.
1080 sleep(1);
1081 ASSERT_NE(-1, kill(pid_, SIGQUIT));
1082
1083 std::vector<char> buffer(4096);
1084 while (true) {
1085 ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
1086 if (bytes == -1 && errno == EAGAIN) {
1087 continue;
1088 }
1089 ASSERT_NE(-1, bytes);
1090 if (bytes == 0) {
1091 break;
1092 }
1093 buffer[bytes] = '\0';
1094 raw_output_ += buffer.data();
1095 }
1096 close(fd_);
1097 int status;
1098 ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << raw_output_;
1099 SanitizeOutput();
1100 ASSERT_EQ(
1101 "Note: Google Test filter = *.DISABLED_job*\n"
1102 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
1103 "List of current running tests:\n"
1104 " SystemTests.DISABLED_job_1 (elapsed time XX ms)\n"
1105 " SystemTests.DISABLED_job_2 (elapsed time XX ms)\n"
1106 " SystemTests.DISABLED_job_3 (elapsed time XX ms)\n"
1107 "[ RUN ] SystemTests.DISABLED_job_2\n"
1108 "[ OK ] SystemTests.DISABLED_job_2 (XX ms)\n"
1109 "[ RUN ] SystemTests.DISABLED_job_3\n"
1110 "[ OK ] SystemTests.DISABLED_job_3 (XX ms)\n"
1111 "[ RUN ] SystemTests.DISABLED_job_1\n"
1112 "[ OK ] SystemTests.DISABLED_job_1 (XX ms)\n"
1113 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
1114 "[ PASSED ] 3 tests.\n",
1115 sanitized_output_);
1116 ASSERT_EQ(0, WEXITSTATUS(status));
1117 }
1118
TEST_F(SystemTests,verify_SIGQUIT_after_test_finish)1119 TEST_F(SystemTests, verify_SIGQUIT_after_test_finish) {
1120 // Verify that SIGQUIT prints all of the tests after a test finishes.
1121 Exec(std::vector<const char*>{"--gtest_filter=*.DISABLED_sigquit_*",
1122 "--gtest_also_run_disabled_tests", "-j20"});
1123 // It is expected that one tests will have finished, but the rest will still
1124 // be running.
1125 sleep(1);
1126 ASSERT_NE(-1, kill(pid_, SIGQUIT));
1127
1128 std::vector<char> buffer(4096);
1129 while (true) {
1130 ssize_t bytes = TEMP_FAILURE_RETRY(read(fd_, buffer.data(), buffer.size() - 1));
1131 if (bytes == -1 && errno == EAGAIN) {
1132 continue;
1133 }
1134 ASSERT_NE(-1, bytes);
1135 if (bytes == 0) {
1136 break;
1137 }
1138 buffer[bytes] = '\0';
1139 raw_output_ += buffer.data();
1140 }
1141 close(fd_);
1142 int status;
1143 ASSERT_EQ(pid_, TEMP_FAILURE_RETRY(waitpid(pid_, &status, 0))) << "Test output:\n" << raw_output_;
1144 SanitizeOutput();
1145 ASSERT_EQ(
1146 "Note: Google Test filter = *.DISABLED_sigquit_*\n"
1147 "[==========] Running 3 tests from 1 test suite (20 jobs).\n"
1148 "[ RUN ] SystemTests.DISABLED_sigquit_no_sleep\n"
1149 "[ OK ] SystemTests.DISABLED_sigquit_no_sleep (XX ms)\n"
1150 "List of current running tests:\n"
1151 " SystemTests.DISABLED_sigquit_sleep_5 (elapsed time XX ms)\n"
1152 " SystemTests.DISABLED_sigquit_sleep_6 (elapsed time XX ms)\n"
1153 "[ RUN ] SystemTests.DISABLED_sigquit_sleep_5\n"
1154 "[ OK ] SystemTests.DISABLED_sigquit_sleep_5 (XX ms)\n"
1155 "[ RUN ] SystemTests.DISABLED_sigquit_sleep_6\n"
1156 "[ OK ] SystemTests.DISABLED_sigquit_sleep_6 (XX ms)\n"
1157 "[==========] 3 tests from 1 test suite ran. (XX ms total)\n"
1158 "[ PASSED ] 3 tests.\n",
1159 sanitized_output_);
1160 ASSERT_EQ(0, WEXITSTATUS(status));
1161 }
1162
TEST_F(SystemTests,verify_memory)1163 TEST_F(SystemTests, verify_memory) {
1164 // This test verifies that memory isn't leaking when running repeatedly.
1165 std::vector<const char*> args{"--gtest_filter=*.DISABLED_memory",
1166 "--gtest_also_run_disabled_tests", "--gtest_repeat=400"};
1167 ASSERT_NO_FATAL_FAILURE(ExecAndCapture(args));
1168 ASSERT_EQ(0, exitcode_) << "Test output:\n" << raw_output_;
1169 std::vector<std::string> lines(android::base::Split(raw_output_, "\n"));
1170
1171 constexpr static size_t kMaxLeakBytes = 32 * 1024;
1172 size_t memory_iteration = 0;
1173 size_t memory_start = 0;
1174 size_t memory_last = 0;
1175 for (auto& line : lines) {
1176 size_t memory;
1177 if (android::base::StartsWith(line, "Allocated ") &&
1178 sscanf(line.c_str(), "Allocated %zu", &memory) == 1) {
1179 if (memory_iteration == 0) {
1180 memory_start = memory;
1181 } else {
1182 // Check the increase from the last loop.
1183 if (memory > memory_last) {
1184 ASSERT_GT(kMaxLeakBytes, memory - memory_last)
1185 << "On iteration " << memory_iteration << " memory increased beyond expected value."
1186 << std::endl
1187 << "Last memory bytes " << memory_last << std::endl
1188 << "Current memory bytes " << memory;
1189 }
1190 // Check the increase from the first loop.
1191 if (memory > memory_start) {
1192 ASSERT_GT(kMaxLeakBytes, memory - memory_start)
1193 << "On iteration " << memory_iteration
1194 << " total memory increased beyond expected value." << std::endl
1195 << "Starting memory bytes " << memory_start << std::endl
1196 << "Current memory bytes " << memory;
1197 }
1198 }
1199 memory_last = memory;
1200 memory_iteration++;
1201 }
1202 }
1203 ASSERT_EQ(400U, memory_iteration)
1204 << "Did not find the expected 400 lines of memory data." << std::endl
1205 << "Raw output:" << std::endl
1206 << raw_output_;
1207 }
1208
TEST_F(SystemTests,verify_sharding)1209 TEST_F(SystemTests, verify_sharding) {
1210 std::string expected =
1211 "Output Sorted\n"
1212 "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1213 "Note: This is test shard 1 of 4\n"
1214 "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1215 "[ RUN ] SystemTestsShard1.DISABLED_case1_test1\n"
1216 "[ OK ] SystemTestsShard1.DISABLED_case1_test1 (XX ms)\n"
1217 "[ RUN ] SystemTestsShard2.DISABLED_case2_test1\n"
1218 "[ OK ] SystemTestsShard2.DISABLED_case2_test1 (XX ms)\n"
1219 "[ RUN ] SystemTestsShard3.DISABLED_case3_test1\n"
1220 "[ OK ] SystemTestsShard3.DISABLED_case3_test1 (XX ms)\n"
1221 "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1222 "[ PASSED ] 3 tests.\n";
1223 ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1224 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "0", 1));
1225 ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1226
1227 expected =
1228 "Output Sorted\n"
1229 "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1230 "Note: This is test shard 2 of 4\n"
1231 "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1232 "[ RUN ] SystemTestsShard1.DISABLED_case1_test2\n"
1233 "[ OK ] SystemTestsShard1.DISABLED_case1_test2 (XX ms)\n"
1234 "[ RUN ] SystemTestsShard2.DISABLED_case2_test2\n"
1235 "[ OK ] SystemTestsShard2.DISABLED_case2_test2 (XX ms)\n"
1236 "[ RUN ] SystemTestsShard3.DISABLED_case3_test2\n"
1237 "[ OK ] SystemTestsShard3.DISABLED_case3_test2 (XX ms)\n"
1238 "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1239 "[ PASSED ] 3 tests.\n";
1240 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "1", 1));
1241 ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1242
1243 expected =
1244 "Output Sorted\n"
1245 "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1246 "Note: This is test shard 3 of 4\n"
1247 "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1248 "[ RUN ] SystemTestsShard1.DISABLED_case1_test3\n"
1249 "[ OK ] SystemTestsShard1.DISABLED_case1_test3 (XX ms)\n"
1250 "[ RUN ] SystemTestsShard2.DISABLED_case2_test3\n"
1251 "[ OK ] SystemTestsShard2.DISABLED_case2_test3 (XX ms)\n"
1252 "[ RUN ] SystemTestsShard3.DISABLED_case3_test3\n"
1253 "[ OK ] SystemTestsShard3.DISABLED_case3_test3 (XX ms)\n"
1254 "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1255 "[ PASSED ] 3 tests.\n";
1256 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "2", 1));
1257 ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1258
1259 expected =
1260 "Output Sorted\n"
1261 "Note: Google Test filter = SystemTestsShard*.DISABLED*\n"
1262 "Note: This is test shard 4 of 4\n"
1263 "[==========] Running 3 tests from 3 test suites (20 jobs).\n"
1264 "[ RUN ] SystemTestsShard1.DISABLED_case1_test4\n"
1265 "[ OK ] SystemTestsShard1.DISABLED_case1_test4 (XX ms)\n"
1266 "[ RUN ] SystemTestsShard2.DISABLED_case2_test4\n"
1267 "[ OK ] SystemTestsShard2.DISABLED_case2_test4 (XX ms)\n"
1268 "[ RUN ] SystemTestsShard3.DISABLED_case3_test4\n"
1269 "[ OK ] SystemTestsShard3.DISABLED_case3_test4 (XX ms)\n"
1270 "[==========] 3 tests from 3 test suites ran. (XX ms total)\n"
1271 "[ PASSED ] 3 tests.\n";
1272 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "3", 1));
1273 ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0));
1274 }
1275
TEST_F(SystemTests,verify_sharding_color)1276 TEST_F(SystemTests, verify_sharding_color) {
1277 std::string expected =
1278 "Output Sorted\n"
1279 "\x1B[0;33mNote: Google Test filter = SystemTestsShard*.DISABLED*\x1B[m\n"
1280 "\x1B[0;33mNote: This is test shard 1 of 4\x1B[m\n"
1281 "\x1B[0;32m[==========]\x1B[m Running 3 tests from 3 test suites (20 jobs).\n"
1282 "\x1B[0;32m[ RUN ]\x1B[m SystemTestsShard1.DISABLED_case1_test1\n"
1283 "\x1B[0;32m[ OK ]\x1B[m SystemTestsShard1.DISABLED_case1_test1 (XX ms)\n"
1284 "\x1B[0;32m[ RUN ]\x1B[m SystemTestsShard2.DISABLED_case2_test1\n"
1285 "\x1B[0;32m[ OK ]\x1B[m SystemTestsShard2.DISABLED_case2_test1 (XX ms)\n"
1286 "\x1B[0;32m[ RUN ]\x1B[m SystemTestsShard3.DISABLED_case3_test1\n"
1287 "\x1B[0;32m[ OK ]\x1B[m SystemTestsShard3.DISABLED_case3_test1 (XX ms)\n"
1288 "\x1B[0;32m[==========]\x1B[m 3 tests from 3 test suites ran. (XX ms total)\n"
1289 "\x1B[0;32m[ PASSED ]\x1B[m 3 tests.\n";
1290 ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1291 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "0", 1));
1292 ASSERT_NO_FATAL_FAILURE(VerifySortedOutput("SystemTestsShard*.DISABLED*", expected, 0,
1293 std::vector<const char*>{"--gtest_color=yes"}));
1294 }
1295
TEST_F(SystemTests,verify_sharding_error)1296 TEST_F(SystemTests, verify_sharding_error) {
1297 std::string expected =
1298 "Invalid environment variables: we require 0 <= GTEST_SHARD_INDEX < GTEST_TOTAL_SHARDS, but "
1299 "you have GTEST_SHARD_INDEX=4, GTEST_TOTAL_SHARDS=4\n";
1300 ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1301 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "4", 1));
1302 ASSERT_NO_FATAL_FAILURE(Verify("SystemTestsShard*.DISABLED*", expected, 1));
1303 }
1304
TEST_F(SystemTests,verify_sharding_error_color)1305 TEST_F(SystemTests, verify_sharding_error_color) {
1306 std::string expected =
1307 "\x1B[0;31mInvalid environment variables: we require 0 <= GTEST_SHARD_INDEX < "
1308 "GTEST_TOTAL_SHARDS, but you have GTEST_SHARD_INDEX=4, GTEST_TOTAL_SHARDS=4\x1B[m\n";
1309 ASSERT_NE(-1, setenv("GTEST_TOTAL_SHARDS", "4", 1));
1310 ASSERT_NE(-1, setenv("GTEST_SHARD_INDEX", "4", 1));
1311 ASSERT_NO_FATAL_FAILURE(Verify("SystemTestsShard*.DISABLED*", expected, 1,
1312 std::vector<const char*>{"--gtest_color=yes"}));
1313 }
1314
TEST_F(SystemTests,verify_gtest_flagfile)1315 TEST_F(SystemTests, verify_gtest_flagfile) {
1316 TemporaryFile tf;
1317 ASSERT_TRUE(android::base::WriteStringToFile("--gtest_print_time=0\n", tf.path));
1318 std::string flagfile("--gtest_flagfile=");
1319 flagfile += tf.path;
1320 std::string expected =
1321 "Note: Google Test filter = *.DISABLED_pass\n"
1322 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1323 "[ RUN ] SystemTests.DISABLED_pass\n"
1324 "[ OK ] SystemTests.DISABLED_pass\n"
1325 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1326 "[ PASSED ] 1 test.\n";
1327 ASSERT_NO_FATAL_FAILURE(
1328 Verify("*.DISABLED_pass", expected, 0, std::vector<const char*>{flagfile.c_str()}));
1329 }
1330
TEST_F(SystemTests,verify_repeat_stop_on_error)1331 TEST_F(SystemTests, verify_repeat_stop_on_error) {
1332 std::string expected =
1333 "Note: Google Test filter = *.DISABLED_fail\n"
1334 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1335 "[ RUN ] SystemTests.DISABLED_fail\n"
1336 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
1337 "Expected equality of these values:\n"
1338 " 1\n"
1339 " 0\n"
1340 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
1341 "[ FAILED ] SystemTests.DISABLED_fail (XX ms)\n"
1342 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1343 "[ PASSED ] 0 tests.\n"
1344 "[ FAILED ] 1 test, listed below:\n"
1345 "[ FAILED ] SystemTests.DISABLED_fail\n"
1346 "\n"
1347 " 1 FAILED TEST\n"
1348 "\n"
1349 "Terminating repeat run due to failing tests (iteration 1).\n";
1350 ASSERT_NO_FATAL_FAILURE(
1351 Verify("*.DISABLED_fail", expected, 1,
1352 std::vector<const char*>{"--gtest_repeat=2", "--gtest_break_on_failure"}));
1353 }
1354
TEST_F(SystemTests,verify_repeat_no_stop_on_error)1355 TEST_F(SystemTests, verify_repeat_no_stop_on_error) {
1356 std::string expected =
1357 "Note: Google Test filter = *.DISABLED_fail\n"
1358 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1359 "[ RUN ] SystemTests.DISABLED_fail\n"
1360 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
1361 "Expected equality of these values:\n"
1362 " 1\n"
1363 " 0\n"
1364 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
1365 "[ FAILED ] SystemTests.DISABLED_fail (XX ms)\n"
1366 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1367 "[ PASSED ] 0 tests.\n"
1368 "[ FAILED ] 1 test, listed below:\n"
1369 "[ FAILED ] SystemTests.DISABLED_fail\n"
1370 "\n"
1371 " 1 FAILED TEST\n"
1372 "\n"
1373 "Repeating all tests (iteration 2) . . .\n"
1374 "\n"
1375 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1376 "[ RUN ] SystemTests.DISABLED_fail\n"
1377 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
1378 "Expected equality of these values:\n"
1379 " 1\n"
1380 " 0\n"
1381 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
1382 "[ FAILED ] SystemTests.DISABLED_fail (XX ms)\n"
1383 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1384 "[ PASSED ] 0 tests.\n"
1385 "[ FAILED ] 1 test, listed below:\n"
1386 "[ FAILED ] SystemTests.DISABLED_fail\n"
1387 "\n"
1388 " 1 FAILED TEST\n";
1389 ASSERT_NO_FATAL_FAILURE(
1390 Verify("*.DISABLED_fail", expected, 1, std::vector<const char*>{"--gtest_repeat=2"}));
1391 }
1392
TEST_F(SystemTests,verify_single_no_terminate_message)1393 TEST_F(SystemTests, verify_single_no_terminate_message) {
1394 std::string expected =
1395 "Note: Google Test filter = *.DISABLED_fail\n"
1396 "[==========] Running 1 test from 1 test suite (20 jobs).\n"
1397 "[ RUN ] SystemTests.DISABLED_fail\n"
1398 "file:(XX) Failure in test SystemTests.DISABLED_fail\n"
1399 "Expected equality of these values:\n"
1400 " 1\n"
1401 " 0\n"
1402 "SystemTests.DISABLED_fail exited with exitcode 1.\n"
1403 "[ FAILED ] SystemTests.DISABLED_fail (XX ms)\n"
1404 "[==========] 1 test from 1 test suite ran. (XX ms total)\n"
1405 "[ PASSED ] 0 tests.\n"
1406 "[ FAILED ] 1 test, listed below:\n"
1407 "[ FAILED ] SystemTests.DISABLED_fail\n"
1408 "\n"
1409 " 1 FAILED TEST\n";
1410 ASSERT_NO_FATAL_FAILURE(
1411 Verify("*.DISABLED_fail", expected, 1, std::vector<const char*>{"--gtest_break_on_failure"}));
1412 }
1413
1414 // These tests are used by the verify_disabled tests.
TEST_F(SystemTests,always_pass)1415 TEST_F(SystemTests, always_pass) {}
1416
TEST_F(SystemTests,DISABLED_always_pass)1417 TEST_F(SystemTests, DISABLED_always_pass) {}
1418
1419 // The tests listed below will not run by default. They are executed by
1420 // the above tests.
TEST_F(SystemTests,DISABLED_pass)1421 TEST_F(SystemTests, DISABLED_pass) {}
1422
TEST_F(SystemTests,DISABLED_fail)1423 TEST_F(SystemTests, DISABLED_fail) {
1424 ASSERT_EQ(1, 0);
1425 }
1426
TEST_F(SystemTests,DISABLED_crash)1427 TEST_F(SystemTests, DISABLED_crash) __attribute__((optnone)) {
1428 char* p = reinterpret_cast<char*>(static_cast<intptr_t>(atoi("0")));
1429 *p = 3;
1430 }
1431
TEST_F(SystemTests,DISABLED_sigquit_no_sleep)1432 TEST_F(SystemTests, DISABLED_sigquit_no_sleep) {}
1433
TEST_F(SystemTests,DISABLED_sigquit_sleep_5)1434 TEST_F(SystemTests, DISABLED_sigquit_sleep_5) {
1435 sleep(5);
1436 }
1437
TEST_F(SystemTests,DISABLED_sigquit_sleep_6)1438 TEST_F(SystemTests, DISABLED_sigquit_sleep_6) {
1439 sleep(6);
1440 }
1441
TEST_F(SystemTests,DISABLED_sleep_forever)1442 TEST_F(SystemTests, DISABLED_sleep_forever) {
1443 while (true) {
1444 sleep(10000);
1445 }
1446 }
1447
TEST_F(SystemTests,DISABLED_sleep5)1448 TEST_F(SystemTests, DISABLED_sleep5) {
1449 sleep(5);
1450 }
1451
1452 // These tests will finish 1, 2, 3 in non-isolated mode and 3, 2, 1 in isolated
1453 // mode.
TEST_F(SystemTests,DISABLED_order_1)1454 TEST_F(SystemTests, DISABLED_order_1) {
1455 sleep(6);
1456 }
1457
TEST_F(SystemTests,DISABLED_order_2)1458 TEST_F(SystemTests, DISABLED_order_2) {
1459 sleep(3);
1460 }
1461
TEST_F(SystemTests,DISABLED_order_3)1462 TEST_F(SystemTests, DISABLED_order_3) {}
1463
TEST_F(SystemTests,DISABLED_fail_0)1464 TEST_F(SystemTests, DISABLED_fail_0) {
1465 ASSERT_EQ(1, 0);
1466 }
1467
TEST_F(SystemTests,DISABLED_fail_1)1468 TEST_F(SystemTests, DISABLED_fail_1) {
1469 ASSERT_EQ(1, 0);
1470 }
1471
TEST_F(SystemTests,DISABLED_fail_2)1472 TEST_F(SystemTests, DISABLED_fail_2) {
1473 ASSERT_EQ(1, 0);
1474 }
1475
TEST_F(SystemTests,DISABLED_fail_3)1476 TEST_F(SystemTests, DISABLED_fail_3) {
1477 ASSERT_EQ(1, 0);
1478 }
1479
TEST_F(SystemTests,DISABLED_fail_4)1480 TEST_F(SystemTests, DISABLED_fail_4) {
1481 ASSERT_EQ(1, 0);
1482 }
1483
TEST_F(SystemTests,DISABLED_fail_5)1484 TEST_F(SystemTests, DISABLED_fail_5) {
1485 ASSERT_EQ(1, 0);
1486 }
1487
TEST_F(SystemTests,DISABLED_fail_6)1488 TEST_F(SystemTests, DISABLED_fail_6) {
1489 ASSERT_EQ(1, 0);
1490 }
1491
TEST_F(SystemTests,DISABLED_fail_7)1492 TEST_F(SystemTests, DISABLED_fail_7) {
1493 ASSERT_EQ(1, 0);
1494 }
1495
TEST_F(SystemTests,DISABLED_fail_8)1496 TEST_F(SystemTests, DISABLED_fail_8) {
1497 ASSERT_EQ(1, 0);
1498 }
1499
TEST_F(SystemTests,DISABLED_fail_9)1500 TEST_F(SystemTests, DISABLED_fail_9) {
1501 ASSERT_EQ(1, 0);
1502 }
1503
TEST_F(SystemTests,DISABLED_all_pass_1)1504 TEST_F(SystemTests, DISABLED_all_pass_1) {}
1505
TEST_F(SystemTests,DISABLED_all_pass_2)1506 TEST_F(SystemTests, DISABLED_all_pass_2) {}
1507
TEST_F(SystemTests,DISABLED_all_skip_1)1508 TEST_F(SystemTests, DISABLED_all_skip_1) {
1509 GTEST_SKIP();
1510 }
1511
TEST_F(SystemTests,DISABLED_all_skip_2)1512 TEST_F(SystemTests, DISABLED_all_skip_2) {
1513 GTEST_SKIP() << "Skip message present";
1514 }
1515
TEST_F(SystemTests,DISABLED_all_slow_1)1516 TEST_F(SystemTests, DISABLED_all_slow_1) {
1517 sleep(3);
1518 }
1519
TEST_F(SystemTests,DISABLED_all_slow_2)1520 TEST_F(SystemTests, DISABLED_all_slow_2) {
1521 sleep(3);
1522 }
1523
TEST_F(SystemTests,DISABLED_all_fail_1)1524 TEST_F(SystemTests, DISABLED_all_fail_1) {
1525 ASSERT_EQ(1, 0);
1526 }
1527
TEST_F(SystemTests,DISABLED_all_fail_2)1528 TEST_F(SystemTests, DISABLED_all_fail_2) {
1529 ASSERT_EQ(1, 0);
1530 }
1531
TEST_F(SystemTests,DISABLED_all_timeout_1)1532 TEST_F(SystemTests, DISABLED_all_timeout_1) {
1533 sleep(6);
1534 }
1535
TEST_F(SystemTests,DISABLED_all_timeout_2)1536 TEST_F(SystemTests, DISABLED_all_timeout_2) {
1537 sleep(6);
1538 }
1539
TEST_F(SystemTests,DISABLED_job_1)1540 TEST_F(SystemTests, DISABLED_job_1) {
1541 sleep(5);
1542 }
1543
TEST_F(SystemTests,DISABLED_job_2)1544 TEST_F(SystemTests, DISABLED_job_2) {
1545 sleep(3);
1546 }
1547
TEST_F(SystemTests,DISABLED_job_3)1548 TEST_F(SystemTests, DISABLED_job_3) {
1549 sleep(4);
1550 }
1551
TEST_F(SystemTests,DISABLED_skip_no_message)1552 TEST_F(SystemTests, DISABLED_skip_no_message) {
1553 GTEST_SKIP();
1554 }
1555
TEST_F(SystemTests,DISABLED_skip_with_message)1556 TEST_F(SystemTests, DISABLED_skip_with_message) {
1557 GTEST_SKIP() << "This is a skip message";
1558 }
1559
TEST_F(SystemTests,DISABLED_skip_with_output_before)1560 TEST_F(SystemTests, DISABLED_skip_with_output_before) {
1561 printf("This is the message before the skip message\n");
1562 GTEST_SKIP() << "This is the skip message";
1563 }
1564
1565 // Do not optimize this call away so that the print after the skip
1566 // will actually occur.
AvoidSkipStopping(int tag=0)1567 void AvoidSkipStopping(int tag = 0) __attribute__((optnone)) {
1568 if (tag == 0) {
1569 GTEST_SKIP() << "This is the skip message";
1570 } else {
1571 GTEST_SKIP() << "This is the skip message " << std::to_string(tag);
1572 }
1573 }
1574
TEST_F(SystemTests,DISABLED_skip_with_output_after)1575 TEST_F(SystemTests, DISABLED_skip_with_output_after) {
1576 AvoidSkipStopping();
1577 printf("This is the message after the skip message\n");
1578 }
1579
TEST_F(SystemTests,DISABLED_skip_with_skipped_line)1580 TEST_F(SystemTests, DISABLED_skip_with_skipped_line) {
1581 printf("\nSkipped\n");
1582 AvoidSkipStopping(1);
1583 printf("Skipped\n");
1584 AvoidSkipStopping(2);
1585 printf("Skipped\n");
1586 }
1587
TEST_F(SystemTests,DISABLED_skip_multiple)1588 TEST_F(SystemTests, DISABLED_skip_multiple) {
1589 printf("This is not a skip message 1\n");
1590 AvoidSkipStopping(1);
1591 printf("This is not a skip message 2\n");
1592 AvoidSkipStopping(2);
1593 AvoidSkipStopping(3);
1594 printf("This is not a skip message 4\n");
1595 }
1596
1597 class DISABLED_SystemTestsXfail : public ::testing::Test {};
1598
TEST_F(DISABLED_SystemTestsXfail,xfail_fail)1599 TEST_F(DISABLED_SystemTestsXfail, xfail_fail) {
1600 ASSERT_EQ(1, 0);
1601 }
1602
TEST_F(DISABLED_SystemTestsXfail,xfail_pass)1603 TEST_F(DISABLED_SystemTestsXfail, xfail_pass) {}
1604
1605 class SystemTestsDeathTest : public ::testing::Test {
1606 protected:
SetUp()1607 virtual void SetUp() { ::testing::FLAGS_gtest_death_test_style = "threadsafe"; }
1608 };
1609
DeathTestHelperPass()1610 static void DeathTestHelperPass() {
1611 ASSERT_EQ(1, 1);
1612 exit(0);
1613 }
1614
TEST_F(SystemTestsDeathTest,DISABLED_death_pass)1615 TEST_F(SystemTestsDeathTest, DISABLED_death_pass) {
1616 ASSERT_EXIT(DeathTestHelperPass(), ::testing::ExitedWithCode(0), "");
1617 }
1618
DeathTestHelperFail()1619 static void DeathTestHelperFail() {
1620 ASSERT_EQ(1, 0);
1621 }
1622
TEST_F(SystemTestsDeathTest,DISABLED_death_fail)1623 TEST_F(SystemTestsDeathTest, DISABLED_death_fail) {
1624 ASSERT_EXIT(DeathTestHelperFail(), ::testing::ExitedWithCode(0), "");
1625 }
1626
TEST(SystemTestsXml1,DISABLED_xml_1)1627 TEST(SystemTestsXml1, DISABLED_xml_1) {}
1628
TEST(SystemTestsXml1,DISABLED_xml_2)1629 TEST(SystemTestsXml1, DISABLED_xml_2) {
1630 ASSERT_EQ(1, 0);
1631 }
1632
TEST(SystemTestsXml2,DISABLED_xml_1)1633 TEST(SystemTestsXml2, DISABLED_xml_1) {
1634 ASSERT_EQ(1, 0);
1635 }
1636
TEST(SystemTestsXml2,DISABLED_xml_2)1637 TEST(SystemTestsXml2, DISABLED_xml_2) {}
1638
TEST(SystemTestsXml3,DISABLED_xml_1)1639 TEST(SystemTestsXml3, DISABLED_xml_1) {}
1640
TEST(SystemTestsXml3,DISABLED_xml_2)1641 TEST(SystemTestsXml3, DISABLED_xml_2) {
1642 ASSERT_EQ(1, 0);
1643 }
1644
TEST(SystemTestsMemory,DISABLED_memory)1645 TEST(SystemTestsMemory, DISABLED_memory) {
1646 #if !defined(__APPLE__)
1647 struct mallinfo info = mallinfo();
1648 #if defined(__ANDROID__)
1649 printf("Allocated %zu\n", info.uordblks);
1650 #else
1651 printf("Allocated %d\n", info.uordblks);
1652 #endif
1653 #else
1654 printf("Allocated 0\n");
1655 #endif
1656 }
1657
TEST(SystemTestsShard1,DISABLED_case1_test1)1658 TEST(SystemTestsShard1, DISABLED_case1_test1) {}
1659
TEST(SystemTestsShard1,DISABLED_case1_test2)1660 TEST(SystemTestsShard1, DISABLED_case1_test2) {}
1661
TEST(SystemTestsShard1,DISABLED_case1_test3)1662 TEST(SystemTestsShard1, DISABLED_case1_test3) {}
1663
TEST(SystemTestsShard1,DISABLED_case1_test4)1664 TEST(SystemTestsShard1, DISABLED_case1_test4) {}
1665
TEST(SystemTestsShard2,DISABLED_case2_test1)1666 TEST(SystemTestsShard2, DISABLED_case2_test1) {
1667 }
1668
TEST(SystemTestsShard2,DISABLED_case2_test2)1669 TEST(SystemTestsShard2, DISABLED_case2_test2) {
1670 }
1671
TEST(SystemTestsShard2,DISABLED_case2_test3)1672 TEST(SystemTestsShard2, DISABLED_case2_test3) {
1673 }
1674
TEST(SystemTestsShard2,DISABLED_case2_test4)1675 TEST(SystemTestsShard2, DISABLED_case2_test4) {
1676 }
1677
TEST(SystemTestsShard3,DISABLED_case3_test1)1678 TEST(SystemTestsShard3, DISABLED_case3_test1) {
1679 }
1680
TEST(SystemTestsShard3,DISABLED_case3_test2)1681 TEST(SystemTestsShard3, DISABLED_case3_test2) {
1682 }
1683
TEST(SystemTestsShard3,DISABLED_case3_test3)1684 TEST(SystemTestsShard3, DISABLED_case3_test3) {
1685 }
1686
TEST(SystemTestsShard3,DISABLED_case3_test4)1687 TEST(SystemTestsShard3, DISABLED_case3_test4) {
1688 }
1689
1690 } // namespace gtest_extras
1691 } // namespace android
1692