1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/test/launcher/test_launcher.h"
6
7 #include <stdio.h>
8
9 #include <algorithm>
10 #include <map>
11 #include <random>
12 #include <utility>
13
14 #include "base/at_exit.h"
15 #include "base/bind.h"
16 #include "base/command_line.h"
17 #include "base/environment.h"
18 #include "base/files/file_path.h"
19 #include "base/files/file_util.h"
20 #include "base/files/scoped_file.h"
21 #include "base/format_macros.h"
22 #include "base/hash.h"
23 #include "base/lazy_instance.h"
24 #include "base/location.h"
25 #include "base/logging.h"
26 #include "base/macros.h"
27 #include "base/memory/ptr_util.h"
28 #include "base/numerics/safe_conversions.h"
29 #include "base/process/kill.h"
30 #include "base/process/launch.h"
31 #include "base/run_loop.h"
32 #include "base/single_thread_task_runner.h"
33 #include "base/strings/pattern.h"
34 #include "base/strings/string_number_conversions.h"
35 #include "base/strings/string_piece.h"
36 #include "base/strings/string_split.h"
37 #include "base/strings/string_util.h"
38 #include "base/strings/stringize_macros.h"
39 #include "base/strings/stringprintf.h"
40 #include "base/strings/utf_string_conversions.h"
41 #include "base/sys_info.h"
42 #include "base/task_scheduler/post_task.h"
43 #include "base/task_scheduler/task_scheduler.h"
44 #include "base/test/gtest_util.h"
45 #include "base/test/launcher/test_launcher_tracer.h"
46 #include "base/test/launcher/test_results_tracker.h"
47 #include "base/test/test_switches.h"
48 #include "base/test/test_timeouts.h"
49 #include "base/threading/thread_restrictions.h"
50 #include "base/threading/thread_task_runner_handle.h"
51 #include "base/time/time.h"
52 #include "build/build_config.h"
53 #include "testing/gtest/include/gtest/gtest.h"
54
55 #if defined(OS_POSIX)
56 #include <fcntl.h>
57
58 #include "base/files/file_descriptor_watcher_posix.h"
59 #endif
60
61 #if defined(OS_MACOSX)
62 #include "base/mac/scoped_nsautorelease_pool.h"
63 #endif
64
65 #if defined(OS_WIN)
66 #include "base/win/windows_version.h"
67 #endif
68
69 #if defined(OS_FUCHSIA)
70 #include <lib/zx/job.h>
71 #include "base/fuchsia/default_job.h"
72 #include "base/fuchsia/fuchsia_logging.h"
73 #endif
74
75 namespace base {
76
77 // See https://groups.google.com/a/chromium.org/d/msg/chromium-dev/nkdTP7sstSc/uT3FaE_sgkAJ .
78 using ::operator<<;
79
80 // The environment variable name for the total number of test shards.
81 const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS";
82 // The environment variable name for the test shard index.
83 const char kTestShardIndex[] = "GTEST_SHARD_INDEX";
84
85 namespace {
86
87 // Global tag for test runs where the results are incomplete or unreliable
88 // for any reason, e.g. early exit because of too many broken tests.
89 const char kUnreliableResultsTag[] = "UNRELIABLE_RESULTS";
90
91 // Maximum time of no output after which we print list of processes still
92 // running. This deliberately doesn't use TestTimeouts (which is otherwise
93 // a recommended solution), because they can be increased. This would defeat
94 // the purpose of this timeout, which is 1) to avoid buildbot "no output for
95 // X seconds" timeout killing the process 2) help communicate status of
96 // the test launcher to people looking at the output (no output for a long
97 // time is mysterious and gives no info about what is happening) 3) help
98 // debugging in case the process hangs anyway.
99 constexpr TimeDelta kOutputTimeout = TimeDelta::FromSeconds(15);
100
101 // Limit of output snippet lines when printing to stdout.
102 // Avoids flooding the logs with amount of output that gums up
103 // the infrastructure.
104 const size_t kOutputSnippetLinesLimit = 5000;
105
106 // Limit of output snippet size. Exceeding this limit
107 // results in truncating the output and failing the test.
108 const size_t kOutputSnippetBytesLimit = 300 * 1024;
109
110 // Limit of seed values for gtest shuffling. Arbitrary, but based on
111 // gtest's similarly arbitrary choice.
112 const uint32_t kRandomSeedUpperBound = 100000;
113
114 // Set of live launch test processes with corresponding lock (it is allowed
115 // for callers to launch processes on different threads).
GetLiveProcessesLock()116 Lock* GetLiveProcessesLock() {
117 static auto* lock = new Lock;
118 return lock;
119 }
120
GetLiveProcesses()121 std::map<ProcessHandle, CommandLine>* GetLiveProcesses() {
122 static auto* map = new std::map<ProcessHandle, CommandLine>;
123 return map;
124 }
125
126 // Performance trace generator.
GetTestLauncherTracer()127 TestLauncherTracer* GetTestLauncherTracer() {
128 static auto* tracer = new TestLauncherTracer;
129 return tracer;
130 }
131
132 // Creates and starts a TaskScheduler with |num_parallel_jobs| dedicated to
133 // foreground blocking tasks (corresponds to the traits used to launch and wait
134 // for child processes).
CreateAndStartTaskScheduler(int num_parallel_jobs)135 void CreateAndStartTaskScheduler(int num_parallel_jobs) {
136 // These values are taken from TaskScheduler::StartWithDefaultParams(), which
137 // is not used directly to allow a custom number of threads in the foreground
138 // blocking pool.
139 constexpr int kMaxBackgroundThreads = 1;
140 constexpr int kMaxBackgroundBlockingThreads = 2;
141 const int max_foreground_threads =
142 std::max(1, base::SysInfo::NumberOfProcessors());
143 constexpr base::TimeDelta kSuggestedReclaimTime =
144 base::TimeDelta::FromSeconds(30);
145 base::TaskScheduler::Create("TestLauncher");
146 base::TaskScheduler::GetInstance()->Start(
147 {{kMaxBackgroundThreads, kSuggestedReclaimTime},
148 {kMaxBackgroundBlockingThreads, kSuggestedReclaimTime},
149 {max_foreground_threads, kSuggestedReclaimTime},
150 {num_parallel_jobs, kSuggestedReclaimTime}});
151 }
152
153 #if defined(OS_POSIX)
154 // Self-pipe that makes it possible to do complex shutdown handling
155 // outside of the signal handler.
156 int g_shutdown_pipe[2] = { -1, -1 };
157
ShutdownPipeSignalHandler(int signal)158 void ShutdownPipeSignalHandler(int signal) {
159 HANDLE_EINTR(write(g_shutdown_pipe[1], "q", 1));
160 }
161
KillSpawnedTestProcesses()162 void KillSpawnedTestProcesses() {
163 // Keep the lock until exiting the process to prevent further processes
164 // from being spawned.
165 AutoLock lock(*GetLiveProcessesLock());
166
167 fprintf(stdout, "Sending SIGTERM to %" PRIuS " child processes... ",
168 GetLiveProcesses()->size());
169 fflush(stdout);
170
171 for (const auto& pair : *GetLiveProcesses()) {
172 // Send the signal to entire process group.
173 kill((-1) * (pair.first), SIGTERM);
174 }
175
176 fprintf(stdout,
177 "done.\nGiving processes a chance to terminate cleanly... ");
178 fflush(stdout);
179
180 PlatformThread::Sleep(TimeDelta::FromMilliseconds(500));
181
182 fprintf(stdout, "done.\n");
183 fflush(stdout);
184
185 fprintf(stdout, "Sending SIGKILL to %" PRIuS " child processes... ",
186 GetLiveProcesses()->size());
187 fflush(stdout);
188
189 for (const auto& pair : *GetLiveProcesses()) {
190 // Send the signal to entire process group.
191 kill((-1) * (pair.first), SIGKILL);
192 }
193
194 fprintf(stdout, "done.\n");
195 fflush(stdout);
196 }
197 #endif // defined(OS_POSIX)
198
199 // Parses the environment variable var as an Int32. If it is unset, returns
200 // true. If it is set, unsets it then converts it to Int32 before
201 // returning it in |result|. Returns true on success.
TakeInt32FromEnvironment(const char * const var,int32_t * result)202 bool TakeInt32FromEnvironment(const char* const var, int32_t* result) {
203 std::unique_ptr<Environment> env(Environment::Create());
204 std::string str_val;
205
206 if (!env->GetVar(var, &str_val))
207 return true;
208
209 if (!env->UnSetVar(var)) {
210 LOG(ERROR) << "Invalid environment: we could not unset " << var << ".\n";
211 return false;
212 }
213
214 if (!StringToInt(str_val, result)) {
215 LOG(ERROR) << "Invalid environment: " << var << " is not an integer.\n";
216 return false;
217 }
218
219 return true;
220 }
221
222 // Unsets the environment variable |name| and returns true on success.
223 // Also returns true if the variable just doesn't exist.
UnsetEnvironmentVariableIfExists(const std::string & name)224 bool UnsetEnvironmentVariableIfExists(const std::string& name) {
225 std::unique_ptr<Environment> env(Environment::Create());
226 std::string str_val;
227 if (!env->GetVar(name, &str_val))
228 return true;
229 return env->UnSetVar(name);
230 }
231
232 // Returns true if bot mode has been requested, i.e. defaults optimized
233 // for continuous integration bots. This way developers don't have to remember
234 // special command-line flags.
BotModeEnabled()235 bool BotModeEnabled() {
236 std::unique_ptr<Environment> env(Environment::Create());
237 return CommandLine::ForCurrentProcess()->HasSwitch(
238 switches::kTestLauncherBotMode) ||
239 env->HasVar("CHROMIUM_TEST_LAUNCHER_BOT_MODE");
240 }
241
242 // Returns command line command line after gtest-specific processing
243 // and applying |wrapper|.
PrepareCommandLineForGTest(const CommandLine & command_line,const std::string & wrapper)244 CommandLine PrepareCommandLineForGTest(const CommandLine& command_line,
245 const std::string& wrapper) {
246 CommandLine new_command_line(command_line.GetProgram());
247 CommandLine::SwitchMap switches = command_line.GetSwitches();
248
249 // Handled by the launcher process.
250 switches.erase(kGTestRepeatFlag);
251 switches.erase(kGTestShuffleFlag);
252 switches.erase(kGTestRandomSeedFlag);
253
254 // Don't try to write the final XML report in child processes.
255 switches.erase(kGTestOutputFlag);
256
257 for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
258 iter != switches.end(); ++iter) {
259 new_command_line.AppendSwitchNative((*iter).first, (*iter).second);
260 }
261
262 // Prepend wrapper after last CommandLine quasi-copy operation. CommandLine
263 // does not really support removing switches well, and trying to do that
264 // on a CommandLine with a wrapper is known to break.
265 // TODO(phajdan.jr): Give it a try to support CommandLine removing switches.
266 #if defined(OS_WIN)
267 new_command_line.PrependWrapper(ASCIIToUTF16(wrapper));
268 #else
269 new_command_line.PrependWrapper(wrapper);
270 #endif
271
272 return new_command_line;
273 }
274
275 // Launches a child process using |command_line|. If the child process is still
276 // running after |timeout|, it is terminated and |*was_timeout| is set to true.
277 // Returns exit code of the process.
LaunchChildTestProcessWithOptions(const CommandLine & command_line,const LaunchOptions & options,int flags,TimeDelta timeout,ProcessLifetimeObserver * observer,bool * was_timeout)278 int LaunchChildTestProcessWithOptions(const CommandLine& command_line,
279 const LaunchOptions& options,
280 int flags,
281 TimeDelta timeout,
282 ProcessLifetimeObserver* observer,
283 bool* was_timeout) {
284 TimeTicks start_time(TimeTicks::Now());
285
286 #if defined(OS_POSIX)
287 // Make sure an option we rely on is present - see LaunchChildGTestProcess.
288 DCHECK(options.new_process_group);
289 #endif
290
291 LaunchOptions new_options(options);
292
293 #if defined(OS_WIN)
294 DCHECK(!new_options.job_handle);
295
296 win::ScopedHandle job_handle;
297 if (flags & TestLauncher::USE_JOB_OBJECTS) {
298 job_handle.Set(CreateJobObject(NULL, NULL));
299 if (!job_handle.IsValid()) {
300 LOG(ERROR) << "Could not create JobObject.";
301 return -1;
302 }
303
304 DWORD job_flags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
305
306 // Allow break-away from job since sandbox and few other places rely on it
307 // on Windows versions prior to Windows 8 (which supports nested jobs).
308 if (win::GetVersion() < win::VERSION_WIN8 &&
309 flags & TestLauncher::ALLOW_BREAKAWAY_FROM_JOB) {
310 job_flags |= JOB_OBJECT_LIMIT_BREAKAWAY_OK;
311 }
312
313 if (!SetJobObjectLimitFlags(job_handle.Get(), job_flags)) {
314 LOG(ERROR) << "Could not SetJobObjectLimitFlags.";
315 return -1;
316 }
317
318 new_options.job_handle = job_handle.Get();
319 }
320 #elif defined(OS_FUCHSIA)
321 DCHECK(!new_options.job_handle);
322
323 zx::job job_handle;
324 zx_status_t result = zx::job::create(*GetDefaultJob(), 0, &job_handle);
325 ZX_CHECK(ZX_OK == result, result) << "zx_job_create";
326 new_options.job_handle = job_handle.get();
327 #endif // defined(OS_FUCHSIA)
328
329 #if defined(OS_LINUX)
330 // To prevent accidental privilege sharing to an untrusted child, processes
331 // are started with PR_SET_NO_NEW_PRIVS. Do not set that here, since this
332 // new child will be privileged and trusted.
333 new_options.allow_new_privs = true;
334 #endif
335
336 Process process;
337
338 {
339 // Note how we grab the lock before the process possibly gets created.
340 // This ensures that when the lock is held, ALL the processes are registered
341 // in the set.
342 AutoLock lock(*GetLiveProcessesLock());
343
344 #if defined(OS_WIN)
345 // Allow the handle used to capture stdio and stdout to be inherited by the
346 // child. Note that this is done under GetLiveProcessesLock() to ensure that
347 // only the desired child receives the handle.
348 if (new_options.stdout_handle) {
349 ::SetHandleInformation(new_options.stdout_handle, HANDLE_FLAG_INHERIT,
350 HANDLE_FLAG_INHERIT);
351 }
352 #endif
353
354 process = LaunchProcess(command_line, new_options);
355
356 #if defined(OS_WIN)
357 // Revoke inheritance so that the handle isn't leaked into other children.
358 // Note that this is done under GetLiveProcessesLock() to ensure that only
359 // the desired child receives the handle.
360 if (new_options.stdout_handle)
361 ::SetHandleInformation(new_options.stdout_handle, HANDLE_FLAG_INHERIT, 0);
362 #endif
363
364 if (!process.IsValid())
365 return -1;
366
367 // TODO(rvargas) crbug.com/417532: Don't store process handles.
368 GetLiveProcesses()->insert(std::make_pair(process.Handle(), command_line));
369 }
370
371 if (observer)
372 observer->OnLaunched(process.Handle(), process.Pid());
373
374 int exit_code = 0;
375 bool did_exit = false;
376
377 {
378 base::ScopedAllowBaseSyncPrimitivesForTesting allow_base_sync_primitives;
379 did_exit = process.WaitForExitWithTimeout(timeout, &exit_code);
380 }
381
382 if (!did_exit) {
383 if (observer)
384 observer->OnTimedOut(command_line);
385
386 *was_timeout = true;
387 exit_code = -1; // Set a non-zero exit code to signal a failure.
388
389 {
390 base::ScopedAllowBaseSyncPrimitivesForTesting allow_base_sync_primitives;
391 // Ensure that the process terminates.
392 process.Terminate(-1, true);
393 }
394 }
395
396 {
397 // Note how we grab the log before issuing a possibly broad process kill.
398 // Other code parts that grab the log kill processes, so avoid trying
399 // to do that twice and trigger all kinds of log messages.
400 AutoLock lock(*GetLiveProcessesLock());
401
402 #if defined(OS_FUCHSIA)
403 zx_status_t status = job_handle.kill();
404 ZX_CHECK(status == ZX_OK, status);
405 #elif defined(OS_POSIX)
406 if (exit_code != 0) {
407 // On POSIX, in case the test does not exit cleanly, either due to a crash
408 // or due to it timing out, we need to clean up any child processes that
409 // it might have created. On Windows, child processes are automatically
410 // cleaned up using JobObjects.
411 KillProcessGroup(process.Handle());
412 }
413 #endif
414
415 GetLiveProcesses()->erase(process.Handle());
416 }
417
418 GetTestLauncherTracer()->RecordProcessExecution(
419 start_time, TimeTicks::Now() - start_time);
420
421 return exit_code;
422 }
423
DoLaunchChildTestProcess(const CommandLine & command_line,TimeDelta timeout,const TestLauncher::LaunchOptions & test_launch_options,bool redirect_stdio,SingleThreadTaskRunner * task_runner,std::unique_ptr<ProcessLifetimeObserver> observer)424 void DoLaunchChildTestProcess(
425 const CommandLine& command_line,
426 TimeDelta timeout,
427 const TestLauncher::LaunchOptions& test_launch_options,
428 bool redirect_stdio,
429 SingleThreadTaskRunner* task_runner,
430 std::unique_ptr<ProcessLifetimeObserver> observer) {
431 TimeTicks start_time = TimeTicks::Now();
432
433 ScopedFILE output_file;
434 FilePath output_filename;
435 if (redirect_stdio) {
436 FILE* raw_output_file = CreateAndOpenTemporaryFile(&output_filename);
437 output_file.reset(raw_output_file);
438 CHECK(output_file);
439 }
440
441 LaunchOptions options;
442 #if defined(OS_WIN)
443
444 options.inherit_mode = test_launch_options.inherit_mode;
445 options.handles_to_inherit = test_launch_options.handles_to_inherit;
446 if (redirect_stdio) {
447 HANDLE handle =
448 reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(output_file.get())));
449 CHECK_NE(INVALID_HANDLE_VALUE, handle);
450 options.stdin_handle = INVALID_HANDLE_VALUE;
451 options.stdout_handle = handle;
452 options.stderr_handle = handle;
453 // See LaunchOptions.stdout_handle comments for why this compares against
454 // FILE_TYPE_CHAR.
455 if (options.inherit_mode == base::LaunchOptions::Inherit::kSpecific &&
456 GetFileType(handle) != FILE_TYPE_CHAR) {
457 options.handles_to_inherit.push_back(handle);
458 }
459 }
460
461 #else // if !defined(OS_WIN)
462
463 options.fds_to_remap = test_launch_options.fds_to_remap;
464 if (redirect_stdio) {
465 int output_file_fd = fileno(output_file.get());
466 CHECK_LE(0, output_file_fd);
467 options.fds_to_remap.push_back(
468 std::make_pair(output_file_fd, STDOUT_FILENO));
469 options.fds_to_remap.push_back(
470 std::make_pair(output_file_fd, STDERR_FILENO));
471 }
472
473 #if !defined(OS_FUCHSIA)
474 options.new_process_group = true;
475 #endif
476 #if defined(OS_LINUX)
477 options.kill_on_parent_death = true;
478 #endif
479
480 #endif // !defined(OS_WIN)
481
482 bool was_timeout = false;
483 int exit_code = LaunchChildTestProcessWithOptions(
484 command_line, options, test_launch_options.flags, timeout, observer.get(),
485 &was_timeout);
486
487 std::string output_file_contents;
488 if (redirect_stdio) {
489 fflush(output_file.get());
490 output_file.reset();
491 // Reading the file can sometimes fail when the process was killed midflight
492 // (e.g. on test suite timeout): https://crbug.com/826408. Attempt to read
493 // the output file anyways, but do not crash on failure in this case.
494 CHECK(ReadFileToString(output_filename, &output_file_contents) ||
495 exit_code != 0);
496
497 if (!DeleteFile(output_filename, false)) {
498 // This needs to be non-fatal at least for Windows.
499 LOG(WARNING) << "Failed to delete " << output_filename.AsUTF8Unsafe();
500 }
501 }
502
503 // Invoke OnCompleted on the thread it was originating from, not on a worker
504 // pool thread.
505 task_runner->PostTask(
506 FROM_HERE,
507 BindOnce(&ProcessLifetimeObserver::OnCompleted, std::move(observer),
508 exit_code, TimeTicks::Now() - start_time, was_timeout,
509 output_file_contents));
510 }
511
512 } // namespace
513
514 const char kGTestBreakOnFailure[] = "gtest_break_on_failure";
515 const char kGTestFilterFlag[] = "gtest_filter";
516 const char kGTestFlagfileFlag[] = "gtest_flagfile";
517 const char kGTestHelpFlag[] = "gtest_help";
518 const char kGTestListTestsFlag[] = "gtest_list_tests";
519 const char kGTestRepeatFlag[] = "gtest_repeat";
520 const char kGTestRunDisabledTestsFlag[] = "gtest_also_run_disabled_tests";
521 const char kGTestOutputFlag[] = "gtest_output";
522 const char kGTestShuffleFlag[] = "gtest_shuffle";
523 const char kGTestRandomSeedFlag[] = "gtest_random_seed";
524
525 TestLauncherDelegate::~TestLauncherDelegate() = default;
526
527 TestLauncher::LaunchOptions::LaunchOptions() = default;
528 TestLauncher::LaunchOptions::LaunchOptions(const LaunchOptions& other) =
529 default;
530 TestLauncher::LaunchOptions::~LaunchOptions() = default;
531
TestLauncher(TestLauncherDelegate * launcher_delegate,size_t parallel_jobs)532 TestLauncher::TestLauncher(TestLauncherDelegate* launcher_delegate,
533 size_t parallel_jobs)
534 : launcher_delegate_(launcher_delegate),
535 total_shards_(1),
536 shard_index_(0),
537 cycles_(1),
538 test_found_count_(0),
539 test_started_count_(0),
540 test_finished_count_(0),
541 test_success_count_(0),
542 test_broken_count_(0),
543 retry_count_(0),
544 retry_limit_(0),
545 force_run_broken_tests_(false),
546 run_result_(true),
547 shuffle_(false),
548 shuffle_seed_(0),
549 watchdog_timer_(FROM_HERE,
550 kOutputTimeout,
551 this,
552 &TestLauncher::OnOutputTimeout),
553 parallel_jobs_(parallel_jobs) {}
554
~TestLauncher()555 TestLauncher::~TestLauncher() {
556 if (base::TaskScheduler::GetInstance()) {
557 base::TaskScheduler::GetInstance()->Shutdown();
558 }
559 }
560
Run()561 bool TestLauncher::Run() {
562 if (!Init())
563 return false;
564
565 // Value of |cycles_| changes after each iteration. Keep track of the
566 // original value.
567 int requested_cycles = cycles_;
568
569 #if defined(OS_POSIX)
570 CHECK_EQ(0, pipe(g_shutdown_pipe));
571
572 struct sigaction action;
573 memset(&action, 0, sizeof(action));
574 sigemptyset(&action.sa_mask);
575 action.sa_handler = &ShutdownPipeSignalHandler;
576
577 CHECK_EQ(0, sigaction(SIGINT, &action, nullptr));
578 CHECK_EQ(0, sigaction(SIGQUIT, &action, nullptr));
579 CHECK_EQ(0, sigaction(SIGTERM, &action, nullptr));
580
581 auto controller = base::FileDescriptorWatcher::WatchReadable(
582 g_shutdown_pipe[0],
583 base::Bind(&TestLauncher::OnShutdownPipeReadable, Unretained(this)));
584 #endif // defined(OS_POSIX)
585
586 // Start the watchdog timer.
587 watchdog_timer_.Reset();
588
589 ThreadTaskRunnerHandle::Get()->PostTask(
590 FROM_HERE, BindOnce(&TestLauncher::RunTestIteration, Unretained(this)));
591
592 RunLoop().Run();
593
594 if (requested_cycles != 1)
595 results_tracker_.PrintSummaryOfAllIterations();
596
597 MaybeSaveSummaryAsJSON(std::vector<std::string>());
598
599 return run_result_;
600 }
601
LaunchChildGTestProcess(const CommandLine & command_line,const std::string & wrapper,TimeDelta timeout,const LaunchOptions & options,std::unique_ptr<ProcessLifetimeObserver> observer)602 void TestLauncher::LaunchChildGTestProcess(
603 const CommandLine& command_line,
604 const std::string& wrapper,
605 TimeDelta timeout,
606 const LaunchOptions& options,
607 std::unique_ptr<ProcessLifetimeObserver> observer) {
608 DCHECK(thread_checker_.CalledOnValidThread());
609
610 // Record the exact command line used to launch the child.
611 CommandLine new_command_line(
612 PrepareCommandLineForGTest(command_line, wrapper));
613
614 // When running in parallel mode we need to redirect stdio to avoid mixed-up
615 // output. We also always redirect on the bots to get the test output into
616 // JSON summary.
617 bool redirect_stdio = (parallel_jobs_ > 1) || BotModeEnabled();
618
619 PostTaskWithTraits(
620 FROM_HERE, {MayBlock(), TaskShutdownBehavior::BLOCK_SHUTDOWN},
621 BindOnce(&DoLaunchChildTestProcess, new_command_line, timeout, options,
622 redirect_stdio, RetainedRef(ThreadTaskRunnerHandle::Get()),
623 std::move(observer)));
624 }
625
OnTestFinished(const TestResult & original_result)626 void TestLauncher::OnTestFinished(const TestResult& original_result) {
627 ++test_finished_count_;
628
629 TestResult result(original_result);
630
631 if (result.output_snippet.length() > kOutputSnippetBytesLimit) {
632 if (result.status == TestResult::TEST_SUCCESS)
633 result.status = TestResult::TEST_EXCESSIVE_OUTPUT;
634
635 // Keep the top and bottom of the log and truncate the middle part.
636 result.output_snippet =
637 result.output_snippet.substr(0, kOutputSnippetBytesLimit / 2) + "\n" +
638 StringPrintf("<truncated (%" PRIuS " bytes)>\n",
639 result.output_snippet.length()) +
640 result.output_snippet.substr(result.output_snippet.length() -
641 kOutputSnippetBytesLimit / 2) +
642 "\n";
643 }
644
645 bool print_snippet = false;
646 std::string print_test_stdio("auto");
647 if (CommandLine::ForCurrentProcess()->HasSwitch(
648 switches::kTestLauncherPrintTestStdio)) {
649 print_test_stdio = CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
650 switches::kTestLauncherPrintTestStdio);
651 }
652 if (print_test_stdio == "auto") {
653 print_snippet = (result.status != TestResult::TEST_SUCCESS);
654 } else if (print_test_stdio == "always") {
655 print_snippet = true;
656 } else if (print_test_stdio == "never") {
657 print_snippet = false;
658 } else {
659 LOG(WARNING) << "Invalid value of " << switches::kTestLauncherPrintTestStdio
660 << ": " << print_test_stdio;
661 }
662 if (print_snippet) {
663 std::vector<base::StringPiece> snippet_lines =
664 SplitStringPiece(result.output_snippet, "\n", base::KEEP_WHITESPACE,
665 base::SPLIT_WANT_ALL);
666 if (snippet_lines.size() > kOutputSnippetLinesLimit) {
667 size_t truncated_size = snippet_lines.size() - kOutputSnippetLinesLimit;
668 snippet_lines.erase(
669 snippet_lines.begin(),
670 snippet_lines.begin() + truncated_size);
671 snippet_lines.insert(snippet_lines.begin(), "<truncated>");
672 }
673 fprintf(stdout, "%s", base::JoinString(snippet_lines, "\n").c_str());
674 fflush(stdout);
675 }
676
677 if (result.status == TestResult::TEST_SUCCESS) {
678 ++test_success_count_;
679 } else {
680 tests_to_retry_.insert(result.full_name);
681 }
682
683 results_tracker_.AddTestResult(result);
684
685 // TODO(phajdan.jr): Align counter (padding).
686 std::string status_line(
687 StringPrintf("[%" PRIuS "/%" PRIuS "] %s ",
688 test_finished_count_,
689 test_started_count_,
690 result.full_name.c_str()));
691 if (result.completed()) {
692 status_line.append(StringPrintf("(%" PRId64 " ms)",
693 result.elapsed_time.InMilliseconds()));
694 } else if (result.status == TestResult::TEST_TIMEOUT) {
695 status_line.append("(TIMED OUT)");
696 } else if (result.status == TestResult::TEST_CRASH) {
697 status_line.append("(CRASHED)");
698 } else if (result.status == TestResult::TEST_SKIPPED) {
699 status_line.append("(SKIPPED)");
700 } else if (result.status == TestResult::TEST_UNKNOWN) {
701 status_line.append("(UNKNOWN)");
702 } else {
703 // Fail very loudly so it's not ignored.
704 CHECK(false) << "Unhandled test result status: " << result.status;
705 }
706 fprintf(stdout, "%s\n", status_line.c_str());
707 fflush(stdout);
708
709 // We just printed a status line, reset the watchdog timer.
710 watchdog_timer_.Reset();
711
712 // Do not waste time on timeouts. We include tests with unknown results here
713 // because sometimes (e.g. hang in between unit tests) that's how a timeout
714 // gets reported.
715 if (result.status == TestResult::TEST_TIMEOUT ||
716 result.status == TestResult::TEST_UNKNOWN) {
717 test_broken_count_++;
718 }
719 size_t broken_threshold =
720 std::max(static_cast<size_t>(20), test_found_count_ / 10);
721 if (!force_run_broken_tests_ && test_broken_count_ >= broken_threshold) {
722 fprintf(stdout, "Too many badly broken tests (%" PRIuS "), exiting now.\n",
723 test_broken_count_);
724 fflush(stdout);
725
726 #if defined(OS_POSIX)
727 KillSpawnedTestProcesses();
728 #endif // defined(OS_POSIX)
729
730 MaybeSaveSummaryAsJSON({"BROKEN_TEST_EARLY_EXIT", kUnreliableResultsTag});
731
732 exit(1);
733 }
734
735 if (test_finished_count_ != test_started_count_)
736 return;
737
738 if (tests_to_retry_.empty() || retry_count_ >= retry_limit_) {
739 OnTestIterationFinished();
740 return;
741 }
742
743 if (!force_run_broken_tests_ && tests_to_retry_.size() >= broken_threshold) {
744 fprintf(stdout,
745 "Too many failing tests (%" PRIuS "), skipping retries.\n",
746 tests_to_retry_.size());
747 fflush(stdout);
748
749 results_tracker_.AddGlobalTag("BROKEN_TEST_SKIPPED_RETRIES");
750 results_tracker_.AddGlobalTag(kUnreliableResultsTag);
751
752 OnTestIterationFinished();
753 return;
754 }
755
756 retry_count_++;
757
758 std::vector<std::string> test_names(tests_to_retry_.begin(),
759 tests_to_retry_.end());
760
761 tests_to_retry_.clear();
762
763 size_t retry_started_count = launcher_delegate_->RetryTests(this, test_names);
764 if (retry_started_count == 0) {
765 // Signal failure, but continue to run all requested test iterations.
766 // With the summary of all iterations at the end this is a good default.
767 run_result_ = false;
768
769 OnTestIterationFinished();
770 return;
771 }
772
773 fprintf(stdout, "Retrying %" PRIuS " test%s (retry #%" PRIuS ")\n",
774 retry_started_count,
775 retry_started_count > 1 ? "s" : "",
776 retry_count_);
777 fflush(stdout);
778
779 test_started_count_ += retry_started_count;
780 }
781
782 // Helper used to parse test filter files. Syntax is documented in
783 // //testing/buildbot/filters/README.md .
LoadFilterFile(const FilePath & file_path,std::vector<std::string> * positive_filter,std::vector<std::string> * negative_filter)784 bool LoadFilterFile(const FilePath& file_path,
785 std::vector<std::string>* positive_filter,
786 std::vector<std::string>* negative_filter) {
787 std::string file_content;
788 if (!ReadFileToString(file_path, &file_content)) {
789 LOG(ERROR) << "Failed to read the filter file.";
790 return false;
791 }
792
793 std::vector<std::string> filter_lines = SplitString(
794 file_content, "\n", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
795 int line_num = 0;
796 for (const std::string& filter_line : filter_lines) {
797 line_num++;
798
799 size_t hash_pos = filter_line.find('#');
800
801 // In case when # symbol is not in the beginning of the line and is not
802 // proceeded with a space then it's likely that the comment was
803 // unintentional.
804 if (hash_pos != std::string::npos && hash_pos > 0 &&
805 filter_line[hash_pos - 1] != ' ') {
806 LOG(WARNING) << "Content of line " << line_num << " in " << file_path
807 << " after # is treated as a comment, " << filter_line;
808 }
809
810 // Strip comments and whitespace from each line.
811 std::string trimmed_line =
812 TrimWhitespaceASCII(filter_line.substr(0, hash_pos), TRIM_ALL)
813 .as_string();
814
815 if (trimmed_line.substr(0, 2) == "//") {
816 LOG(ERROR) << "Line " << line_num << " in " << file_path
817 << " starts with //, use # for comments.";
818 return false;
819 }
820
821 // Treat a line starting with '//' as a comment.
822 if (trimmed_line.empty())
823 continue;
824
825 if (trimmed_line[0] == '-')
826 negative_filter->push_back(trimmed_line.substr(1));
827 else
828 positive_filter->push_back(trimmed_line);
829 }
830
831 return true;
832 }
833
Init()834 bool TestLauncher::Init() {
835 const CommandLine* command_line = CommandLine::ForCurrentProcess();
836
837 // Initialize sharding. Command line takes precedence over legacy environment
838 // variables.
839 if (command_line->HasSwitch(switches::kTestLauncherTotalShards) &&
840 command_line->HasSwitch(switches::kTestLauncherShardIndex)) {
841 if (!StringToInt(
842 command_line->GetSwitchValueASCII(
843 switches::kTestLauncherTotalShards),
844 &total_shards_)) {
845 LOG(ERROR) << "Invalid value for " << switches::kTestLauncherTotalShards;
846 return false;
847 }
848 if (!StringToInt(
849 command_line->GetSwitchValueASCII(
850 switches::kTestLauncherShardIndex),
851 &shard_index_)) {
852 LOG(ERROR) << "Invalid value for " << switches::kTestLauncherShardIndex;
853 return false;
854 }
855 fprintf(stdout,
856 "Using sharding settings from command line. This is shard %d/%d\n",
857 shard_index_, total_shards_);
858 fflush(stdout);
859 } else {
860 if (!TakeInt32FromEnvironment(kTestTotalShards, &total_shards_))
861 return false;
862 if (!TakeInt32FromEnvironment(kTestShardIndex, &shard_index_))
863 return false;
864 fprintf(stdout,
865 "Using sharding settings from environment. This is shard %d/%d\n",
866 shard_index_, total_shards_);
867 fflush(stdout);
868 }
869 if (shard_index_ < 0 ||
870 total_shards_ < 0 ||
871 shard_index_ >= total_shards_) {
872 LOG(ERROR) << "Invalid sharding settings: we require 0 <= "
873 << kTestShardIndex << " < " << kTestTotalShards
874 << ", but you have " << kTestShardIndex << "=" << shard_index_
875 << ", " << kTestTotalShards << "=" << total_shards_ << ".\n";
876 return false;
877 }
878
879 // Make sure we don't pass any sharding-related environment to the child
880 // processes. This test launcher implements the sharding completely.
881 CHECK(UnsetEnvironmentVariableIfExists("GTEST_TOTAL_SHARDS"));
882 CHECK(UnsetEnvironmentVariableIfExists("GTEST_SHARD_INDEX"));
883
884 if (command_line->HasSwitch(kGTestRepeatFlag) &&
885 !StringToInt(command_line->GetSwitchValueASCII(kGTestRepeatFlag),
886 &cycles_)) {
887 LOG(ERROR) << "Invalid value for " << kGTestRepeatFlag;
888 return false;
889 }
890
891 if (command_line->HasSwitch(switches::kTestLauncherRetryLimit)) {
892 int retry_limit = -1;
893 if (!StringToInt(command_line->GetSwitchValueASCII(
894 switches::kTestLauncherRetryLimit), &retry_limit) ||
895 retry_limit < 0) {
896 LOG(ERROR) << "Invalid value for " << switches::kTestLauncherRetryLimit;
897 return false;
898 }
899
900 retry_limit_ = retry_limit;
901 } else if (!command_line->HasSwitch(kGTestFilterFlag) || BotModeEnabled()) {
902 // Retry failures 3 times by default if we are running all of the tests or
903 // in bot mode.
904 retry_limit_ = 3;
905 }
906
907 if (command_line->HasSwitch(switches::kTestLauncherForceRunBrokenTests))
908 force_run_broken_tests_ = true;
909
910 // Some of the TestLauncherDelegate implementations don't call into gtest
911 // until they've already split into test-specific processes. This results
912 // in gtest's native shuffle implementation attempting to shuffle one test.
913 // Shuffling the list of tests in the test launcher (before the delegate
914 // gets involved) ensures that the entire shard is shuffled.
915 if (command_line->HasSwitch(kGTestShuffleFlag)) {
916 shuffle_ = true;
917
918 if (command_line->HasSwitch(kGTestRandomSeedFlag)) {
919 const std::string custom_seed_str =
920 command_line->GetSwitchValueASCII(kGTestRandomSeedFlag);
921 uint32_t custom_seed = 0;
922 if (!StringToUint(custom_seed_str, &custom_seed)) {
923 LOG(ERROR) << "Unable to parse seed \"" << custom_seed_str << "\".";
924 return false;
925 }
926 if (custom_seed >= kRandomSeedUpperBound) {
927 LOG(ERROR) << "Seed " << custom_seed << " outside of expected range "
928 << "[0, " << kRandomSeedUpperBound << ")";
929 return false;
930 }
931 shuffle_seed_ = custom_seed;
932 } else {
933 std::uniform_int_distribution<uint32_t> dist(0, kRandomSeedUpperBound);
934 std::random_device random_dev;
935 shuffle_seed_ = dist(random_dev);
936 }
937 } else if (command_line->HasSwitch(kGTestRandomSeedFlag)) {
938 LOG(ERROR) << kGTestRandomSeedFlag << " requires " << kGTestShuffleFlag;
939 return false;
940 }
941
942 fprintf(stdout, "Using %" PRIuS " parallel jobs.\n", parallel_jobs_);
943 fflush(stdout);
944
945 CreateAndStartTaskScheduler(static_cast<int>(parallel_jobs_));
946
947 std::vector<std::string> positive_file_filter;
948 std::vector<std::string> positive_gtest_filter;
949
950 if (command_line->HasSwitch(switches::kTestLauncherFilterFile)) {
951 base::FilePath filter_file_path = base::MakeAbsoluteFilePath(
952 command_line->GetSwitchValuePath(switches::kTestLauncherFilterFile));
953 if (!LoadFilterFile(filter_file_path, &positive_file_filter,
954 &negative_test_filter_))
955 return false;
956 }
957
958 // Split --gtest_filter at '-', if there is one, to separate into
959 // positive filter and negative filter portions.
960 std::string filter = command_line->GetSwitchValueASCII(kGTestFilterFlag);
961 size_t dash_pos = filter.find('-');
962 if (dash_pos == std::string::npos) {
963 positive_gtest_filter =
964 SplitString(filter, ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
965 } else {
966 // Everything up to the dash.
967 positive_gtest_filter =
968 SplitString(filter.substr(0, dash_pos), ":", base::TRIM_WHITESPACE,
969 base::SPLIT_WANT_ALL);
970
971 // Everything after the dash.
972 for (std::string pattern :
973 SplitString(filter.substr(dash_pos + 1), ":", base::TRIM_WHITESPACE,
974 base::SPLIT_WANT_ALL)) {
975 negative_test_filter_.push_back(pattern);
976 }
977 }
978
979 if (!launcher_delegate_->GetTests(&tests_)) {
980 LOG(ERROR) << "Failed to get list of tests.";
981 return false;
982 }
983
984 CombinePositiveTestFilters(std::move(positive_gtest_filter),
985 std::move(positive_file_filter));
986
987 if (!results_tracker_.Init(*command_line)) {
988 LOG(ERROR) << "Failed to initialize test results tracker.";
989 return 1;
990 }
991
992 #if defined(NDEBUG)
993 results_tracker_.AddGlobalTag("MODE_RELEASE");
994 #else
995 results_tracker_.AddGlobalTag("MODE_DEBUG");
996 #endif
997
998 // Operating systems (sorted alphabetically).
999 // Note that they can deliberately overlap, e.g. OS_LINUX is a subset
1000 // of OS_POSIX.
1001 #if defined(OS_ANDROID)
1002 results_tracker_.AddGlobalTag("OS_ANDROID");
1003 #endif
1004
1005 #if defined(OS_BSD)
1006 results_tracker_.AddGlobalTag("OS_BSD");
1007 #endif
1008
1009 #if defined(OS_FREEBSD)
1010 results_tracker_.AddGlobalTag("OS_FREEBSD");
1011 #endif
1012
1013 #if defined(OS_FUCHSIA)
1014 results_tracker_.AddGlobalTag("OS_FUCHSIA");
1015 #endif
1016
1017 #if defined(OS_IOS)
1018 results_tracker_.AddGlobalTag("OS_IOS");
1019 #endif
1020
1021 #if defined(OS_LINUX)
1022 results_tracker_.AddGlobalTag("OS_LINUX");
1023 #endif
1024
1025 #if defined(OS_MACOSX)
1026 results_tracker_.AddGlobalTag("OS_MACOSX");
1027 #endif
1028
1029 #if defined(OS_NACL)
1030 results_tracker_.AddGlobalTag("OS_NACL");
1031 #endif
1032
1033 #if defined(OS_OPENBSD)
1034 results_tracker_.AddGlobalTag("OS_OPENBSD");
1035 #endif
1036
1037 #if defined(OS_POSIX)
1038 results_tracker_.AddGlobalTag("OS_POSIX");
1039 #endif
1040
1041 #if defined(OS_SOLARIS)
1042 results_tracker_.AddGlobalTag("OS_SOLARIS");
1043 #endif
1044
1045 #if defined(OS_WIN)
1046 results_tracker_.AddGlobalTag("OS_WIN");
1047 #endif
1048
1049 // CPU-related tags.
1050 #if defined(ARCH_CPU_32_BITS)
1051 results_tracker_.AddGlobalTag("CPU_32_BITS");
1052 #endif
1053
1054 #if defined(ARCH_CPU_64_BITS)
1055 results_tracker_.AddGlobalTag("CPU_64_BITS");
1056 #endif
1057
1058 return true;
1059 }
1060
CombinePositiveTestFilters(std::vector<std::string> filter_a,std::vector<std::string> filter_b)1061 void TestLauncher::CombinePositiveTestFilters(
1062 std::vector<std::string> filter_a,
1063 std::vector<std::string> filter_b) {
1064 has_at_least_one_positive_filter_ = !filter_a.empty() || !filter_b.empty();
1065 if (!has_at_least_one_positive_filter_) {
1066 return;
1067 }
1068 // If two positive filters are present, only run tests that match a pattern
1069 // in both filters.
1070 if (!filter_a.empty() && !filter_b.empty()) {
1071 for (size_t i = 0; i < tests_.size(); i++) {
1072 std::string test_name =
1073 FormatFullTestName(tests_[i].test_case_name, tests_[i].test_name);
1074 bool found_a = false;
1075 bool found_b = false;
1076 for (size_t k = 0; k < filter_a.size(); ++k) {
1077 found_a = found_a || MatchPattern(test_name, filter_a[k]);
1078 }
1079 for (size_t k = 0; k < filter_b.size(); ++k) {
1080 found_b = found_b || MatchPattern(test_name, filter_b[k]);
1081 }
1082 if (found_a && found_b) {
1083 positive_test_filter_.push_back(test_name);
1084 }
1085 }
1086 } else if (!filter_a.empty()) {
1087 positive_test_filter_ = std::move(filter_a);
1088 } else {
1089 positive_test_filter_ = std::move(filter_b);
1090 }
1091 }
1092
RunTests()1093 void TestLauncher::RunTests() {
1094 std::vector<std::string> test_names;
1095 const CommandLine* command_line = CommandLine::ForCurrentProcess();
1096 for (const TestIdentifier& test_id : tests_) {
1097 std::string test_name =
1098 FormatFullTestName(test_id.test_case_name, test_id.test_name);
1099
1100 results_tracker_.AddTest(test_name);
1101
1102 if (test_name.find("DISABLED") != std::string::npos) {
1103 results_tracker_.AddDisabledTest(test_name);
1104
1105 // Skip disabled tests unless explicitly requested.
1106 if (!command_line->HasSwitch(kGTestRunDisabledTestsFlag))
1107 continue;
1108 }
1109
1110 if (!launcher_delegate_->ShouldRunTest(test_id.test_case_name,
1111 test_id.test_name)) {
1112 continue;
1113 }
1114
1115 // Count tests in the binary, before we apply filter and sharding.
1116 test_found_count_++;
1117
1118 std::string test_name_no_disabled =
1119 TestNameWithoutDisabledPrefix(test_name);
1120
1121 // Skip the test that doesn't match the filter (if given).
1122 if (has_at_least_one_positive_filter_) {
1123 bool found = false;
1124 for (auto filter : positive_test_filter_) {
1125 if (MatchPattern(test_name, filter) ||
1126 MatchPattern(test_name_no_disabled, filter)) {
1127 found = true;
1128 break;
1129 }
1130 }
1131
1132 if (!found)
1133 continue;
1134 }
1135 if (!negative_test_filter_.empty()) {
1136 bool excluded = false;
1137 for (auto filter : negative_test_filter_) {
1138 if (MatchPattern(test_name, filter) ||
1139 MatchPattern(test_name_no_disabled, filter)) {
1140 excluded = true;
1141 break;
1142 }
1143 }
1144
1145 if (excluded)
1146 continue;
1147 }
1148
1149 if (Hash(test_name) % total_shards_ != static_cast<uint32_t>(shard_index_))
1150 continue;
1151
1152 // Report test locations after applying all filters, so that we report test
1153 // locations only for those tests that were run as part of this shard.
1154 results_tracker_.AddTestLocation(test_name, test_id.file, test_id.line);
1155
1156 test_names.push_back(test_name);
1157 }
1158
1159 if (shuffle_) {
1160 std::mt19937 randomizer;
1161 randomizer.seed(shuffle_seed_);
1162 std::shuffle(test_names.begin(), test_names.end(), randomizer);
1163
1164 fprintf(stdout, "Randomizing with seed %u\n", shuffle_seed_);
1165 fflush(stdout);
1166 }
1167
1168 // Save an early test summary in case the launcher crashes or gets killed.
1169 MaybeSaveSummaryAsJSON({"EARLY_SUMMARY", kUnreliableResultsTag});
1170
1171 test_started_count_ = launcher_delegate_->RunTests(this, test_names);
1172
1173 if (test_started_count_ == 0) {
1174 fprintf(stdout, "0 tests run\n");
1175 fflush(stdout);
1176
1177 // No tests have actually been started, so kick off the next iteration.
1178 ThreadTaskRunnerHandle::Get()->PostTask(
1179 FROM_HERE, BindOnce(&TestLauncher::RunTestIteration, Unretained(this)));
1180 }
1181 }
1182
RunTestIteration()1183 void TestLauncher::RunTestIteration() {
1184 const bool stop_on_failure =
1185 CommandLine::ForCurrentProcess()->HasSwitch(kGTestBreakOnFailure);
1186 if (cycles_ == 0 ||
1187 (stop_on_failure && test_success_count_ != test_finished_count_)) {
1188 RunLoop::QuitCurrentWhenIdleDeprecated();
1189 return;
1190 }
1191
1192 // Special value "-1" means "repeat indefinitely".
1193 cycles_ = (cycles_ == -1) ? cycles_ : cycles_ - 1;
1194
1195 test_found_count_ = 0;
1196 test_started_count_ = 0;
1197 test_finished_count_ = 0;
1198 test_success_count_ = 0;
1199 test_broken_count_ = 0;
1200 retry_count_ = 0;
1201 tests_to_retry_.clear();
1202 results_tracker_.OnTestIterationStarting();
1203
1204 ThreadTaskRunnerHandle::Get()->PostTask(
1205 FROM_HERE, BindOnce(&TestLauncher::RunTests, Unretained(this)));
1206 }
1207
1208 #if defined(OS_POSIX)
1209 // I/O watcher for the reading end of the self-pipe above.
1210 // Terminates any launched child processes and exits the process.
OnShutdownPipeReadable()1211 void TestLauncher::OnShutdownPipeReadable() {
1212 fprintf(stdout, "\nCaught signal. Killing spawned test processes...\n");
1213 fflush(stdout);
1214
1215 KillSpawnedTestProcesses();
1216
1217 MaybeSaveSummaryAsJSON({"CAUGHT_TERMINATION_SIGNAL", kUnreliableResultsTag});
1218
1219 // The signal would normally kill the process, so exit now.
1220 _exit(1);
1221 }
1222 #endif // defined(OS_POSIX)
1223
MaybeSaveSummaryAsJSON(const std::vector<std::string> & additional_tags)1224 void TestLauncher::MaybeSaveSummaryAsJSON(
1225 const std::vector<std::string>& additional_tags) {
1226 const CommandLine* command_line = CommandLine::ForCurrentProcess();
1227 if (command_line->HasSwitch(switches::kTestLauncherSummaryOutput)) {
1228 FilePath summary_path(command_line->GetSwitchValuePath(
1229 switches::kTestLauncherSummaryOutput));
1230 if (!results_tracker_.SaveSummaryAsJSON(summary_path, additional_tags)) {
1231 LOG(ERROR) << "Failed to save test launcher output summary.";
1232 }
1233 }
1234 if (command_line->HasSwitch(switches::kTestLauncherTrace)) {
1235 FilePath trace_path(
1236 command_line->GetSwitchValuePath(switches::kTestLauncherTrace));
1237 if (!GetTestLauncherTracer()->Dump(trace_path)) {
1238 LOG(ERROR) << "Failed to save test launcher trace.";
1239 }
1240 }
1241 }
1242
OnTestIterationFinished()1243 void TestLauncher::OnTestIterationFinished() {
1244 TestResultsTracker::TestStatusMap tests_by_status(
1245 results_tracker_.GetTestStatusMapForCurrentIteration());
1246 if (!tests_by_status[TestResult::TEST_UNKNOWN].empty())
1247 results_tracker_.AddGlobalTag(kUnreliableResultsTag);
1248
1249 // When we retry tests, success is determined by having nothing more
1250 // to retry (everything eventually passed), as opposed to having
1251 // no failures at all.
1252 if (tests_to_retry_.empty()) {
1253 fprintf(stdout, "SUCCESS: all tests passed.\n");
1254 fflush(stdout);
1255 } else {
1256 // Signal failure, but continue to run all requested test iterations.
1257 // With the summary of all iterations at the end this is a good default.
1258 run_result_ = false;
1259 }
1260
1261 results_tracker_.PrintSummaryOfCurrentIteration();
1262
1263 // Kick off the next iteration.
1264 ThreadTaskRunnerHandle::Get()->PostTask(
1265 FROM_HERE, BindOnce(&TestLauncher::RunTestIteration, Unretained(this)));
1266 }
1267
OnOutputTimeout()1268 void TestLauncher::OnOutputTimeout() {
1269 DCHECK(thread_checker_.CalledOnValidThread());
1270
1271 AutoLock lock(*GetLiveProcessesLock());
1272
1273 fprintf(stdout, "Still waiting for the following processes to finish:\n");
1274
1275 for (const auto& pair : *GetLiveProcesses()) {
1276 #if defined(OS_WIN)
1277 fwprintf(stdout, L"\t%s\n", pair.second.GetCommandLineString().c_str());
1278 #else
1279 fprintf(stdout, "\t%s\n", pair.second.GetCommandLineString().c_str());
1280 #endif
1281 }
1282
1283 fflush(stdout);
1284
1285 // Arm the timer again - otherwise it would fire only once.
1286 watchdog_timer_.Reset();
1287 }
1288
NumParallelJobs()1289 size_t NumParallelJobs() {
1290 const CommandLine* command_line = CommandLine::ForCurrentProcess();
1291 if (command_line->HasSwitch(switches::kTestLauncherJobs)) {
1292 // If the number of test launcher jobs was specified, return that number.
1293 size_t jobs = 0U;
1294
1295 if (!StringToSizeT(
1296 command_line->GetSwitchValueASCII(switches::kTestLauncherJobs),
1297 &jobs) ||
1298 !jobs) {
1299 LOG(ERROR) << "Invalid value for " << switches::kTestLauncherJobs;
1300 return 0U;
1301 }
1302 return jobs;
1303 }
1304 if (command_line->HasSwitch(kGTestFilterFlag) && !BotModeEnabled()) {
1305 // Do not run jobs in parallel by default if we are running a subset of
1306 // the tests and if bot mode is off.
1307 return 1U;
1308 }
1309
1310 // Default to the number of processor cores.
1311 return base::checked_cast<size_t>(SysInfo::NumberOfProcessors());
1312 }
1313
GetTestOutputSnippet(const TestResult & result,const std::string & full_output)1314 std::string GetTestOutputSnippet(const TestResult& result,
1315 const std::string& full_output) {
1316 size_t run_pos = full_output.find(std::string("[ RUN ] ") +
1317 result.full_name);
1318 if (run_pos == std::string::npos)
1319 return std::string();
1320
1321 size_t end_pos = full_output.find(std::string("[ FAILED ] ") +
1322 result.full_name,
1323 run_pos);
1324 // Only clip the snippet to the "OK" message if the test really
1325 // succeeded. It still might have e.g. crashed after printing it.
1326 if (end_pos == std::string::npos &&
1327 result.status == TestResult::TEST_SUCCESS) {
1328 end_pos = full_output.find(std::string("[ OK ] ") +
1329 result.full_name,
1330 run_pos);
1331 }
1332 if (end_pos != std::string::npos) {
1333 size_t newline_pos = full_output.find("\n", end_pos);
1334 if (newline_pos != std::string::npos)
1335 end_pos = newline_pos + 1;
1336 }
1337
1338 std::string snippet(full_output.substr(run_pos));
1339 if (end_pos != std::string::npos)
1340 snippet = full_output.substr(run_pos, end_pos - run_pos);
1341
1342 return snippet;
1343 }
1344
1345 } // namespace base
1346