• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "base/test/launcher/test_launcher.h"
6 
7 #include <stdio.h>
8 
9 #include <algorithm>
10 #include <map>
11 #include <random>
12 #include <utility>
13 
14 #include "base/at_exit.h"
15 #include "base/bind.h"
16 #include "base/command_line.h"
17 #include "base/environment.h"
18 #include "base/files/file_path.h"
19 #include "base/files/file_util.h"
20 #include "base/files/scoped_file.h"
21 #include "base/format_macros.h"
22 #include "base/hash.h"
23 #include "base/lazy_instance.h"
24 #include "base/location.h"
25 #include "base/logging.h"
26 #include "base/macros.h"
27 #include "base/memory/ptr_util.h"
28 #include "base/numerics/safe_conversions.h"
29 #include "base/process/kill.h"
30 #include "base/process/launch.h"
31 #include "base/run_loop.h"
32 #include "base/single_thread_task_runner.h"
33 #include "base/strings/pattern.h"
34 #include "base/strings/string_number_conversions.h"
35 #include "base/strings/string_piece.h"
36 #include "base/strings/string_split.h"
37 #include "base/strings/string_util.h"
38 #include "base/strings/stringize_macros.h"
39 #include "base/strings/stringprintf.h"
40 #include "base/strings/utf_string_conversions.h"
41 #include "base/sys_info.h"
42 #include "base/task_scheduler/post_task.h"
43 #include "base/task_scheduler/task_scheduler.h"
44 #include "base/test/gtest_util.h"
45 #include "base/test/launcher/test_launcher_tracer.h"
46 #include "base/test/launcher/test_results_tracker.h"
47 #include "base/test/test_switches.h"
48 #include "base/test/test_timeouts.h"
49 #include "base/threading/thread_restrictions.h"
50 #include "base/threading/thread_task_runner_handle.h"
51 #include "base/time/time.h"
52 #include "build/build_config.h"
53 #include "testing/gtest/include/gtest/gtest.h"
54 
55 #if defined(OS_POSIX)
56 #include <fcntl.h>
57 
58 #include "base/files/file_descriptor_watcher_posix.h"
59 #endif
60 
61 #if defined(OS_MACOSX)
62 #include "base/mac/scoped_nsautorelease_pool.h"
63 #endif
64 
65 #if defined(OS_WIN)
66 #include "base/win/windows_version.h"
67 #endif
68 
69 #if defined(OS_FUCHSIA)
70 #include <lib/zx/job.h>
71 #include "base/fuchsia/default_job.h"
72 #include "base/fuchsia/fuchsia_logging.h"
73 #endif
74 
75 namespace base {
76 
77 // See https://groups.google.com/a/chromium.org/d/msg/chromium-dev/nkdTP7sstSc/uT3FaE_sgkAJ .
78 using ::operator<<;
79 
80 // The environment variable name for the total number of test shards.
81 const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS";
82 // The environment variable name for the test shard index.
83 const char kTestShardIndex[] = "GTEST_SHARD_INDEX";
84 
85 namespace {
86 
87 // Global tag for test runs where the results are incomplete or unreliable
88 // for any reason, e.g. early exit because of too many broken tests.
89 const char kUnreliableResultsTag[] = "UNRELIABLE_RESULTS";
90 
91 // Maximum time of no output after which we print list of processes still
92 // running. This deliberately doesn't use TestTimeouts (which is otherwise
93 // a recommended solution), because they can be increased. This would defeat
94 // the purpose of this timeout, which is 1) to avoid buildbot "no output for
95 // X seconds" timeout killing the process 2) help communicate status of
96 // the test launcher to people looking at the output (no output for a long
97 // time is mysterious and gives no info about what is happening) 3) help
98 // debugging in case the process hangs anyway.
99 constexpr TimeDelta kOutputTimeout = TimeDelta::FromSeconds(15);
100 
101 // Limit of output snippet lines when printing to stdout.
102 // Avoids flooding the logs with amount of output that gums up
103 // the infrastructure.
104 const size_t kOutputSnippetLinesLimit = 5000;
105 
106 // Limit of output snippet size. Exceeding this limit
107 // results in truncating the output and failing the test.
108 const size_t kOutputSnippetBytesLimit = 300 * 1024;
109 
110 // Limit of seed values for gtest shuffling. Arbitrary, but based on
111 // gtest's similarly arbitrary choice.
112 const uint32_t kRandomSeedUpperBound = 100000;
113 
114 // Set of live launch test processes with corresponding lock (it is allowed
115 // for callers to launch processes on different threads).
GetLiveProcessesLock()116 Lock* GetLiveProcessesLock() {
117   static auto* lock = new Lock;
118   return lock;
119 }
120 
GetLiveProcesses()121 std::map<ProcessHandle, CommandLine>* GetLiveProcesses() {
122   static auto* map = new std::map<ProcessHandle, CommandLine>;
123   return map;
124 }
125 
126 // Performance trace generator.
GetTestLauncherTracer()127 TestLauncherTracer* GetTestLauncherTracer() {
128   static auto* tracer = new TestLauncherTracer;
129   return tracer;
130 }
131 
132 // Creates and starts a TaskScheduler with |num_parallel_jobs| dedicated to
133 // foreground blocking tasks (corresponds to the traits used to launch and wait
134 // for child processes).
CreateAndStartTaskScheduler(int num_parallel_jobs)135 void CreateAndStartTaskScheduler(int num_parallel_jobs) {
136   // These values are taken from TaskScheduler::StartWithDefaultParams(), which
137   // is not used directly to allow a custom number of threads in the foreground
138   // blocking pool.
139   constexpr int kMaxBackgroundThreads = 1;
140   constexpr int kMaxBackgroundBlockingThreads = 2;
141   const int max_foreground_threads =
142       std::max(1, base::SysInfo::NumberOfProcessors());
143   constexpr base::TimeDelta kSuggestedReclaimTime =
144       base::TimeDelta::FromSeconds(30);
145   base::TaskScheduler::Create("TestLauncher");
146   base::TaskScheduler::GetInstance()->Start(
147       {{kMaxBackgroundThreads, kSuggestedReclaimTime},
148        {kMaxBackgroundBlockingThreads, kSuggestedReclaimTime},
149        {max_foreground_threads, kSuggestedReclaimTime},
150        {num_parallel_jobs, kSuggestedReclaimTime}});
151 }
152 
153 #if defined(OS_POSIX)
154 // Self-pipe that makes it possible to do complex shutdown handling
155 // outside of the signal handler.
156 int g_shutdown_pipe[2] = { -1, -1 };
157 
ShutdownPipeSignalHandler(int signal)158 void ShutdownPipeSignalHandler(int signal) {
159   HANDLE_EINTR(write(g_shutdown_pipe[1], "q", 1));
160 }
161 
KillSpawnedTestProcesses()162 void KillSpawnedTestProcesses() {
163   // Keep the lock until exiting the process to prevent further processes
164   // from being spawned.
165   AutoLock lock(*GetLiveProcessesLock());
166 
167   fprintf(stdout, "Sending SIGTERM to %" PRIuS " child processes... ",
168           GetLiveProcesses()->size());
169   fflush(stdout);
170 
171   for (const auto& pair : *GetLiveProcesses()) {
172     // Send the signal to entire process group.
173     kill((-1) * (pair.first), SIGTERM);
174   }
175 
176   fprintf(stdout,
177           "done.\nGiving processes a chance to terminate cleanly... ");
178   fflush(stdout);
179 
180   PlatformThread::Sleep(TimeDelta::FromMilliseconds(500));
181 
182   fprintf(stdout, "done.\n");
183   fflush(stdout);
184 
185   fprintf(stdout, "Sending SIGKILL to %" PRIuS " child processes... ",
186           GetLiveProcesses()->size());
187   fflush(stdout);
188 
189   for (const auto& pair : *GetLiveProcesses()) {
190     // Send the signal to entire process group.
191     kill((-1) * (pair.first), SIGKILL);
192   }
193 
194   fprintf(stdout, "done.\n");
195   fflush(stdout);
196 }
197 #endif  // defined(OS_POSIX)
198 
199 // Parses the environment variable var as an Int32.  If it is unset, returns
200 // true.  If it is set, unsets it then converts it to Int32 before
201 // returning it in |result|.  Returns true on success.
TakeInt32FromEnvironment(const char * const var,int32_t * result)202 bool TakeInt32FromEnvironment(const char* const var, int32_t* result) {
203   std::unique_ptr<Environment> env(Environment::Create());
204   std::string str_val;
205 
206   if (!env->GetVar(var, &str_val))
207     return true;
208 
209   if (!env->UnSetVar(var)) {
210     LOG(ERROR) << "Invalid environment: we could not unset " << var << ".\n";
211     return false;
212   }
213 
214   if (!StringToInt(str_val, result)) {
215     LOG(ERROR) << "Invalid environment: " << var << " is not an integer.\n";
216     return false;
217   }
218 
219   return true;
220 }
221 
222 // Unsets the environment variable |name| and returns true on success.
223 // Also returns true if the variable just doesn't exist.
UnsetEnvironmentVariableIfExists(const std::string & name)224 bool UnsetEnvironmentVariableIfExists(const std::string& name) {
225   std::unique_ptr<Environment> env(Environment::Create());
226   std::string str_val;
227   if (!env->GetVar(name, &str_val))
228     return true;
229   return env->UnSetVar(name);
230 }
231 
232 // Returns true if bot mode has been requested, i.e. defaults optimized
233 // for continuous integration bots. This way developers don't have to remember
234 // special command-line flags.
BotModeEnabled()235 bool BotModeEnabled() {
236   std::unique_ptr<Environment> env(Environment::Create());
237   return CommandLine::ForCurrentProcess()->HasSwitch(
238       switches::kTestLauncherBotMode) ||
239       env->HasVar("CHROMIUM_TEST_LAUNCHER_BOT_MODE");
240 }
241 
242 // Returns command line command line after gtest-specific processing
243 // and applying |wrapper|.
PrepareCommandLineForGTest(const CommandLine & command_line,const std::string & wrapper)244 CommandLine PrepareCommandLineForGTest(const CommandLine& command_line,
245                                        const std::string& wrapper) {
246   CommandLine new_command_line(command_line.GetProgram());
247   CommandLine::SwitchMap switches = command_line.GetSwitches();
248 
249   // Handled by the launcher process.
250   switches.erase(kGTestRepeatFlag);
251   switches.erase(kGTestShuffleFlag);
252   switches.erase(kGTestRandomSeedFlag);
253 
254   // Don't try to write the final XML report in child processes.
255   switches.erase(kGTestOutputFlag);
256 
257   for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
258        iter != switches.end(); ++iter) {
259     new_command_line.AppendSwitchNative((*iter).first, (*iter).second);
260   }
261 
262   // Prepend wrapper after last CommandLine quasi-copy operation. CommandLine
263   // does not really support removing switches well, and trying to do that
264   // on a CommandLine with a wrapper is known to break.
265   // TODO(phajdan.jr): Give it a try to support CommandLine removing switches.
266 #if defined(OS_WIN)
267   new_command_line.PrependWrapper(ASCIIToUTF16(wrapper));
268 #else
269   new_command_line.PrependWrapper(wrapper);
270 #endif
271 
272   return new_command_line;
273 }
274 
275 // Launches a child process using |command_line|. If the child process is still
276 // running after |timeout|, it is terminated and |*was_timeout| is set to true.
277 // Returns exit code of the process.
LaunchChildTestProcessWithOptions(const CommandLine & command_line,const LaunchOptions & options,int flags,TimeDelta timeout,ProcessLifetimeObserver * observer,bool * was_timeout)278 int LaunchChildTestProcessWithOptions(const CommandLine& command_line,
279                                       const LaunchOptions& options,
280                                       int flags,
281                                       TimeDelta timeout,
282                                       ProcessLifetimeObserver* observer,
283                                       bool* was_timeout) {
284   TimeTicks start_time(TimeTicks::Now());
285 
286 #if defined(OS_POSIX)
287   // Make sure an option we rely on is present - see LaunchChildGTestProcess.
288   DCHECK(options.new_process_group);
289 #endif
290 
291   LaunchOptions new_options(options);
292 
293 #if defined(OS_WIN)
294   DCHECK(!new_options.job_handle);
295 
296   win::ScopedHandle job_handle;
297   if (flags & TestLauncher::USE_JOB_OBJECTS) {
298     job_handle.Set(CreateJobObject(NULL, NULL));
299     if (!job_handle.IsValid()) {
300       LOG(ERROR) << "Could not create JobObject.";
301       return -1;
302     }
303 
304     DWORD job_flags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
305 
306     // Allow break-away from job since sandbox and few other places rely on it
307     // on Windows versions prior to Windows 8 (which supports nested jobs).
308     if (win::GetVersion() < win::VERSION_WIN8 &&
309         flags & TestLauncher::ALLOW_BREAKAWAY_FROM_JOB) {
310       job_flags |= JOB_OBJECT_LIMIT_BREAKAWAY_OK;
311     }
312 
313     if (!SetJobObjectLimitFlags(job_handle.Get(), job_flags)) {
314       LOG(ERROR) << "Could not SetJobObjectLimitFlags.";
315       return -1;
316     }
317 
318     new_options.job_handle = job_handle.Get();
319   }
320 #elif defined(OS_FUCHSIA)
321   DCHECK(!new_options.job_handle);
322 
323   zx::job job_handle;
324   zx_status_t result = zx::job::create(*GetDefaultJob(), 0, &job_handle);
325   ZX_CHECK(ZX_OK == result, result) << "zx_job_create";
326   new_options.job_handle = job_handle.get();
327 #endif  // defined(OS_FUCHSIA)
328 
329 #if defined(OS_LINUX)
330   // To prevent accidental privilege sharing to an untrusted child, processes
331   // are started with PR_SET_NO_NEW_PRIVS. Do not set that here, since this
332   // new child will be privileged and trusted.
333   new_options.allow_new_privs = true;
334 #endif
335 
336   Process process;
337 
338   {
339     // Note how we grab the lock before the process possibly gets created.
340     // This ensures that when the lock is held, ALL the processes are registered
341     // in the set.
342     AutoLock lock(*GetLiveProcessesLock());
343 
344 #if defined(OS_WIN)
345     // Allow the handle used to capture stdio and stdout to be inherited by the
346     // child. Note that this is done under GetLiveProcessesLock() to ensure that
347     // only the desired child receives the handle.
348     if (new_options.stdout_handle) {
349       ::SetHandleInformation(new_options.stdout_handle, HANDLE_FLAG_INHERIT,
350                              HANDLE_FLAG_INHERIT);
351     }
352 #endif
353 
354     process = LaunchProcess(command_line, new_options);
355 
356 #if defined(OS_WIN)
357     // Revoke inheritance so that the handle isn't leaked into other children.
358     // Note that this is done under GetLiveProcessesLock() to ensure that only
359     // the desired child receives the handle.
360     if (new_options.stdout_handle)
361       ::SetHandleInformation(new_options.stdout_handle, HANDLE_FLAG_INHERIT, 0);
362 #endif
363 
364     if (!process.IsValid())
365       return -1;
366 
367     // TODO(rvargas) crbug.com/417532: Don't store process handles.
368     GetLiveProcesses()->insert(std::make_pair(process.Handle(), command_line));
369   }
370 
371   if (observer)
372     observer->OnLaunched(process.Handle(), process.Pid());
373 
374   int exit_code = 0;
375   bool did_exit = false;
376 
377   {
378     base::ScopedAllowBaseSyncPrimitivesForTesting allow_base_sync_primitives;
379     did_exit = process.WaitForExitWithTimeout(timeout, &exit_code);
380   }
381 
382   if (!did_exit) {
383     if (observer)
384       observer->OnTimedOut(command_line);
385 
386     *was_timeout = true;
387     exit_code = -1;  // Set a non-zero exit code to signal a failure.
388 
389     {
390       base::ScopedAllowBaseSyncPrimitivesForTesting allow_base_sync_primitives;
391       // Ensure that the process terminates.
392       process.Terminate(-1, true);
393     }
394   }
395 
396   {
397     // Note how we grab the log before issuing a possibly broad process kill.
398     // Other code parts that grab the log kill processes, so avoid trying
399     // to do that twice and trigger all kinds of log messages.
400     AutoLock lock(*GetLiveProcessesLock());
401 
402 #if defined(OS_FUCHSIA)
403     zx_status_t status = job_handle.kill();
404     ZX_CHECK(status == ZX_OK, status);
405 #elif defined(OS_POSIX)
406     if (exit_code != 0) {
407       // On POSIX, in case the test does not exit cleanly, either due to a crash
408       // or due to it timing out, we need to clean up any child processes that
409       // it might have created. On Windows, child processes are automatically
410       // cleaned up using JobObjects.
411       KillProcessGroup(process.Handle());
412     }
413 #endif
414 
415     GetLiveProcesses()->erase(process.Handle());
416   }
417 
418   GetTestLauncherTracer()->RecordProcessExecution(
419       start_time, TimeTicks::Now() - start_time);
420 
421   return exit_code;
422 }
423 
DoLaunchChildTestProcess(const CommandLine & command_line,TimeDelta timeout,const TestLauncher::LaunchOptions & test_launch_options,bool redirect_stdio,SingleThreadTaskRunner * task_runner,std::unique_ptr<ProcessLifetimeObserver> observer)424 void DoLaunchChildTestProcess(
425     const CommandLine& command_line,
426     TimeDelta timeout,
427     const TestLauncher::LaunchOptions& test_launch_options,
428     bool redirect_stdio,
429     SingleThreadTaskRunner* task_runner,
430     std::unique_ptr<ProcessLifetimeObserver> observer) {
431   TimeTicks start_time = TimeTicks::Now();
432 
433   ScopedFILE output_file;
434   FilePath output_filename;
435   if (redirect_stdio) {
436     FILE* raw_output_file = CreateAndOpenTemporaryFile(&output_filename);
437     output_file.reset(raw_output_file);
438     CHECK(output_file);
439   }
440 
441   LaunchOptions options;
442 #if defined(OS_WIN)
443 
444   options.inherit_mode = test_launch_options.inherit_mode;
445   options.handles_to_inherit = test_launch_options.handles_to_inherit;
446   if (redirect_stdio) {
447     HANDLE handle =
448         reinterpret_cast<HANDLE>(_get_osfhandle(_fileno(output_file.get())));
449     CHECK_NE(INVALID_HANDLE_VALUE, handle);
450     options.stdin_handle = INVALID_HANDLE_VALUE;
451     options.stdout_handle = handle;
452     options.stderr_handle = handle;
453     // See LaunchOptions.stdout_handle comments for why this compares against
454     // FILE_TYPE_CHAR.
455     if (options.inherit_mode == base::LaunchOptions::Inherit::kSpecific &&
456         GetFileType(handle) != FILE_TYPE_CHAR) {
457       options.handles_to_inherit.push_back(handle);
458     }
459   }
460 
461 #else  // if !defined(OS_WIN)
462 
463   options.fds_to_remap = test_launch_options.fds_to_remap;
464   if (redirect_stdio) {
465     int output_file_fd = fileno(output_file.get());
466     CHECK_LE(0, output_file_fd);
467     options.fds_to_remap.push_back(
468         std::make_pair(output_file_fd, STDOUT_FILENO));
469     options.fds_to_remap.push_back(
470         std::make_pair(output_file_fd, STDERR_FILENO));
471   }
472 
473 #if !defined(OS_FUCHSIA)
474   options.new_process_group = true;
475 #endif
476 #if defined(OS_LINUX)
477   options.kill_on_parent_death = true;
478 #endif
479 
480 #endif  // !defined(OS_WIN)
481 
482   bool was_timeout = false;
483   int exit_code = LaunchChildTestProcessWithOptions(
484       command_line, options, test_launch_options.flags, timeout, observer.get(),
485       &was_timeout);
486 
487   std::string output_file_contents;
488   if (redirect_stdio) {
489     fflush(output_file.get());
490     output_file.reset();
491     // Reading the file can sometimes fail when the process was killed midflight
492     // (e.g. on test suite timeout): https://crbug.com/826408. Attempt to read
493     // the output file anyways, but do not crash on failure in this case.
494     CHECK(ReadFileToString(output_filename, &output_file_contents) ||
495           exit_code != 0);
496 
497     if (!DeleteFile(output_filename, false)) {
498       // This needs to be non-fatal at least for Windows.
499       LOG(WARNING) << "Failed to delete " << output_filename.AsUTF8Unsafe();
500     }
501   }
502 
503   // Invoke OnCompleted on the thread it was originating from, not on a worker
504   // pool thread.
505   task_runner->PostTask(
506       FROM_HERE,
507       BindOnce(&ProcessLifetimeObserver::OnCompleted, std::move(observer),
508                exit_code, TimeTicks::Now() - start_time, was_timeout,
509                output_file_contents));
510 }
511 
512 }  // namespace
513 
514 const char kGTestBreakOnFailure[] = "gtest_break_on_failure";
515 const char kGTestFilterFlag[] = "gtest_filter";
516 const char kGTestFlagfileFlag[] = "gtest_flagfile";
517 const char kGTestHelpFlag[]   = "gtest_help";
518 const char kGTestListTestsFlag[] = "gtest_list_tests";
519 const char kGTestRepeatFlag[] = "gtest_repeat";
520 const char kGTestRunDisabledTestsFlag[] = "gtest_also_run_disabled_tests";
521 const char kGTestOutputFlag[] = "gtest_output";
522 const char kGTestShuffleFlag[] = "gtest_shuffle";
523 const char kGTestRandomSeedFlag[] = "gtest_random_seed";
524 
525 TestLauncherDelegate::~TestLauncherDelegate() = default;
526 
527 TestLauncher::LaunchOptions::LaunchOptions() = default;
528 TestLauncher::LaunchOptions::LaunchOptions(const LaunchOptions& other) =
529     default;
530 TestLauncher::LaunchOptions::~LaunchOptions() = default;
531 
TestLauncher(TestLauncherDelegate * launcher_delegate,size_t parallel_jobs)532 TestLauncher::TestLauncher(TestLauncherDelegate* launcher_delegate,
533                            size_t parallel_jobs)
534     : launcher_delegate_(launcher_delegate),
535       total_shards_(1),
536       shard_index_(0),
537       cycles_(1),
538       test_found_count_(0),
539       test_started_count_(0),
540       test_finished_count_(0),
541       test_success_count_(0),
542       test_broken_count_(0),
543       retry_count_(0),
544       retry_limit_(0),
545       force_run_broken_tests_(false),
546       run_result_(true),
547       shuffle_(false),
548       shuffle_seed_(0),
549       watchdog_timer_(FROM_HERE,
550                       kOutputTimeout,
551                       this,
552                       &TestLauncher::OnOutputTimeout),
553       parallel_jobs_(parallel_jobs) {}
554 
~TestLauncher()555 TestLauncher::~TestLauncher() {
556   if (base::TaskScheduler::GetInstance()) {
557     base::TaskScheduler::GetInstance()->Shutdown();
558   }
559 }
560 
Run()561 bool TestLauncher::Run() {
562   if (!Init())
563     return false;
564 
565   // Value of |cycles_| changes after each iteration. Keep track of the
566   // original value.
567   int requested_cycles = cycles_;
568 
569 #if defined(OS_POSIX)
570   CHECK_EQ(0, pipe(g_shutdown_pipe));
571 
572   struct sigaction action;
573   memset(&action, 0, sizeof(action));
574   sigemptyset(&action.sa_mask);
575   action.sa_handler = &ShutdownPipeSignalHandler;
576 
577   CHECK_EQ(0, sigaction(SIGINT, &action, nullptr));
578   CHECK_EQ(0, sigaction(SIGQUIT, &action, nullptr));
579   CHECK_EQ(0, sigaction(SIGTERM, &action, nullptr));
580 
581   auto controller = base::FileDescriptorWatcher::WatchReadable(
582       g_shutdown_pipe[0],
583       base::Bind(&TestLauncher::OnShutdownPipeReadable, Unretained(this)));
584 #endif  // defined(OS_POSIX)
585 
586   // Start the watchdog timer.
587   watchdog_timer_.Reset();
588 
589   ThreadTaskRunnerHandle::Get()->PostTask(
590       FROM_HERE, BindOnce(&TestLauncher::RunTestIteration, Unretained(this)));
591 
592   RunLoop().Run();
593 
594   if (requested_cycles != 1)
595     results_tracker_.PrintSummaryOfAllIterations();
596 
597   MaybeSaveSummaryAsJSON(std::vector<std::string>());
598 
599   return run_result_;
600 }
601 
LaunchChildGTestProcess(const CommandLine & command_line,const std::string & wrapper,TimeDelta timeout,const LaunchOptions & options,std::unique_ptr<ProcessLifetimeObserver> observer)602 void TestLauncher::LaunchChildGTestProcess(
603     const CommandLine& command_line,
604     const std::string& wrapper,
605     TimeDelta timeout,
606     const LaunchOptions& options,
607     std::unique_ptr<ProcessLifetimeObserver> observer) {
608   DCHECK(thread_checker_.CalledOnValidThread());
609 
610   // Record the exact command line used to launch the child.
611   CommandLine new_command_line(
612       PrepareCommandLineForGTest(command_line, wrapper));
613 
614   // When running in parallel mode we need to redirect stdio to avoid mixed-up
615   // output. We also always redirect on the bots to get the test output into
616   // JSON summary.
617   bool redirect_stdio = (parallel_jobs_ > 1) || BotModeEnabled();
618 
619   PostTaskWithTraits(
620       FROM_HERE, {MayBlock(), TaskShutdownBehavior::BLOCK_SHUTDOWN},
621       BindOnce(&DoLaunchChildTestProcess, new_command_line, timeout, options,
622                redirect_stdio, RetainedRef(ThreadTaskRunnerHandle::Get()),
623                std::move(observer)));
624 }
625 
OnTestFinished(const TestResult & original_result)626 void TestLauncher::OnTestFinished(const TestResult& original_result) {
627   ++test_finished_count_;
628 
629   TestResult result(original_result);
630 
631   if (result.output_snippet.length() > kOutputSnippetBytesLimit) {
632     if (result.status == TestResult::TEST_SUCCESS)
633       result.status = TestResult::TEST_EXCESSIVE_OUTPUT;
634 
635     // Keep the top and bottom of the log and truncate the middle part.
636     result.output_snippet =
637         result.output_snippet.substr(0, kOutputSnippetBytesLimit / 2) + "\n" +
638         StringPrintf("<truncated (%" PRIuS " bytes)>\n",
639                      result.output_snippet.length()) +
640         result.output_snippet.substr(result.output_snippet.length() -
641                                      kOutputSnippetBytesLimit / 2) +
642         "\n";
643   }
644 
645   bool print_snippet = false;
646   std::string print_test_stdio("auto");
647   if (CommandLine::ForCurrentProcess()->HasSwitch(
648           switches::kTestLauncherPrintTestStdio)) {
649     print_test_stdio = CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
650         switches::kTestLauncherPrintTestStdio);
651   }
652   if (print_test_stdio == "auto") {
653     print_snippet = (result.status != TestResult::TEST_SUCCESS);
654   } else if (print_test_stdio == "always") {
655     print_snippet = true;
656   } else if (print_test_stdio == "never") {
657     print_snippet = false;
658   } else {
659     LOG(WARNING) << "Invalid value of " << switches::kTestLauncherPrintTestStdio
660                  << ": " << print_test_stdio;
661   }
662   if (print_snippet) {
663     std::vector<base::StringPiece> snippet_lines =
664         SplitStringPiece(result.output_snippet, "\n", base::KEEP_WHITESPACE,
665                          base::SPLIT_WANT_ALL);
666     if (snippet_lines.size() > kOutputSnippetLinesLimit) {
667       size_t truncated_size = snippet_lines.size() - kOutputSnippetLinesLimit;
668       snippet_lines.erase(
669           snippet_lines.begin(),
670           snippet_lines.begin() + truncated_size);
671       snippet_lines.insert(snippet_lines.begin(), "<truncated>");
672     }
673     fprintf(stdout, "%s", base::JoinString(snippet_lines, "\n").c_str());
674     fflush(stdout);
675   }
676 
677   if (result.status == TestResult::TEST_SUCCESS) {
678     ++test_success_count_;
679   } else {
680     tests_to_retry_.insert(result.full_name);
681   }
682 
683   results_tracker_.AddTestResult(result);
684 
685   // TODO(phajdan.jr): Align counter (padding).
686   std::string status_line(
687       StringPrintf("[%" PRIuS "/%" PRIuS "] %s ",
688                    test_finished_count_,
689                    test_started_count_,
690                    result.full_name.c_str()));
691   if (result.completed()) {
692     status_line.append(StringPrintf("(%" PRId64 " ms)",
693                                     result.elapsed_time.InMilliseconds()));
694   } else if (result.status == TestResult::TEST_TIMEOUT) {
695     status_line.append("(TIMED OUT)");
696   } else if (result.status == TestResult::TEST_CRASH) {
697     status_line.append("(CRASHED)");
698   } else if (result.status == TestResult::TEST_SKIPPED) {
699     status_line.append("(SKIPPED)");
700   } else if (result.status == TestResult::TEST_UNKNOWN) {
701     status_line.append("(UNKNOWN)");
702   } else {
703     // Fail very loudly so it's not ignored.
704     CHECK(false) << "Unhandled test result status: " << result.status;
705   }
706   fprintf(stdout, "%s\n", status_line.c_str());
707   fflush(stdout);
708 
709   // We just printed a status line, reset the watchdog timer.
710   watchdog_timer_.Reset();
711 
712   // Do not waste time on timeouts. We include tests with unknown results here
713   // because sometimes (e.g. hang in between unit tests) that's how a timeout
714   // gets reported.
715   if (result.status == TestResult::TEST_TIMEOUT ||
716       result.status == TestResult::TEST_UNKNOWN) {
717     test_broken_count_++;
718   }
719   size_t broken_threshold =
720       std::max(static_cast<size_t>(20), test_found_count_ / 10);
721   if (!force_run_broken_tests_ && test_broken_count_ >= broken_threshold) {
722     fprintf(stdout, "Too many badly broken tests (%" PRIuS "), exiting now.\n",
723             test_broken_count_);
724     fflush(stdout);
725 
726 #if defined(OS_POSIX)
727     KillSpawnedTestProcesses();
728 #endif  // defined(OS_POSIX)
729 
730     MaybeSaveSummaryAsJSON({"BROKEN_TEST_EARLY_EXIT", kUnreliableResultsTag});
731 
732     exit(1);
733   }
734 
735   if (test_finished_count_ != test_started_count_)
736     return;
737 
738   if (tests_to_retry_.empty() || retry_count_ >= retry_limit_) {
739     OnTestIterationFinished();
740     return;
741   }
742 
743   if (!force_run_broken_tests_ && tests_to_retry_.size() >= broken_threshold) {
744     fprintf(stdout,
745             "Too many failing tests (%" PRIuS "), skipping retries.\n",
746             tests_to_retry_.size());
747     fflush(stdout);
748 
749     results_tracker_.AddGlobalTag("BROKEN_TEST_SKIPPED_RETRIES");
750     results_tracker_.AddGlobalTag(kUnreliableResultsTag);
751 
752     OnTestIterationFinished();
753     return;
754   }
755 
756   retry_count_++;
757 
758   std::vector<std::string> test_names(tests_to_retry_.begin(),
759                                       tests_to_retry_.end());
760 
761   tests_to_retry_.clear();
762 
763   size_t retry_started_count = launcher_delegate_->RetryTests(this, test_names);
764   if (retry_started_count == 0) {
765     // Signal failure, but continue to run all requested test iterations.
766     // With the summary of all iterations at the end this is a good default.
767     run_result_ = false;
768 
769     OnTestIterationFinished();
770     return;
771   }
772 
773   fprintf(stdout, "Retrying %" PRIuS " test%s (retry #%" PRIuS ")\n",
774           retry_started_count,
775           retry_started_count > 1 ? "s" : "",
776           retry_count_);
777   fflush(stdout);
778 
779   test_started_count_ += retry_started_count;
780 }
781 
782 // Helper used to parse test filter files. Syntax is documented in
783 // //testing/buildbot/filters/README.md .
LoadFilterFile(const FilePath & file_path,std::vector<std::string> * positive_filter,std::vector<std::string> * negative_filter)784 bool LoadFilterFile(const FilePath& file_path,
785                     std::vector<std::string>* positive_filter,
786                     std::vector<std::string>* negative_filter) {
787   std::string file_content;
788   if (!ReadFileToString(file_path, &file_content)) {
789     LOG(ERROR) << "Failed to read the filter file.";
790     return false;
791   }
792 
793   std::vector<std::string> filter_lines = SplitString(
794       file_content, "\n", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
795   int line_num = 0;
796   for (const std::string& filter_line : filter_lines) {
797     line_num++;
798 
799     size_t hash_pos = filter_line.find('#');
800 
801     // In case when # symbol is not in the beginning of the line and is not
802     // proceeded with a space then it's likely that the comment was
803     // unintentional.
804     if (hash_pos != std::string::npos && hash_pos > 0 &&
805         filter_line[hash_pos - 1] != ' ') {
806       LOG(WARNING) << "Content of line " << line_num << " in " << file_path
807                    << " after # is treated as a comment, " << filter_line;
808     }
809 
810     // Strip comments and whitespace from each line.
811     std::string trimmed_line =
812         TrimWhitespaceASCII(filter_line.substr(0, hash_pos), TRIM_ALL)
813             .as_string();
814 
815     if (trimmed_line.substr(0, 2) == "//") {
816       LOG(ERROR) << "Line " << line_num << " in " << file_path
817                  << " starts with //, use # for comments.";
818       return false;
819     }
820 
821     // Treat a line starting with '//' as a comment.
822     if (trimmed_line.empty())
823       continue;
824 
825     if (trimmed_line[0] == '-')
826       negative_filter->push_back(trimmed_line.substr(1));
827     else
828       positive_filter->push_back(trimmed_line);
829   }
830 
831   return true;
832 }
833 
Init()834 bool TestLauncher::Init() {
835   const CommandLine* command_line = CommandLine::ForCurrentProcess();
836 
837   // Initialize sharding. Command line takes precedence over legacy environment
838   // variables.
839   if (command_line->HasSwitch(switches::kTestLauncherTotalShards) &&
840       command_line->HasSwitch(switches::kTestLauncherShardIndex)) {
841     if (!StringToInt(
842             command_line->GetSwitchValueASCII(
843                 switches::kTestLauncherTotalShards),
844             &total_shards_)) {
845       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherTotalShards;
846       return false;
847     }
848     if (!StringToInt(
849             command_line->GetSwitchValueASCII(
850                 switches::kTestLauncherShardIndex),
851             &shard_index_)) {
852       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherShardIndex;
853       return false;
854     }
855     fprintf(stdout,
856             "Using sharding settings from command line. This is shard %d/%d\n",
857             shard_index_, total_shards_);
858     fflush(stdout);
859   } else {
860     if (!TakeInt32FromEnvironment(kTestTotalShards, &total_shards_))
861       return false;
862     if (!TakeInt32FromEnvironment(kTestShardIndex, &shard_index_))
863       return false;
864     fprintf(stdout,
865             "Using sharding settings from environment. This is shard %d/%d\n",
866             shard_index_, total_shards_);
867     fflush(stdout);
868   }
869   if (shard_index_ < 0 ||
870       total_shards_ < 0 ||
871       shard_index_ >= total_shards_) {
872     LOG(ERROR) << "Invalid sharding settings: we require 0 <= "
873                << kTestShardIndex << " < " << kTestTotalShards
874                << ", but you have " << kTestShardIndex << "=" << shard_index_
875                << ", " << kTestTotalShards << "=" << total_shards_ << ".\n";
876     return false;
877   }
878 
879   // Make sure we don't pass any sharding-related environment to the child
880   // processes. This test launcher implements the sharding completely.
881   CHECK(UnsetEnvironmentVariableIfExists("GTEST_TOTAL_SHARDS"));
882   CHECK(UnsetEnvironmentVariableIfExists("GTEST_SHARD_INDEX"));
883 
884   if (command_line->HasSwitch(kGTestRepeatFlag) &&
885       !StringToInt(command_line->GetSwitchValueASCII(kGTestRepeatFlag),
886                    &cycles_)) {
887     LOG(ERROR) << "Invalid value for " << kGTestRepeatFlag;
888     return false;
889   }
890 
891   if (command_line->HasSwitch(switches::kTestLauncherRetryLimit)) {
892     int retry_limit = -1;
893     if (!StringToInt(command_line->GetSwitchValueASCII(
894                          switches::kTestLauncherRetryLimit), &retry_limit) ||
895         retry_limit < 0) {
896       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherRetryLimit;
897       return false;
898     }
899 
900     retry_limit_ = retry_limit;
901   } else if (!command_line->HasSwitch(kGTestFilterFlag) || BotModeEnabled()) {
902     // Retry failures 3 times by default if we are running all of the tests or
903     // in bot mode.
904     retry_limit_ = 3;
905   }
906 
907   if (command_line->HasSwitch(switches::kTestLauncherForceRunBrokenTests))
908     force_run_broken_tests_ = true;
909 
910   // Some of the TestLauncherDelegate implementations don't call into gtest
911   // until they've already split into test-specific processes. This results
912   // in gtest's native shuffle implementation attempting to shuffle one test.
913   // Shuffling the list of tests in the test launcher (before the delegate
914   // gets involved) ensures that the entire shard is shuffled.
915   if (command_line->HasSwitch(kGTestShuffleFlag)) {
916     shuffle_ = true;
917 
918     if (command_line->HasSwitch(kGTestRandomSeedFlag)) {
919       const std::string custom_seed_str =
920           command_line->GetSwitchValueASCII(kGTestRandomSeedFlag);
921       uint32_t custom_seed = 0;
922       if (!StringToUint(custom_seed_str, &custom_seed)) {
923         LOG(ERROR) << "Unable to parse seed \"" << custom_seed_str << "\".";
924         return false;
925       }
926       if (custom_seed >= kRandomSeedUpperBound) {
927         LOG(ERROR) << "Seed " << custom_seed << " outside of expected range "
928                    << "[0, " << kRandomSeedUpperBound << ")";
929         return false;
930       }
931       shuffle_seed_ = custom_seed;
932     } else {
933       std::uniform_int_distribution<uint32_t> dist(0, kRandomSeedUpperBound);
934       std::random_device random_dev;
935       shuffle_seed_ = dist(random_dev);
936     }
937   } else if (command_line->HasSwitch(kGTestRandomSeedFlag)) {
938     LOG(ERROR) << kGTestRandomSeedFlag << " requires " << kGTestShuffleFlag;
939     return false;
940   }
941 
942   fprintf(stdout, "Using %" PRIuS " parallel jobs.\n", parallel_jobs_);
943   fflush(stdout);
944 
945   CreateAndStartTaskScheduler(static_cast<int>(parallel_jobs_));
946 
947   std::vector<std::string> positive_file_filter;
948   std::vector<std::string> positive_gtest_filter;
949 
950   if (command_line->HasSwitch(switches::kTestLauncherFilterFile)) {
951     base::FilePath filter_file_path = base::MakeAbsoluteFilePath(
952         command_line->GetSwitchValuePath(switches::kTestLauncherFilterFile));
953     if (!LoadFilterFile(filter_file_path, &positive_file_filter,
954                         &negative_test_filter_))
955       return false;
956   }
957 
958   // Split --gtest_filter at '-', if there is one, to separate into
959   // positive filter and negative filter portions.
960   std::string filter = command_line->GetSwitchValueASCII(kGTestFilterFlag);
961   size_t dash_pos = filter.find('-');
962   if (dash_pos == std::string::npos) {
963     positive_gtest_filter =
964         SplitString(filter, ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
965   } else {
966     // Everything up to the dash.
967     positive_gtest_filter =
968         SplitString(filter.substr(0, dash_pos), ":", base::TRIM_WHITESPACE,
969                     base::SPLIT_WANT_ALL);
970 
971     // Everything after the dash.
972     for (std::string pattern :
973          SplitString(filter.substr(dash_pos + 1), ":", base::TRIM_WHITESPACE,
974                      base::SPLIT_WANT_ALL)) {
975       negative_test_filter_.push_back(pattern);
976     }
977   }
978 
979   if (!launcher_delegate_->GetTests(&tests_)) {
980     LOG(ERROR) << "Failed to get list of tests.";
981     return false;
982   }
983 
984   CombinePositiveTestFilters(std::move(positive_gtest_filter),
985                              std::move(positive_file_filter));
986 
987   if (!results_tracker_.Init(*command_line)) {
988     LOG(ERROR) << "Failed to initialize test results tracker.";
989     return 1;
990   }
991 
992 #if defined(NDEBUG)
993   results_tracker_.AddGlobalTag("MODE_RELEASE");
994 #else
995   results_tracker_.AddGlobalTag("MODE_DEBUG");
996 #endif
997 
998   // Operating systems (sorted alphabetically).
999   // Note that they can deliberately overlap, e.g. OS_LINUX is a subset
1000   // of OS_POSIX.
1001 #if defined(OS_ANDROID)
1002   results_tracker_.AddGlobalTag("OS_ANDROID");
1003 #endif
1004 
1005 #if defined(OS_BSD)
1006   results_tracker_.AddGlobalTag("OS_BSD");
1007 #endif
1008 
1009 #if defined(OS_FREEBSD)
1010   results_tracker_.AddGlobalTag("OS_FREEBSD");
1011 #endif
1012 
1013 #if defined(OS_FUCHSIA)
1014   results_tracker_.AddGlobalTag("OS_FUCHSIA");
1015 #endif
1016 
1017 #if defined(OS_IOS)
1018   results_tracker_.AddGlobalTag("OS_IOS");
1019 #endif
1020 
1021 #if defined(OS_LINUX)
1022   results_tracker_.AddGlobalTag("OS_LINUX");
1023 #endif
1024 
1025 #if defined(OS_MACOSX)
1026   results_tracker_.AddGlobalTag("OS_MACOSX");
1027 #endif
1028 
1029 #if defined(OS_NACL)
1030   results_tracker_.AddGlobalTag("OS_NACL");
1031 #endif
1032 
1033 #if defined(OS_OPENBSD)
1034   results_tracker_.AddGlobalTag("OS_OPENBSD");
1035 #endif
1036 
1037 #if defined(OS_POSIX)
1038   results_tracker_.AddGlobalTag("OS_POSIX");
1039 #endif
1040 
1041 #if defined(OS_SOLARIS)
1042   results_tracker_.AddGlobalTag("OS_SOLARIS");
1043 #endif
1044 
1045 #if defined(OS_WIN)
1046   results_tracker_.AddGlobalTag("OS_WIN");
1047 #endif
1048 
1049   // CPU-related tags.
1050 #if defined(ARCH_CPU_32_BITS)
1051   results_tracker_.AddGlobalTag("CPU_32_BITS");
1052 #endif
1053 
1054 #if defined(ARCH_CPU_64_BITS)
1055   results_tracker_.AddGlobalTag("CPU_64_BITS");
1056 #endif
1057 
1058   return true;
1059 }
1060 
CombinePositiveTestFilters(std::vector<std::string> filter_a,std::vector<std::string> filter_b)1061 void TestLauncher::CombinePositiveTestFilters(
1062     std::vector<std::string> filter_a,
1063     std::vector<std::string> filter_b) {
1064   has_at_least_one_positive_filter_ = !filter_a.empty() || !filter_b.empty();
1065   if (!has_at_least_one_positive_filter_) {
1066     return;
1067   }
1068   // If two positive filters are present, only run tests that match a pattern
1069   // in both filters.
1070   if (!filter_a.empty() && !filter_b.empty()) {
1071     for (size_t i = 0; i < tests_.size(); i++) {
1072       std::string test_name =
1073           FormatFullTestName(tests_[i].test_case_name, tests_[i].test_name);
1074       bool found_a = false;
1075       bool found_b = false;
1076       for (size_t k = 0; k < filter_a.size(); ++k) {
1077         found_a = found_a || MatchPattern(test_name, filter_a[k]);
1078       }
1079       for (size_t k = 0; k < filter_b.size(); ++k) {
1080         found_b = found_b || MatchPattern(test_name, filter_b[k]);
1081       }
1082       if (found_a && found_b) {
1083         positive_test_filter_.push_back(test_name);
1084       }
1085     }
1086   } else if (!filter_a.empty()) {
1087     positive_test_filter_ = std::move(filter_a);
1088   } else {
1089     positive_test_filter_ = std::move(filter_b);
1090   }
1091 }
1092 
RunTests()1093 void TestLauncher::RunTests() {
1094   std::vector<std::string> test_names;
1095   const CommandLine* command_line = CommandLine::ForCurrentProcess();
1096   for (const TestIdentifier& test_id : tests_) {
1097     std::string test_name =
1098         FormatFullTestName(test_id.test_case_name, test_id.test_name);
1099 
1100     results_tracker_.AddTest(test_name);
1101 
1102     if (test_name.find("DISABLED") != std::string::npos) {
1103       results_tracker_.AddDisabledTest(test_name);
1104 
1105       // Skip disabled tests unless explicitly requested.
1106       if (!command_line->HasSwitch(kGTestRunDisabledTestsFlag))
1107         continue;
1108     }
1109 
1110     if (!launcher_delegate_->ShouldRunTest(test_id.test_case_name,
1111                                            test_id.test_name)) {
1112       continue;
1113     }
1114 
1115     // Count tests in the binary, before we apply filter and sharding.
1116     test_found_count_++;
1117 
1118     std::string test_name_no_disabled =
1119         TestNameWithoutDisabledPrefix(test_name);
1120 
1121     // Skip the test that doesn't match the filter (if given).
1122     if (has_at_least_one_positive_filter_) {
1123       bool found = false;
1124       for (auto filter : positive_test_filter_) {
1125         if (MatchPattern(test_name, filter) ||
1126             MatchPattern(test_name_no_disabled, filter)) {
1127           found = true;
1128           break;
1129         }
1130       }
1131 
1132       if (!found)
1133         continue;
1134     }
1135     if (!negative_test_filter_.empty()) {
1136       bool excluded = false;
1137       for (auto filter : negative_test_filter_) {
1138         if (MatchPattern(test_name, filter) ||
1139             MatchPattern(test_name_no_disabled, filter)) {
1140           excluded = true;
1141           break;
1142         }
1143       }
1144 
1145       if (excluded)
1146         continue;
1147     }
1148 
1149     if (Hash(test_name) % total_shards_ != static_cast<uint32_t>(shard_index_))
1150       continue;
1151 
1152     // Report test locations after applying all filters, so that we report test
1153     // locations only for those tests that were run as part of this shard.
1154     results_tracker_.AddTestLocation(test_name, test_id.file, test_id.line);
1155 
1156     test_names.push_back(test_name);
1157   }
1158 
1159   if (shuffle_) {
1160     std::mt19937 randomizer;
1161     randomizer.seed(shuffle_seed_);
1162     std::shuffle(test_names.begin(), test_names.end(), randomizer);
1163 
1164     fprintf(stdout, "Randomizing with seed %u\n", shuffle_seed_);
1165     fflush(stdout);
1166   }
1167 
1168   // Save an early test summary in case the launcher crashes or gets killed.
1169   MaybeSaveSummaryAsJSON({"EARLY_SUMMARY", kUnreliableResultsTag});
1170 
1171   test_started_count_ = launcher_delegate_->RunTests(this, test_names);
1172 
1173   if (test_started_count_ == 0) {
1174     fprintf(stdout, "0 tests run\n");
1175     fflush(stdout);
1176 
1177     // No tests have actually been started, so kick off the next iteration.
1178     ThreadTaskRunnerHandle::Get()->PostTask(
1179         FROM_HERE, BindOnce(&TestLauncher::RunTestIteration, Unretained(this)));
1180   }
1181 }
1182 
RunTestIteration()1183 void TestLauncher::RunTestIteration() {
1184   const bool stop_on_failure =
1185       CommandLine::ForCurrentProcess()->HasSwitch(kGTestBreakOnFailure);
1186   if (cycles_ == 0 ||
1187       (stop_on_failure && test_success_count_ != test_finished_count_)) {
1188     RunLoop::QuitCurrentWhenIdleDeprecated();
1189     return;
1190   }
1191 
1192   // Special value "-1" means "repeat indefinitely".
1193   cycles_ = (cycles_ == -1) ? cycles_ : cycles_ - 1;
1194 
1195   test_found_count_ = 0;
1196   test_started_count_ = 0;
1197   test_finished_count_ = 0;
1198   test_success_count_ = 0;
1199   test_broken_count_ = 0;
1200   retry_count_ = 0;
1201   tests_to_retry_.clear();
1202   results_tracker_.OnTestIterationStarting();
1203 
1204   ThreadTaskRunnerHandle::Get()->PostTask(
1205       FROM_HERE, BindOnce(&TestLauncher::RunTests, Unretained(this)));
1206 }
1207 
1208 #if defined(OS_POSIX)
1209 // I/O watcher for the reading end of the self-pipe above.
1210 // Terminates any launched child processes and exits the process.
OnShutdownPipeReadable()1211 void TestLauncher::OnShutdownPipeReadable() {
1212   fprintf(stdout, "\nCaught signal. Killing spawned test processes...\n");
1213   fflush(stdout);
1214 
1215   KillSpawnedTestProcesses();
1216 
1217   MaybeSaveSummaryAsJSON({"CAUGHT_TERMINATION_SIGNAL", kUnreliableResultsTag});
1218 
1219   // The signal would normally kill the process, so exit now.
1220   _exit(1);
1221 }
1222 #endif  // defined(OS_POSIX)
1223 
MaybeSaveSummaryAsJSON(const std::vector<std::string> & additional_tags)1224 void TestLauncher::MaybeSaveSummaryAsJSON(
1225     const std::vector<std::string>& additional_tags) {
1226   const CommandLine* command_line = CommandLine::ForCurrentProcess();
1227   if (command_line->HasSwitch(switches::kTestLauncherSummaryOutput)) {
1228     FilePath summary_path(command_line->GetSwitchValuePath(
1229                               switches::kTestLauncherSummaryOutput));
1230     if (!results_tracker_.SaveSummaryAsJSON(summary_path, additional_tags)) {
1231       LOG(ERROR) << "Failed to save test launcher output summary.";
1232     }
1233   }
1234   if (command_line->HasSwitch(switches::kTestLauncherTrace)) {
1235     FilePath trace_path(
1236         command_line->GetSwitchValuePath(switches::kTestLauncherTrace));
1237     if (!GetTestLauncherTracer()->Dump(trace_path)) {
1238       LOG(ERROR) << "Failed to save test launcher trace.";
1239     }
1240   }
1241 }
1242 
OnTestIterationFinished()1243 void TestLauncher::OnTestIterationFinished() {
1244   TestResultsTracker::TestStatusMap tests_by_status(
1245       results_tracker_.GetTestStatusMapForCurrentIteration());
1246   if (!tests_by_status[TestResult::TEST_UNKNOWN].empty())
1247     results_tracker_.AddGlobalTag(kUnreliableResultsTag);
1248 
1249   // When we retry tests, success is determined by having nothing more
1250   // to retry (everything eventually passed), as opposed to having
1251   // no failures at all.
1252   if (tests_to_retry_.empty()) {
1253     fprintf(stdout, "SUCCESS: all tests passed.\n");
1254     fflush(stdout);
1255   } else {
1256     // Signal failure, but continue to run all requested test iterations.
1257     // With the summary of all iterations at the end this is a good default.
1258     run_result_ = false;
1259   }
1260 
1261   results_tracker_.PrintSummaryOfCurrentIteration();
1262 
1263   // Kick off the next iteration.
1264   ThreadTaskRunnerHandle::Get()->PostTask(
1265       FROM_HERE, BindOnce(&TestLauncher::RunTestIteration, Unretained(this)));
1266 }
1267 
OnOutputTimeout()1268 void TestLauncher::OnOutputTimeout() {
1269   DCHECK(thread_checker_.CalledOnValidThread());
1270 
1271   AutoLock lock(*GetLiveProcessesLock());
1272 
1273   fprintf(stdout, "Still waiting for the following processes to finish:\n");
1274 
1275   for (const auto& pair : *GetLiveProcesses()) {
1276 #if defined(OS_WIN)
1277     fwprintf(stdout, L"\t%s\n", pair.second.GetCommandLineString().c_str());
1278 #else
1279     fprintf(stdout, "\t%s\n", pair.second.GetCommandLineString().c_str());
1280 #endif
1281   }
1282 
1283   fflush(stdout);
1284 
1285   // Arm the timer again - otherwise it would fire only once.
1286   watchdog_timer_.Reset();
1287 }
1288 
NumParallelJobs()1289 size_t NumParallelJobs() {
1290   const CommandLine* command_line = CommandLine::ForCurrentProcess();
1291   if (command_line->HasSwitch(switches::kTestLauncherJobs)) {
1292     // If the number of test launcher jobs was specified, return that number.
1293     size_t jobs = 0U;
1294 
1295     if (!StringToSizeT(
1296             command_line->GetSwitchValueASCII(switches::kTestLauncherJobs),
1297             &jobs) ||
1298         !jobs) {
1299       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherJobs;
1300       return 0U;
1301     }
1302     return jobs;
1303   }
1304   if (command_line->HasSwitch(kGTestFilterFlag) && !BotModeEnabled()) {
1305     // Do not run jobs in parallel by default if we are running a subset of
1306     // the tests and if bot mode is off.
1307     return 1U;
1308   }
1309 
1310   // Default to the number of processor cores.
1311   return base::checked_cast<size_t>(SysInfo::NumberOfProcessors());
1312 }
1313 
GetTestOutputSnippet(const TestResult & result,const std::string & full_output)1314 std::string GetTestOutputSnippet(const TestResult& result,
1315                                  const std::string& full_output) {
1316   size_t run_pos = full_output.find(std::string("[ RUN      ] ") +
1317                                     result.full_name);
1318   if (run_pos == std::string::npos)
1319     return std::string();
1320 
1321   size_t end_pos = full_output.find(std::string("[  FAILED  ] ") +
1322                                     result.full_name,
1323                                     run_pos);
1324   // Only clip the snippet to the "OK" message if the test really
1325   // succeeded. It still might have e.g. crashed after printing it.
1326   if (end_pos == std::string::npos &&
1327       result.status == TestResult::TEST_SUCCESS) {
1328     end_pos = full_output.find(std::string("[       OK ] ") +
1329                                result.full_name,
1330                                run_pos);
1331   }
1332   if (end_pos != std::string::npos) {
1333     size_t newline_pos = full_output.find("\n", end_pos);
1334     if (newline_pos != std::string::npos)
1335       end_pos = newline_pos + 1;
1336   }
1337 
1338   std::string snippet(full_output.substr(run_pos));
1339   if (end_pos != std::string::npos)
1340     snippet = full_output.substr(run_pos, end_pos - run_pos);
1341 
1342   return snippet;
1343 }
1344 
1345 }  // namespace base
1346