1 // Copyright 2013 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/test/launcher/unit_test_launcher.h"
6
7 #include <map>
8 #include <memory>
9 #include <string_view>
10 #include <utility>
11
12 #include "base/base_paths.h"
13 #include "base/base_switches.h"
14 #include "base/command_line.h"
15 #include "base/compiler_specific.h"
16 #include "base/debug/debugger.h"
17 #include "base/files/file_util.h"
18 #include "base/format_macros.h"
19 #include "base/functional/bind.h"
20 #include "base/functional/callback_helpers.h"
21 #include "base/i18n/icu_util.h"
22 #include "base/location.h"
23 #include "base/logging.h"
24 #include "base/message_loop/message_pump_type.h"
25 #include "base/sequence_checker.h"
26 #include "base/strings/strcat.h"
27 #include "base/strings/string_number_conversions.h"
28 #include "base/strings/string_util.h"
29 #include "base/system/sys_info.h"
30 #include "base/task/single_thread_task_executor.h"
31 #include "base/task/single_thread_task_runner.h"
32 #include "base/test/allow_check_is_test_for_testing.h"
33 #include "base/test/launcher/test_launcher.h"
34 #include "base/test/scoped_block_tests_writing_to_special_dirs.h"
35 #include "base/test/test_switches.h"
36 #include "base/test/test_timeouts.h"
37 #include "base/threading/thread_checker.h"
38 #include "build/build_config.h"
39 #include "testing/gtest/include/gtest/gtest.h"
40 #include "testing/libfuzzer/fuzztest_init_helper.h"
41
42 #if BUILDFLAG(IS_POSIX)
43 #include "base/files/file_descriptor_watcher_posix.h"
44 #endif
45
46 #if BUILDFLAG(IS_IOS)
47 #include "base/test/test_support_ios.h"
48 #endif
49
50 namespace base {
51
52 namespace {
53
54 // This constant controls how many tests are run in a single batch by default.
55 const size_t kDefaultTestBatchLimit =
56 #if BUILDFLAG(IS_IOS)
57 100;
58 #else
59 10;
60 #endif
61
62 #if !BUILDFLAG(IS_ANDROID)
PrintUsage()63 void PrintUsage() {
64 fprintf(
65 stdout,
66 "Runs tests using the gtest framework, each batch of tests being\n"
67 "run in their own process. Supported command-line flags:\n"
68 "\n"
69 " Common flags:\n"
70 " --gtest_filter=...\n"
71 " Runs a subset of tests (see --gtest_help for more info).\n"
72 "\n"
73 " --help\n"
74 " Shows this message.\n"
75 "\n"
76 " --gtest_help\n"
77 " Shows the gtest help message.\n"
78 "\n"
79 " --test-launcher-jobs=N\n"
80 " Sets the number of parallel test jobs to N.\n"
81 "\n"
82 " --single-process-tests\n"
83 " Runs the tests and the launcher in the same process. Useful\n"
84 " for debugging a specific test in a debugger.\n"
85 "\n"
86 " Other flags:\n"
87 " --test-launcher-filter-file=PATH\n"
88 " Like --gtest_filter, but read the test filter from PATH.\n"
89 " Supports multiple filter paths separated by ';'.\n"
90 " One pattern per line; lines starting with '-' are exclusions.\n"
91 " See also //testing/buildbot/filters/README.md file.\n"
92 "\n"
93 " --test-launcher-batch-limit=N\n"
94 " Sets the limit of test batch to run in a single process to N.\n"
95 "\n"
96 " --test-launcher-debug-launcher\n"
97 " Disables autodetection of debuggers and similar tools,\n"
98 " making it possible to use them to debug launcher itself.\n"
99 "\n"
100 " --test-launcher-retry-limit=N\n"
101 " Sets the limit of test retries on failures to N.\n"
102 " --gtest_repeat=N\n"
103 " Forces the launcher to run every test N times. -1 is a special"
104 " value, causing the infinite amount of iterations."
105 " Repeated tests are run in parallel, unless the number of"
106 " iterations is infinite or --gtest_break_on_failure is specified"
107 " (see below)."
108 " Consider using --test_launcher-jobs flag to speed up the"
109 " parallel execution."
110 "\n"
111 " --gtest_break_on_failure\n"
112 " Stop running repeated tests as soon as one repeat of the test fails."
113 " This flag forces sequential repeats and prevents parallelised"
114 " execution."
115 "\n"
116 " --test-launcher-summary-output=PATH\n"
117 " Saves a JSON machine-readable summary of the run.\n"
118 "\n"
119 " --test-launcher-print-test-stdio=auto|always|never\n"
120 " Controls when full test output is printed.\n"
121 " auto means to print it when the test failed.\n"
122 "\n"
123 " --test-launcher-test-part-results-limit=N\n"
124 " Sets the limit of failed EXPECT/ASSERT entries in the xml and\n"
125 " JSON outputs per test to N (default N=10). Negative value \n"
126 " will disable this limit.\n"
127 "\n"
128 " --test-launcher-total-shards=N\n"
129 " Sets the total number of shards to N.\n"
130 "\n"
131 " --test-launcher-shard-index=N\n"
132 " Sets the shard index to run to N (from 0 to TOTAL - 1).\n"
133 "\n"
134 " --test-launcher-print-temp-leaks\n"
135 " Prints information about leaked files and/or directories in\n"
136 " child process's temporary directories (Windows and macOS).\n");
137 fflush(stdout);
138 }
139
GetSwitchValueAsInt(const std::string & switch_name,int * result)140 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
141 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
142 return true;
143
144 std::string switch_value =
145 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
146 if (!StringToInt(switch_value, result) || *result < 0) {
147 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
148 return false;
149 }
150
151 return true;
152 }
153
RunTestSuite(RunTestSuiteCallback run_test_suite,size_t parallel_jobs,int default_batch_limit,size_t retry_limit,bool use_job_objects,RepeatingClosure timeout_callback,OnceClosure gtest_init)154 int RunTestSuite(RunTestSuiteCallback run_test_suite,
155 size_t parallel_jobs,
156 int default_batch_limit,
157 size_t retry_limit,
158 bool use_job_objects,
159 RepeatingClosure timeout_callback,
160 OnceClosure gtest_init) {
161 bool force_single_process = false;
162 if (CommandLine::ForCurrentProcess()->HasSwitch(
163 switches::kTestLauncherDebugLauncher)) {
164 fprintf(stdout, "Forcing test launcher debugging mode.\n");
165 fflush(stdout);
166 } else {
167 if (base::debug::BeingDebugged()) {
168 fprintf(stdout,
169 "Debugger detected, switching to single process mode.\n"
170 "Pass --test-launcher-debug-launcher to debug the launcher "
171 "itself.\n");
172 fflush(stdout);
173 force_single_process = true;
174 }
175 }
176
177 if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
178 CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
179 CommandLine::ForCurrentProcess()->HasSwitch(
180 switches::kSingleProcessTests) ||
181 CommandLine::ForCurrentProcess()->HasSwitch(
182 switches::kTestChildProcess) ||
183 CommandLine::ForCurrentProcess()->HasSwitch(switches::kFuzz) ||
184 CommandLine::ForCurrentProcess()->HasSwitch(switches::kFuzzFor) ||
185 CommandLine::ForCurrentProcess()->HasSwitch(switches::kListFuzzTests) ||
186 force_single_process) {
187 return std::move(run_test_suite).Run();
188 }
189
190 // ICU must be initialized before any attempts to format times, e.g. for logs.
191 CHECK(base::i18n::InitializeICU());
192
193 if (CommandLine::ForCurrentProcess()->HasSwitch(switches::kHelpFlag)) {
194 PrintUsage();
195 return 0;
196 }
197
198 TimeTicks start_time(TimeTicks::Now());
199
200 std::move(gtest_init).Run();
201 TestTimeouts::Initialize();
202
203 int batch_limit = default_batch_limit;
204 if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
205 return 1;
206
207 fprintf(stdout,
208 "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
209 "own process. For debugging a test inside a debugger, use the\n"
210 "--gtest_filter=<your_test_name> flag along with\n"
211 "--single-process-tests.\n");
212 fflush(stdout);
213
214 base::SingleThreadTaskExecutor executor(base::MessagePumpType::IO);
215 #if BUILDFLAG(IS_POSIX)
216 FileDescriptorWatcher file_descriptor_watcher(executor.task_runner());
217 #endif
218 DefaultUnitTestPlatformDelegate platform_delegate;
219 UnitTestLauncherDelegate delegate(&platform_delegate, batch_limit,
220 use_job_objects, timeout_callback);
221 TestLauncher launcher(&delegate, parallel_jobs, retry_limit);
222 bool success = launcher.Run();
223
224 fprintf(stdout, "Tests took %" PRId64 " seconds.\n",
225 (TimeTicks::Now() - start_time).InSeconds());
226 fflush(stdout);
227
228 return (success ? 0 : 1);
229 }
230 #endif
231
LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,size_t parallel_jobs,int default_batch_limit,size_t retry_limit,bool use_job_objects,RepeatingClosure timeout_callback,OnceClosure gtest_init)232 int LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,
233 size_t parallel_jobs,
234 int default_batch_limit,
235 size_t retry_limit,
236 bool use_job_objects,
237 RepeatingClosure timeout_callback,
238 OnceClosure gtest_init) {
239 base::test::AllowCheckIsTestForTesting();
240
241 #if BUILDFLAG(IS_ANDROID)
242 // We can't easily fork on Android, just run the test suite directly.
243 return std::move(run_test_suite).Run();
244 #elif BUILDFLAG(IS_IOS)
245 InitIOSRunHook(base::BindOnce(&RunTestSuite, std::move(run_test_suite),
246 parallel_jobs, default_batch_limit, retry_limit,
247 use_job_objects, timeout_callback,
248 std::move(gtest_init)));
249 return RunTestsFromIOSApp();
250 #else
251 ScopedBlockTestsWritingToSpecialDirs scoped_blocker(
252 {
253 // Please keep these in alphabetic order within each platform type.
254 base::DIR_SRC_TEST_DATA_ROOT, base::DIR_USER_DESKTOP,
255 #if BUILDFLAG(IS_WIN)
256 base::DIR_COMMON_DESKTOP, base::DIR_START_MENU,
257 base::DIR_USER_STARTUP,
258
259 #endif // BUILDFLAG(IS_WIN)
260 },
261 ([](const base::FilePath& path) {
262 ADD_FAILURE()
263 << "Attempting to write file in dir " << path
264 << " Use ScopedPathOverride or other mechanism to not write to this"
265 " directory.";
266 }));
267 return RunTestSuite(std::move(run_test_suite), parallel_jobs,
268 default_batch_limit, retry_limit, use_job_objects,
269 timeout_callback, std::move(gtest_init));
270 #endif
271 }
272
InitGoogleTestChar(int * argc,char ** argv)273 void InitGoogleTestChar(int* argc, char** argv) {
274 testing::InitGoogleTest(argc, argv);
275 MaybeInitFuzztest(*argc, argv);
276 }
277
278 #if BUILDFLAG(IS_WIN)
279
280 // Safety: as is normal in command lines, argc and argv must correspond
281 // to one another. Otherwise there will be out-of-bounds accesses.
InitGoogleTestWChar(int * argc,wchar_t ** argv)282 UNSAFE_BUFFER_USAGE void InitGoogleTestWChar(int* argc, wchar_t** argv) {
283 testing::InitGoogleTest(argc, argv);
284 // Fuzztest requires a narrow command-line.
285 CHECK(*argc >= 0);
286 const auto argc_s = static_cast<size_t>(*argc);
287 span<wchar_t*> wide_command_line = UNSAFE_BUFFERS(span(argv, argc_s));
288 std::vector<std::string> narrow_command_line;
289 std::vector<char*> narrow_command_line_pointers;
290 narrow_command_line.reserve(argc_s);
291 narrow_command_line_pointers.reserve(argc_s);
292 for (size_t i = 0; i < argc_s; ++i) {
293 narrow_command_line.push_back(WideToUTF8(wide_command_line[i]));
294 narrow_command_line_pointers.push_back(narrow_command_line[i].data());
295 }
296 MaybeInitFuzztest(*argc, narrow_command_line_pointers.data());
297 }
298 #endif // BUILDFLAG(IS_WIN)
299
300 } // namespace
301
302 MergeTestFilterSwitchHandler::~MergeTestFilterSwitchHandler() = default;
ResolveDuplicate(std::string_view key,CommandLine::StringViewType new_value,CommandLine::StringType & out_value)303 void MergeTestFilterSwitchHandler::ResolveDuplicate(
304 std::string_view key,
305 CommandLine::StringViewType new_value,
306 CommandLine::StringType& out_value) {
307 if (key != switches::kTestLauncherFilterFile) {
308 out_value = CommandLine::StringType(new_value);
309 return;
310 }
311 if (!out_value.empty()) {
312 #if BUILDFLAG(IS_WIN)
313 StrAppend(&out_value, {L";"});
314 #else
315 StrAppend(&out_value, {";"});
316 #endif
317 }
318 StrAppend(&out_value, {new_value});
319 }
320
LaunchUnitTests(int argc,char ** argv,RunTestSuiteCallback run_test_suite,size_t retry_limit)321 int LaunchUnitTests(int argc,
322 char** argv,
323 RunTestSuiteCallback run_test_suite,
324 size_t retry_limit) {
325 CommandLine::SetDuplicateSwitchHandler(
326 std::make_unique<MergeTestFilterSwitchHandler>());
327 CommandLine::Init(argc, argv);
328 size_t parallel_jobs = NumParallelJobs(/*cores_per_job=*/1);
329 if (parallel_jobs == 0U) {
330 return 1;
331 }
332 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
333 kDefaultTestBatchLimit, retry_limit, true,
334 DoNothing(),
335 BindOnce(&InitGoogleTestChar, &argc, argv));
336 }
337
LaunchUnitTestsSerially(int argc,char ** argv,RunTestSuiteCallback run_test_suite)338 int LaunchUnitTestsSerially(int argc,
339 char** argv,
340 RunTestSuiteCallback run_test_suite) {
341 CommandLine::Init(argc, argv);
342 return LaunchUnitTestsInternal(std::move(run_test_suite), 1U,
343 kDefaultTestBatchLimit, 1U, true, DoNothing(),
344 BindOnce(&InitGoogleTestChar, &argc, argv));
345 }
346
LaunchUnitTestsWithOptions(int argc,char ** argv,size_t parallel_jobs,int default_batch_limit,bool use_job_objects,RepeatingClosure timeout_callback,RunTestSuiteCallback run_test_suite)347 int LaunchUnitTestsWithOptions(int argc,
348 char** argv,
349 size_t parallel_jobs,
350 int default_batch_limit,
351 bool use_job_objects,
352 RepeatingClosure timeout_callback,
353 RunTestSuiteCallback run_test_suite) {
354 CommandLine::Init(argc, argv);
355 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
356 default_batch_limit, 1U, use_job_objects,
357 timeout_callback,
358 BindOnce(&InitGoogleTestChar, &argc, argv));
359 }
360
361 #if BUILDFLAG(IS_WIN)
LaunchUnitTests(int argc,wchar_t ** argv,bool use_job_objects,RunTestSuiteCallback run_test_suite)362 int LaunchUnitTests(int argc,
363 wchar_t** argv,
364 bool use_job_objects,
365 RunTestSuiteCallback run_test_suite) {
366 // Windows CommandLine::Init ignores argv anyway.
367 CommandLine::Init(argc, NULL);
368 size_t parallel_jobs = NumParallelJobs(/*cores_per_job=*/1);
369 if (parallel_jobs == 0U) {
370 return 1;
371 }
372 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
373 kDefaultTestBatchLimit, 1U, use_job_objects,
374 DoNothing(),
375 BindOnce(&InitGoogleTestWChar, &argc, argv));
376 }
377 #endif // BUILDFLAG(IS_WIN)
378
379 DefaultUnitTestPlatformDelegate::DefaultUnitTestPlatformDelegate() = default;
380
GetTests(std::vector<TestIdentifier> * output)381 bool DefaultUnitTestPlatformDelegate::GetTests(
382 std::vector<TestIdentifier>* output) {
383 *output = GetCompiledInTests();
384 return true;
385 }
386
CreateResultsFile(const base::FilePath & temp_dir,base::FilePath * path)387 bool DefaultUnitTestPlatformDelegate::CreateResultsFile(
388 const base::FilePath& temp_dir,
389 base::FilePath* path) {
390 if (!CreateTemporaryDirInDir(temp_dir, FilePath::StringType(), path))
391 return false;
392 *path = path->AppendASCII("test_results.xml");
393 return true;
394 }
395
CreateTemporaryFile(const base::FilePath & temp_dir,base::FilePath * path)396 bool DefaultUnitTestPlatformDelegate::CreateTemporaryFile(
397 const base::FilePath& temp_dir,
398 base::FilePath* path) {
399 if (temp_dir.empty())
400 return false;
401 return CreateTemporaryFileInDir(temp_dir, path);
402 }
403
GetCommandLineForChildGTestProcess(const std::vector<std::string> & test_names,const base::FilePath & output_file,const base::FilePath & flag_file)404 CommandLine DefaultUnitTestPlatformDelegate::GetCommandLineForChildGTestProcess(
405 const std::vector<std::string>& test_names,
406 const base::FilePath& output_file,
407 const base::FilePath& flag_file) {
408 CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
409
410 CHECK(base::PathExists(flag_file));
411
412 // Any `--gtest_filter` flag specified on the original command line is
413 // no longer needed; the test launcher has already determined the list
414 // of actual tests to run in each child process. Since the test launcher
415 // internally uses `--gtest_filter` via a flagfile to pass this info to
416 // the child process, remove any original `--gtest_filter` flags on the
417 // command line, as GoogleTest provides no guarantee about whether the
418 // command line or the flagfile takes precedence.
419 new_cmd_line.RemoveSwitch(kGTestFilterFlag);
420
421 std::string long_flags(
422 StrCat({"--", kGTestFilterFlag, "=", JoinString(test_names, ":")}));
423 CHECK(WriteFile(flag_file, long_flags));
424
425 new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
426 new_cmd_line.AppendSwitchPath(kGTestFlagfileFlag, flag_file);
427 new_cmd_line.AppendSwitch(switches::kSingleProcessTests);
428
429 return new_cmd_line;
430 }
431
GetWrapperForChildGTestProcess()432 std::string DefaultUnitTestPlatformDelegate::GetWrapperForChildGTestProcess() {
433 return std::string();
434 }
435
UnitTestLauncherDelegate(UnitTestPlatformDelegate * platform_delegate,size_t batch_limit,bool use_job_objects,RepeatingClosure timeout_callback)436 UnitTestLauncherDelegate::UnitTestLauncherDelegate(
437 UnitTestPlatformDelegate* platform_delegate,
438 size_t batch_limit,
439 bool use_job_objects,
440 RepeatingClosure timeout_callback)
441 : platform_delegate_(platform_delegate),
442 batch_limit_(batch_limit),
443 use_job_objects_(use_job_objects),
444 timeout_callback_(timeout_callback) {}
445
~UnitTestLauncherDelegate()446 UnitTestLauncherDelegate::~UnitTestLauncherDelegate() {
447 DCHECK(thread_checker_.CalledOnValidThread());
448 }
449
GetTests(std::vector<TestIdentifier> * output)450 bool UnitTestLauncherDelegate::GetTests(std::vector<TestIdentifier>* output) {
451 DCHECK(thread_checker_.CalledOnValidThread());
452 return platform_delegate_->GetTests(output);
453 }
454
GetCommandLine(const std::vector<std::string> & test_names,const FilePath & temp_dir,FilePath * output_file)455 CommandLine UnitTestLauncherDelegate::GetCommandLine(
456 const std::vector<std::string>& test_names,
457 const FilePath& temp_dir,
458 FilePath* output_file) {
459 CHECK(!test_names.empty());
460
461 // Create a dedicated temporary directory to store the xml result data
462 // per run to ensure clean state and make it possible to launch multiple
463 // processes in parallel.
464 CHECK(platform_delegate_->CreateResultsFile(temp_dir, output_file));
465 FilePath flag_file;
466 platform_delegate_->CreateTemporaryFile(temp_dir, &flag_file);
467
468 return CommandLine(platform_delegate_->GetCommandLineForChildGTestProcess(
469 test_names, *output_file, flag_file));
470 }
471
GetWrapper()472 std::string UnitTestLauncherDelegate::GetWrapper() {
473 return platform_delegate_->GetWrapperForChildGTestProcess();
474 }
475
GetLaunchOptions()476 int UnitTestLauncherDelegate::GetLaunchOptions() {
477 return use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0;
478 }
479
GetTimeout()480 TimeDelta UnitTestLauncherDelegate::GetTimeout() {
481 return TestTimeouts::test_launcher_timeout();
482 }
483
GetBatchSize()484 size_t UnitTestLauncherDelegate::GetBatchSize() {
485 return batch_limit_;
486 }
487
OnTestTimedOut(const CommandLine & cmd_line)488 void UnitTestLauncherDelegate::OnTestTimedOut(const CommandLine& cmd_line) {
489 timeout_callback_.Run();
490 }
491
492 } // namespace base
493