1 // Copyright 2013 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/test/launcher/unit_test_launcher.h"
6
7 #include <map>
8 #include <memory>
9 #include <utility>
10
11 #include "base/base_paths.h"
12 #include "base/base_switches.h"
13 #include "base/command_line.h"
14 #include "base/compiler_specific.h"
15 #include "base/debug/debugger.h"
16 #include "base/files/file_util.h"
17 #include "base/format_macros.h"
18 #include "base/functional/bind.h"
19 #include "base/functional/callback_helpers.h"
20 #include "base/i18n/icu_util.h"
21 #include "base/location.h"
22 #include "base/logging.h"
23 #include "base/message_loop/message_pump_type.h"
24 #include "base/sequence_checker.h"
25 #include "base/strings/strcat.h"
26 #include "base/strings/string_number_conversions.h"
27 #include "base/strings/string_util.h"
28 #include "base/system/sys_info.h"
29 #include "base/task/single_thread_task_executor.h"
30 #include "base/task/single_thread_task_runner.h"
31 #include "base/test/allow_check_is_test_for_testing.h"
32 #include "base/test/launcher/test_launcher.h"
33 #include "base/test/scoped_block_tests_writing_to_special_dirs.h"
34 #include "base/test/test_switches.h"
35 #include "base/test/test_timeouts.h"
36 #include "base/threading/thread_checker.h"
37 #include "build/build_config.h"
38 #include "testing/gtest/include/gtest/gtest.h"
39
40 #if BUILDFLAG(IS_POSIX)
41 #include "base/files/file_descriptor_watcher_posix.h"
42 #endif
43
44 #if BUILDFLAG(IS_IOS)
45 #include "base/test/test_support_ios.h"
46 #endif
47
48 namespace base {
49
50 namespace {
51
52 // This constant controls how many tests are run in a single batch by default.
53 const size_t kDefaultTestBatchLimit =
54 #if BUILDFLAG(IS_IOS)
55 100;
56 #else
57 10;
58 #endif
59
60 #if !BUILDFLAG(IS_ANDROID)
PrintUsage()61 void PrintUsage() {
62 fprintf(
63 stdout,
64 "Runs tests using the gtest framework, each batch of tests being\n"
65 "run in their own process. Supported command-line flags:\n"
66 "\n"
67 " Common flags:\n"
68 " --gtest_filter=...\n"
69 " Runs a subset of tests (see --gtest_help for more info).\n"
70 "\n"
71 " --help\n"
72 " Shows this message.\n"
73 "\n"
74 " --gtest_help\n"
75 " Shows the gtest help message.\n"
76 "\n"
77 " --test-launcher-jobs=N\n"
78 " Sets the number of parallel test jobs to N.\n"
79 "\n"
80 " --single-process-tests\n"
81 " Runs the tests and the launcher in the same process. Useful\n"
82 " for debugging a specific test in a debugger.\n"
83 "\n"
84 " Other flags:\n"
85 " --test-launcher-filter-file=PATH\n"
86 " Like --gtest_filter, but read the test filter from PATH.\n"
87 " Supports multiple filter paths separated by ';'.\n"
88 " One pattern per line; lines starting with '-' are exclusions.\n"
89 " See also //testing/buildbot/filters/README.md file.\n"
90 "\n"
91 " --test-launcher-batch-limit=N\n"
92 " Sets the limit of test batch to run in a single process to N.\n"
93 "\n"
94 " --test-launcher-debug-launcher\n"
95 " Disables autodetection of debuggers and similar tools,\n"
96 " making it possible to use them to debug launcher itself.\n"
97 "\n"
98 " --test-launcher-retry-limit=N\n"
99 " Sets the limit of test retries on failures to N.\n"
100 " --gtest_repeat=N\n"
101 " Forces the launcher to run every test N times. -1 is a special"
102 " value, causing the infinite amount of iterations."
103 " Repeated tests are run in parallel, unless the number of"
104 " iterations is infinite or --gtest_break_on_failure is specified"
105 " (see below)."
106 " Consider using --test_launcher-jobs flag to speed up the"
107 " parallel execution."
108 "\n"
109 " --gtest_break_on_failure\n"
110 " Stop running repeated tests as soon as one repeat of the test fails."
111 " This flag forces sequential repeats and prevents parallelised"
112 " execution."
113 "\n"
114 " --test-launcher-summary-output=PATH\n"
115 " Saves a JSON machine-readable summary of the run.\n"
116 "\n"
117 " --test-launcher-print-test-stdio=auto|always|never\n"
118 " Controls when full test output is printed.\n"
119 " auto means to print it when the test failed.\n"
120 "\n"
121 " --test-launcher-test-part-results-limit=N\n"
122 " Sets the limit of failed EXPECT/ASSERT entries in the xml and\n"
123 " JSON outputs per test to N (default N=10). Negative value \n"
124 " will disable this limit.\n"
125 "\n"
126 " --test-launcher-total-shards=N\n"
127 " Sets the total number of shards to N.\n"
128 "\n"
129 " --test-launcher-shard-index=N\n"
130 " Sets the shard index to run to N (from 0 to TOTAL - 1).\n"
131 "\n"
132 " --test-launcher-print-temp-leaks\n"
133 " Prints information about leaked files and/or directories in\n"
134 " child process's temporary directories (Windows and macOS).\n");
135 fflush(stdout);
136 }
137
GetSwitchValueAsInt(const std::string & switch_name,int * result)138 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
139 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
140 return true;
141
142 std::string switch_value =
143 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
144 if (!StringToInt(switch_value, result) || *result < 0) {
145 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
146 return false;
147 }
148
149 return true;
150 }
151
RunTestSuite(RunTestSuiteCallback run_test_suite,size_t parallel_jobs,int default_batch_limit,size_t retry_limit,bool use_job_objects,RepeatingClosure timeout_callback,OnceClosure gtest_init)152 int RunTestSuite(RunTestSuiteCallback run_test_suite,
153 size_t parallel_jobs,
154 int default_batch_limit,
155 size_t retry_limit,
156 bool use_job_objects,
157 RepeatingClosure timeout_callback,
158 OnceClosure gtest_init) {
159 bool force_single_process = false;
160 if (CommandLine::ForCurrentProcess()->HasSwitch(
161 switches::kTestLauncherDebugLauncher)) {
162 fprintf(stdout, "Forcing test launcher debugging mode.\n");
163 fflush(stdout);
164 } else {
165 if (base::debug::BeingDebugged()) {
166 fprintf(stdout,
167 "Debugger detected, switching to single process mode.\n"
168 "Pass --test-launcher-debug-launcher to debug the launcher "
169 "itself.\n");
170 fflush(stdout);
171 force_single_process = true;
172 }
173 }
174
175 if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
176 CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
177 CommandLine::ForCurrentProcess()->HasSwitch(
178 switches::kSingleProcessTests) ||
179 CommandLine::ForCurrentProcess()->HasSwitch(
180 switches::kTestChildProcess) ||
181 force_single_process) {
182 return std::move(run_test_suite).Run();
183 }
184
185 // ICU must be initialized before any attempts to format times, e.g. for logs.
186 if (!base::i18n::InitializeICU()) {
187 return false;
188 }
189
190 if (CommandLine::ForCurrentProcess()->HasSwitch(switches::kHelpFlag)) {
191 PrintUsage();
192 return 0;
193 }
194
195 TimeTicks start_time(TimeTicks::Now());
196
197 std::move(gtest_init).Run();
198 TestTimeouts::Initialize();
199
200 int batch_limit = default_batch_limit;
201 if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
202 return 1;
203
204 fprintf(stdout,
205 "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
206 "own process. For debugging a test inside a debugger, use the\n"
207 "--gtest_filter=<your_test_name> flag along with\n"
208 "--single-process-tests.\n");
209 fflush(stdout);
210
211 base::SingleThreadTaskExecutor executor(base::MessagePumpType::IO);
212 #if BUILDFLAG(IS_POSIX)
213 FileDescriptorWatcher file_descriptor_watcher(executor.task_runner());
214 #endif
215 DefaultUnitTestPlatformDelegate platform_delegate;
216 UnitTestLauncherDelegate delegate(&platform_delegate, batch_limit,
217 use_job_objects, timeout_callback);
218 TestLauncher launcher(&delegate, parallel_jobs, retry_limit);
219 bool success = launcher.Run();
220
221 fprintf(stdout, "Tests took %" PRId64 " seconds.\n",
222 (TimeTicks::Now() - start_time).InSeconds());
223 fflush(stdout);
224
225 return (success ? 0 : 1);
226 }
227 #endif
228
LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,size_t parallel_jobs,int default_batch_limit,size_t retry_limit,bool use_job_objects,RepeatingClosure timeout_callback,OnceClosure gtest_init)229 int LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,
230 size_t parallel_jobs,
231 int default_batch_limit,
232 size_t retry_limit,
233 bool use_job_objects,
234 RepeatingClosure timeout_callback,
235 OnceClosure gtest_init) {
236 base::test::AllowCheckIsTestForTesting();
237
238 #if BUILDFLAG(IS_ANDROID)
239 // We can't easily fork on Android, just run the test suite directly.
240 return std::move(run_test_suite).Run();
241 #elif BUILDFLAG(IS_IOS)
242 InitIOSRunHook(base::BindOnce(&RunTestSuite, std::move(run_test_suite),
243 parallel_jobs, default_batch_limit, retry_limit,
244 use_job_objects, timeout_callback,
245 std::move(gtest_init)));
246 return RunTestsFromIOSApp();
247 #else
248 ScopedBlockTestsWritingToSpecialDirs scoped_blocker(
249 {
250 // Please keep these in alphabetic order within each platform type.
251 base::DIR_SRC_TEST_DATA_ROOT, base::DIR_USER_DESKTOP,
252 #if BUILDFLAG(IS_WIN)
253 base::DIR_COMMON_DESKTOP, base::DIR_START_MENU,
254 base::DIR_USER_STARTUP,
255
256 #endif // BUILDFLAG(IS_WIN)
257 },
258 ([](const base::FilePath& path) {
259 ADD_FAILURE()
260 << "Attempting to write file in dir " << path
261 << " Use ScopedPathOverride or other mechanism to not write to this"
262 " directory.";
263 }));
264 return RunTestSuite(std::move(run_test_suite), parallel_jobs,
265 default_batch_limit, retry_limit, use_job_objects,
266 timeout_callback, std::move(gtest_init));
267 #endif
268 }
269
InitGoogleTestChar(int * argc,char ** argv)270 void InitGoogleTestChar(int* argc, char** argv) {
271 testing::InitGoogleTest(argc, argv);
272 }
273
274 #if BUILDFLAG(IS_WIN)
InitGoogleTestWChar(int * argc,wchar_t ** argv)275 void InitGoogleTestWChar(int* argc, wchar_t** argv) {
276 testing::InitGoogleTest(argc, argv);
277 }
278 #endif // BUILDFLAG(IS_WIN)
279
280 } // namespace
281
282 MergeTestFilterSwitchHandler::~MergeTestFilterSwitchHandler() = default;
ResolveDuplicate(base::StringPiece key,CommandLine::StringPieceType new_value,CommandLine::StringType & out_value)283 void MergeTestFilterSwitchHandler::ResolveDuplicate(
284 base::StringPiece key,
285 CommandLine::StringPieceType new_value,
286 CommandLine::StringType& out_value) {
287 if (key != switches::kTestLauncherFilterFile) {
288 out_value = CommandLine::StringType(new_value);
289 return;
290 }
291 if (!out_value.empty()) {
292 #if BUILDFLAG(IS_WIN)
293 StrAppend(&out_value, {L";"});
294 #else
295 StrAppend(&out_value, {";"});
296 #endif
297 }
298 StrAppend(&out_value, {new_value});
299 }
300
LaunchUnitTests(int argc,char ** argv,RunTestSuiteCallback run_test_suite,size_t retry_limit)301 int LaunchUnitTests(int argc,
302 char** argv,
303 RunTestSuiteCallback run_test_suite,
304 size_t retry_limit) {
305 CommandLine::SetDuplicateSwitchHandler(
306 std::make_unique<MergeTestFilterSwitchHandler>());
307 CommandLine::Init(argc, argv);
308 size_t parallel_jobs = NumParallelJobs(/*cores_per_job=*/1);
309 if (parallel_jobs == 0U) {
310 return 1;
311 }
312 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
313 kDefaultTestBatchLimit, retry_limit, true,
314 DoNothing(),
315 BindOnce(&InitGoogleTestChar, &argc, argv));
316 }
317
LaunchUnitTestsSerially(int argc,char ** argv,RunTestSuiteCallback run_test_suite)318 int LaunchUnitTestsSerially(int argc,
319 char** argv,
320 RunTestSuiteCallback run_test_suite) {
321 CommandLine::Init(argc, argv);
322 return LaunchUnitTestsInternal(std::move(run_test_suite), 1U,
323 kDefaultTestBatchLimit, 1U, true, DoNothing(),
324 BindOnce(&InitGoogleTestChar, &argc, argv));
325 }
326
LaunchUnitTestsWithOptions(int argc,char ** argv,size_t parallel_jobs,int default_batch_limit,bool use_job_objects,RepeatingClosure timeout_callback,RunTestSuiteCallback run_test_suite)327 int LaunchUnitTestsWithOptions(int argc,
328 char** argv,
329 size_t parallel_jobs,
330 int default_batch_limit,
331 bool use_job_objects,
332 RepeatingClosure timeout_callback,
333 RunTestSuiteCallback run_test_suite) {
334 CommandLine::Init(argc, argv);
335 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
336 default_batch_limit, 1U, use_job_objects,
337 timeout_callback,
338 BindOnce(&InitGoogleTestChar, &argc, argv));
339 }
340
341 #if BUILDFLAG(IS_WIN)
LaunchUnitTests(int argc,wchar_t ** argv,bool use_job_objects,RunTestSuiteCallback run_test_suite)342 int LaunchUnitTests(int argc,
343 wchar_t** argv,
344 bool use_job_objects,
345 RunTestSuiteCallback run_test_suite) {
346 // Windows CommandLine::Init ignores argv anyway.
347 CommandLine::Init(argc, NULL);
348 size_t parallel_jobs = NumParallelJobs(/*cores_per_job=*/1);
349 if (parallel_jobs == 0U) {
350 return 1;
351 }
352 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
353 kDefaultTestBatchLimit, 1U, use_job_objects,
354 DoNothing(),
355 BindOnce(&InitGoogleTestWChar, &argc, argv));
356 }
357 #endif // BUILDFLAG(IS_WIN)
358
359 DefaultUnitTestPlatformDelegate::DefaultUnitTestPlatformDelegate() = default;
360
GetTests(std::vector<TestIdentifier> * output)361 bool DefaultUnitTestPlatformDelegate::GetTests(
362 std::vector<TestIdentifier>* output) {
363 *output = GetCompiledInTests();
364 return true;
365 }
366
CreateResultsFile(const base::FilePath & temp_dir,base::FilePath * path)367 bool DefaultUnitTestPlatformDelegate::CreateResultsFile(
368 const base::FilePath& temp_dir,
369 base::FilePath* path) {
370 if (!CreateTemporaryDirInDir(temp_dir, FilePath::StringType(), path))
371 return false;
372 *path = path->AppendASCII("test_results.xml");
373 return true;
374 }
375
CreateTemporaryFile(const base::FilePath & temp_dir,base::FilePath * path)376 bool DefaultUnitTestPlatformDelegate::CreateTemporaryFile(
377 const base::FilePath& temp_dir,
378 base::FilePath* path) {
379 if (temp_dir.empty())
380 return false;
381 return CreateTemporaryFileInDir(temp_dir, path);
382 }
383
GetCommandLineForChildGTestProcess(const std::vector<std::string> & test_names,const base::FilePath & output_file,const base::FilePath & flag_file)384 CommandLine DefaultUnitTestPlatformDelegate::GetCommandLineForChildGTestProcess(
385 const std::vector<std::string>& test_names,
386 const base::FilePath& output_file,
387 const base::FilePath& flag_file) {
388 CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
389
390 CHECK(base::PathExists(flag_file));
391
392 std::string long_flags(
393 StrCat({"--", kGTestFilterFlag, "=", JoinString(test_names, ":")}));
394 CHECK(WriteFile(flag_file, long_flags));
395
396 new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
397 new_cmd_line.AppendSwitchPath(kGTestFlagfileFlag, flag_file);
398 new_cmd_line.AppendSwitch(switches::kSingleProcessTests);
399
400 return new_cmd_line;
401 }
402
GetWrapperForChildGTestProcess()403 std::string DefaultUnitTestPlatformDelegate::GetWrapperForChildGTestProcess() {
404 return std::string();
405 }
406
UnitTestLauncherDelegate(UnitTestPlatformDelegate * platform_delegate,size_t batch_limit,bool use_job_objects,RepeatingClosure timeout_callback)407 UnitTestLauncherDelegate::UnitTestLauncherDelegate(
408 UnitTestPlatformDelegate* platform_delegate,
409 size_t batch_limit,
410 bool use_job_objects,
411 RepeatingClosure timeout_callback)
412 : platform_delegate_(platform_delegate),
413 batch_limit_(batch_limit),
414 use_job_objects_(use_job_objects),
415 timeout_callback_(timeout_callback) {}
416
~UnitTestLauncherDelegate()417 UnitTestLauncherDelegate::~UnitTestLauncherDelegate() {
418 DCHECK(thread_checker_.CalledOnValidThread());
419 }
420
GetTests(std::vector<TestIdentifier> * output)421 bool UnitTestLauncherDelegate::GetTests(std::vector<TestIdentifier>* output) {
422 DCHECK(thread_checker_.CalledOnValidThread());
423 return platform_delegate_->GetTests(output);
424 }
425
GetCommandLine(const std::vector<std::string> & test_names,const FilePath & temp_dir,FilePath * output_file)426 CommandLine UnitTestLauncherDelegate::GetCommandLine(
427 const std::vector<std::string>& test_names,
428 const FilePath& temp_dir,
429 FilePath* output_file) {
430 CHECK(!test_names.empty());
431
432 // Create a dedicated temporary directory to store the xml result data
433 // per run to ensure clean state and make it possible to launch multiple
434 // processes in parallel.
435 CHECK(platform_delegate_->CreateResultsFile(temp_dir, output_file));
436 FilePath flag_file;
437 platform_delegate_->CreateTemporaryFile(temp_dir, &flag_file);
438
439 return CommandLine(platform_delegate_->GetCommandLineForChildGTestProcess(
440 test_names, *output_file, flag_file));
441 }
442
GetWrapper()443 std::string UnitTestLauncherDelegate::GetWrapper() {
444 return platform_delegate_->GetWrapperForChildGTestProcess();
445 }
446
GetLaunchOptions()447 int UnitTestLauncherDelegate::GetLaunchOptions() {
448 return use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0;
449 }
450
GetTimeout()451 TimeDelta UnitTestLauncherDelegate::GetTimeout() {
452 return TestTimeouts::test_launcher_timeout();
453 }
454
GetBatchSize()455 size_t UnitTestLauncherDelegate::GetBatchSize() {
456 return batch_limit_;
457 }
458
OnTestTimedOut(const CommandLine & cmd_line)459 void UnitTestLauncherDelegate::OnTestTimedOut(const CommandLine& cmd_line) {
460 timeout_callback_.Run();
461 }
462
463 } // namespace base
464