1 // Copyright 2013 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/test/launcher/unit_test_launcher.h"
6
7 #include <map>
8 #include <memory>
9 #include <utility>
10
11 #include "base/base_switches.h"
12 #include "base/command_line.h"
13 #include "base/compiler_specific.h"
14 #include "base/debug/debugger.h"
15 #include "base/files/file_util.h"
16 #include "base/format_macros.h"
17 #include "base/functional/bind.h"
18 #include "base/functional/callback_helpers.h"
19 #include "base/location.h"
20 #include "base/logging.h"
21 #include "base/message_loop/message_pump_type.h"
22 #include "base/sequence_checker.h"
23 #include "base/strings/strcat.h"
24 #include "base/strings/string_number_conversions.h"
25 #include "base/strings/string_util.h"
26 #include "base/system/sys_info.h"
27 #include "base/task/single_thread_task_executor.h"
28 #include "base/task/single_thread_task_runner.h"
29 #include "base/test/allow_check_is_test_for_testing.h"
30 #include "base/test/launcher/test_launcher.h"
31 #include "base/test/test_switches.h"
32 #include "base/test/test_timeouts.h"
33 #include "base/threading/thread_checker.h"
34 #include "build/build_config.h"
35 #include "testing/gtest/include/gtest/gtest.h"
36
37 #if BUILDFLAG(IS_POSIX)
38 #include "base/files/file_descriptor_watcher_posix.h"
39 #endif
40
41 #if BUILDFLAG(IS_IOS)
42 #include "base/test/test_support_ios.h"
43 #endif
44
45 namespace base {
46
47 namespace {
48
49 // This constant controls how many tests are run in a single batch by default.
50 const size_t kDefaultTestBatchLimit =
51 #if BUILDFLAG(IS_IOS)
52 100;
53 #else
54 10;
55 #endif
56
57 #if !BUILDFLAG(IS_ANDROID)
PrintUsage()58 void PrintUsage() {
59 fprintf(
60 stdout,
61 "Runs tests using the gtest framework, each batch of tests being\n"
62 "run in their own process. Supported command-line flags:\n"
63 "\n"
64 " Common flags:\n"
65 " --gtest_filter=...\n"
66 " Runs a subset of tests (see --gtest_help for more info).\n"
67 "\n"
68 " --help\n"
69 " Shows this message.\n"
70 "\n"
71 " --gtest_help\n"
72 " Shows the gtest help message.\n"
73 "\n"
74 " --test-launcher-jobs=N\n"
75 " Sets the number of parallel test jobs to N.\n"
76 "\n"
77 " --single-process-tests\n"
78 " Runs the tests and the launcher in the same process. Useful\n"
79 " for debugging a specific test in a debugger.\n"
80 "\n"
81 " Other flags:\n"
82 " --test-launcher-filter-file=PATH\n"
83 " Like --gtest_filter, but read the test filter from PATH.\n"
84 " Supports multiple filter paths separated by ';'.\n"
85 " One pattern per line; lines starting with '-' are exclusions.\n"
86 " See also //testing/buildbot/filters/README.md file.\n"
87 "\n"
88 " --test-launcher-batch-limit=N\n"
89 " Sets the limit of test batch to run in a single process to N.\n"
90 "\n"
91 " --test-launcher-debug-launcher\n"
92 " Disables autodetection of debuggers and similar tools,\n"
93 " making it possible to use them to debug launcher itself.\n"
94 "\n"
95 " --test-launcher-retry-limit=N\n"
96 " Sets the limit of test retries on failures to N.\n"
97 " --gtest_repeat=N\n"
98 " Forces the launcher to run every test N times. -1 is a special"
99 " value, causing the infinite amount of iterations."
100 " Repeated tests are run in parallel, unless the number of"
101 " iterations is infinite or --gtest_break_on_failure is specified"
102 " (see below)."
103 " Consider using --test_launcher-jobs flag to speed up the"
104 " parallel execution."
105 "\n"
106 " --gtest_break_on_failure\n"
107 " Stop running repeated tests as soon as one repeat of the test fails."
108 " This flag forces sequential repeats and prevents parallelised"
109 " execution."
110 "\n"
111 " --test-launcher-summary-output=PATH\n"
112 " Saves a JSON machine-readable summary of the run.\n"
113 "\n"
114 " --test-launcher-print-test-stdio=auto|always|never\n"
115 " Controls when full test output is printed.\n"
116 " auto means to print it when the test failed.\n"
117 "\n"
118 " --test-launcher-test-part-results-limit=N\n"
119 " Sets the limit of failed EXPECT/ASSERT entries in the xml and\n"
120 " JSON outputs per test to N (default N=10). Negative value \n"
121 " will disable this limit.\n"
122 "\n"
123 " --test-launcher-total-shards=N\n"
124 " Sets the total number of shards to N.\n"
125 "\n"
126 " --test-launcher-shard-index=N\n"
127 " Sets the shard index to run to N (from 0 to TOTAL - 1).\n"
128 "\n"
129 " --test-launcher-print-temp-leaks\n"
130 " Prints information about leaked files and/or directories in\n"
131 " child process's temporary directories (Windows and macOS).\n");
132 fflush(stdout);
133 }
134
GetSwitchValueAsInt(const std::string & switch_name,int * result)135 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) {
136 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name))
137 return true;
138
139 std::string switch_value =
140 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name);
141 if (!StringToInt(switch_value, result) || *result < 0) {
142 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value;
143 return false;
144 }
145
146 return true;
147 }
148
RunTestSuite(RunTestSuiteCallback run_test_suite,size_t parallel_jobs,int default_batch_limit,size_t retry_limit,bool use_job_objects,RepeatingClosure timeout_callback,OnceClosure gtest_init)149 int RunTestSuite(RunTestSuiteCallback run_test_suite,
150 size_t parallel_jobs,
151 int default_batch_limit,
152 size_t retry_limit,
153 bool use_job_objects,
154 RepeatingClosure timeout_callback,
155 OnceClosure gtest_init) {
156 bool force_single_process = false;
157 if (CommandLine::ForCurrentProcess()->HasSwitch(
158 switches::kTestLauncherDebugLauncher)) {
159 fprintf(stdout, "Forcing test launcher debugging mode.\n");
160 fflush(stdout);
161 } else {
162 if (base::debug::BeingDebugged()) {
163 fprintf(stdout,
164 "Debugger detected, switching to single process mode.\n"
165 "Pass --test-launcher-debug-launcher to debug the launcher "
166 "itself.\n");
167 fflush(stdout);
168 force_single_process = true;
169 }
170 }
171
172 if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) ||
173 CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) ||
174 CommandLine::ForCurrentProcess()->HasSwitch(
175 switches::kSingleProcessTests) ||
176 CommandLine::ForCurrentProcess()->HasSwitch(
177 switches::kTestChildProcess) ||
178 force_single_process) {
179 return std::move(run_test_suite).Run();
180 }
181
182 if (CommandLine::ForCurrentProcess()->HasSwitch(switches::kHelpFlag)) {
183 PrintUsage();
184 return 0;
185 }
186
187 TimeTicks start_time(TimeTicks::Now());
188
189 std::move(gtest_init).Run();
190 TestTimeouts::Initialize();
191
192 int batch_limit = default_batch_limit;
193 if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit))
194 return 1;
195
196 fprintf(stdout,
197 "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n"
198 "own process. For debugging a test inside a debugger, use the\n"
199 "--gtest_filter=<your_test_name> flag along with\n"
200 "--single-process-tests.\n");
201 fflush(stdout);
202
203 base::SingleThreadTaskExecutor executor(base::MessagePumpType::IO);
204 #if BUILDFLAG(IS_POSIX)
205 FileDescriptorWatcher file_descriptor_watcher(executor.task_runner());
206 #endif
207 DefaultUnitTestPlatformDelegate platform_delegate;
208 UnitTestLauncherDelegate delegate(&platform_delegate, batch_limit,
209 use_job_objects, timeout_callback);
210 TestLauncher launcher(&delegate, parallel_jobs, retry_limit);
211 bool success = launcher.Run();
212
213 fprintf(stdout, "Tests took %" PRId64 " seconds.\n",
214 (TimeTicks::Now() - start_time).InSeconds());
215 fflush(stdout);
216
217 return (success ? 0 : 1);
218 }
219 #endif
220
LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,size_t parallel_jobs,int default_batch_limit,size_t retry_limit,bool use_job_objects,RepeatingClosure timeout_callback,OnceClosure gtest_init)221 int LaunchUnitTestsInternal(RunTestSuiteCallback run_test_suite,
222 size_t parallel_jobs,
223 int default_batch_limit,
224 size_t retry_limit,
225 bool use_job_objects,
226 RepeatingClosure timeout_callback,
227 OnceClosure gtest_init) {
228 base::test::AllowCheckIsTestForTesting();
229
230 #if BUILDFLAG(IS_ANDROID)
231 // We can't easily fork on Android, just run the test suite directly.
232 return std::move(run_test_suite).Run();
233 #elif BUILDFLAG(IS_IOS)
234 InitIOSRunHook(base::BindOnce(&RunTestSuite, std::move(run_test_suite),
235 parallel_jobs, default_batch_limit, retry_limit,
236 use_job_objects, timeout_callback,
237 std::move(gtest_init)));
238 return RunTestsFromIOSApp();
239 #else
240 return RunTestSuite(std::move(run_test_suite), parallel_jobs,
241 default_batch_limit, retry_limit, use_job_objects,
242 timeout_callback, std::move(gtest_init));
243 #endif
244 }
245
InitGoogleTestChar(int * argc,char ** argv)246 void InitGoogleTestChar(int* argc, char** argv) {
247 testing::InitGoogleTest(argc, argv);
248 }
249
250 #if BUILDFLAG(IS_WIN)
InitGoogleTestWChar(int * argc,wchar_t ** argv)251 void InitGoogleTestWChar(int* argc, wchar_t** argv) {
252 testing::InitGoogleTest(argc, argv);
253 }
254 #endif // BUILDFLAG(IS_WIN)
255
256 } // namespace
257
258 MergeTestFilterSwitchHandler::~MergeTestFilterSwitchHandler() = default;
ResolveDuplicate(base::StringPiece key,CommandLine::StringPieceType new_value,CommandLine::StringType & out_value)259 void MergeTestFilterSwitchHandler::ResolveDuplicate(
260 base::StringPiece key,
261 CommandLine::StringPieceType new_value,
262 CommandLine::StringType& out_value) {
263 if (key != switches::kTestLauncherFilterFile) {
264 out_value = CommandLine::StringType(new_value);
265 return;
266 }
267 if (!out_value.empty()) {
268 #if BUILDFLAG(IS_WIN)
269 StrAppend(&out_value, {L";"});
270 #else
271 StrAppend(&out_value, {";"});
272 #endif
273 }
274 StrAppend(&out_value, {new_value});
275 }
276
LaunchUnitTests(int argc,char ** argv,RunTestSuiteCallback run_test_suite,size_t retry_limit)277 int LaunchUnitTests(int argc,
278 char** argv,
279 RunTestSuiteCallback run_test_suite,
280 size_t retry_limit) {
281 CommandLine::SetDuplicateSwitchHandler(
282 std::make_unique<MergeTestFilterSwitchHandler>());
283 CommandLine::Init(argc, argv);
284 size_t parallel_jobs = NumParallelJobs(/*cores_per_job=*/1);
285 if (parallel_jobs == 0U) {
286 return 1;
287 }
288 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
289 kDefaultTestBatchLimit, retry_limit, true,
290 DoNothing(),
291 BindOnce(&InitGoogleTestChar, &argc, argv));
292 }
293
LaunchUnitTestsSerially(int argc,char ** argv,RunTestSuiteCallback run_test_suite)294 int LaunchUnitTestsSerially(int argc,
295 char** argv,
296 RunTestSuiteCallback run_test_suite) {
297 CommandLine::Init(argc, argv);
298 return LaunchUnitTestsInternal(std::move(run_test_suite), 1U,
299 kDefaultTestBatchLimit, 1U, true, DoNothing(),
300 BindOnce(&InitGoogleTestChar, &argc, argv));
301 }
302
LaunchUnitTestsWithOptions(int argc,char ** argv,size_t parallel_jobs,int default_batch_limit,bool use_job_objects,RepeatingClosure timeout_callback,RunTestSuiteCallback run_test_suite)303 int LaunchUnitTestsWithOptions(int argc,
304 char** argv,
305 size_t parallel_jobs,
306 int default_batch_limit,
307 bool use_job_objects,
308 RepeatingClosure timeout_callback,
309 RunTestSuiteCallback run_test_suite) {
310 CommandLine::Init(argc, argv);
311 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
312 default_batch_limit, 1U, use_job_objects,
313 timeout_callback,
314 BindOnce(&InitGoogleTestChar, &argc, argv));
315 }
316
317 #if BUILDFLAG(IS_WIN)
LaunchUnitTests(int argc,wchar_t ** argv,bool use_job_objects,RunTestSuiteCallback run_test_suite)318 int LaunchUnitTests(int argc,
319 wchar_t** argv,
320 bool use_job_objects,
321 RunTestSuiteCallback run_test_suite) {
322 // Windows CommandLine::Init ignores argv anyway.
323 CommandLine::Init(argc, NULL);
324 size_t parallel_jobs = NumParallelJobs(/*cores_per_job=*/1);
325 if (parallel_jobs == 0U) {
326 return 1;
327 }
328 return LaunchUnitTestsInternal(std::move(run_test_suite), parallel_jobs,
329 kDefaultTestBatchLimit, 1U, use_job_objects,
330 DoNothing(),
331 BindOnce(&InitGoogleTestWChar, &argc, argv));
332 }
333 #endif // BUILDFLAG(IS_WIN)
334
335 DefaultUnitTestPlatformDelegate::DefaultUnitTestPlatformDelegate() = default;
336
GetTests(std::vector<TestIdentifier> * output)337 bool DefaultUnitTestPlatformDelegate::GetTests(
338 std::vector<TestIdentifier>* output) {
339 *output = GetCompiledInTests();
340 return true;
341 }
342
CreateResultsFile(const base::FilePath & temp_dir,base::FilePath * path)343 bool DefaultUnitTestPlatformDelegate::CreateResultsFile(
344 const base::FilePath& temp_dir,
345 base::FilePath* path) {
346 if (!CreateTemporaryDirInDir(temp_dir, FilePath::StringType(), path))
347 return false;
348 *path = path->AppendASCII("test_results.xml");
349 return true;
350 }
351
CreateTemporaryFile(const base::FilePath & temp_dir,base::FilePath * path)352 bool DefaultUnitTestPlatformDelegate::CreateTemporaryFile(
353 const base::FilePath& temp_dir,
354 base::FilePath* path) {
355 if (temp_dir.empty())
356 return false;
357 return CreateTemporaryFileInDir(temp_dir, path);
358 }
359
GetCommandLineForChildGTestProcess(const std::vector<std::string> & test_names,const base::FilePath & output_file,const base::FilePath & flag_file)360 CommandLine DefaultUnitTestPlatformDelegate::GetCommandLineForChildGTestProcess(
361 const std::vector<std::string>& test_names,
362 const base::FilePath& output_file,
363 const base::FilePath& flag_file) {
364 CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
365
366 CHECK(base::PathExists(flag_file));
367
368 std::string long_flags(
369 StrCat({"--", kGTestFilterFlag, "=", JoinString(test_names, ":")}));
370 CHECK(WriteFile(flag_file, long_flags));
371
372 new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file);
373 new_cmd_line.AppendSwitchPath(kGTestFlagfileFlag, flag_file);
374 new_cmd_line.AppendSwitch(switches::kSingleProcessTests);
375
376 return new_cmd_line;
377 }
378
GetWrapperForChildGTestProcess()379 std::string DefaultUnitTestPlatformDelegate::GetWrapperForChildGTestProcess() {
380 return std::string();
381 }
382
UnitTestLauncherDelegate(UnitTestPlatformDelegate * platform_delegate,size_t batch_limit,bool use_job_objects,RepeatingClosure timeout_callback)383 UnitTestLauncherDelegate::UnitTestLauncherDelegate(
384 UnitTestPlatformDelegate* platform_delegate,
385 size_t batch_limit,
386 bool use_job_objects,
387 RepeatingClosure timeout_callback)
388 : platform_delegate_(platform_delegate),
389 batch_limit_(batch_limit),
390 use_job_objects_(use_job_objects),
391 timeout_callback_(timeout_callback) {}
392
~UnitTestLauncherDelegate()393 UnitTestLauncherDelegate::~UnitTestLauncherDelegate() {
394 DCHECK(thread_checker_.CalledOnValidThread());
395 }
396
GetTests(std::vector<TestIdentifier> * output)397 bool UnitTestLauncherDelegate::GetTests(std::vector<TestIdentifier>* output) {
398 DCHECK(thread_checker_.CalledOnValidThread());
399 return platform_delegate_->GetTests(output);
400 }
401
GetCommandLine(const std::vector<std::string> & test_names,const FilePath & temp_dir,FilePath * output_file)402 CommandLine UnitTestLauncherDelegate::GetCommandLine(
403 const std::vector<std::string>& test_names,
404 const FilePath& temp_dir,
405 FilePath* output_file) {
406 CHECK(!test_names.empty());
407
408 // Create a dedicated temporary directory to store the xml result data
409 // per run to ensure clean state and make it possible to launch multiple
410 // processes in parallel.
411 CHECK(platform_delegate_->CreateResultsFile(temp_dir, output_file));
412 FilePath flag_file;
413 platform_delegate_->CreateTemporaryFile(temp_dir, &flag_file);
414
415 return CommandLine(platform_delegate_->GetCommandLineForChildGTestProcess(
416 test_names, *output_file, flag_file));
417 }
418
GetWrapper()419 std::string UnitTestLauncherDelegate::GetWrapper() {
420 return platform_delegate_->GetWrapperForChildGTestProcess();
421 }
422
GetLaunchOptions()423 int UnitTestLauncherDelegate::GetLaunchOptions() {
424 return use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0;
425 }
426
GetTimeout()427 TimeDelta UnitTestLauncherDelegate::GetTimeout() {
428 return TestTimeouts::test_launcher_timeout();
429 }
430
GetBatchSize()431 size_t UnitTestLauncherDelegate::GetBatchSize() {
432 return batch_limit_;
433 }
434
OnTestTimedOut(const CommandLine & cmd_line)435 void UnitTestLauncherDelegate::OnTestTimedOut(const CommandLine& cmd_line) {
436 timeout_callback_.Run();
437 }
438
439 } // namespace base
440