1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/command_line.h"
6 #include "base/files/file_util.h"
7 #include "base/logging.h"
8 #include "base/process/kill.h"
9 #include "base/process/launch.h"
10 #include "base/strings/string_number_conversions.h"
11 #include "base/strings/utf_string_conversions.h"
12 #include "base/time/time.h"
13 #include "build/build_config.h"
14 #include "tools/gn/err.h"
15 #include "tools/gn/filesystem_utils.h"
16 #include "tools/gn/functions.h"
17 #include "tools/gn/input_conversion.h"
18 #include "tools/gn/input_file.h"
19 #include "tools/gn/parse_tree.h"
20 #include "tools/gn/scheduler.h"
21 #include "tools/gn/trace.h"
22 #include "tools/gn/value.h"
23
24 #if defined(OS_WIN)
25 #include <windows.h>
26
27 #include "base/win/scoped_handle.h"
28 #include "base/win/scoped_process_information.h"
29 #endif
30
31 #if defined(OS_POSIX)
32 #include <fcntl.h>
33 #include <unistd.h>
34
35 #include "base/posix/file_descriptor_shuffle.h"
36 #endif
37
38 namespace functions {
39
40 namespace {
41
42 const char kNoExecSwitch[] = "no-exec";
43
44 #if defined(OS_WIN)
ExecProcess(const CommandLine & cmdline,const base::FilePath & startup_dir,std::string * std_out,std::string * std_err,int * exit_code)45 bool ExecProcess(const CommandLine& cmdline,
46 const base::FilePath& startup_dir,
47 std::string* std_out,
48 std::string* std_err,
49 int* exit_code) {
50 SECURITY_ATTRIBUTES sa_attr;
51 // Set the bInheritHandle flag so pipe handles are inherited.
52 sa_attr.nLength = sizeof(SECURITY_ATTRIBUTES);
53 sa_attr.bInheritHandle = TRUE;
54 sa_attr.lpSecurityDescriptor = NULL;
55
56 // Create the pipe for the child process's STDOUT.
57 HANDLE out_read = NULL;
58 HANDLE out_write = NULL;
59 if (!CreatePipe(&out_read, &out_write, &sa_attr, 0)) {
60 NOTREACHED() << "Failed to create pipe";
61 return false;
62 }
63 base::win::ScopedHandle scoped_out_read(out_read);
64 base::win::ScopedHandle scoped_out_write(out_write);
65
66 // Create the pipe for the child process's STDERR.
67 HANDLE err_read = NULL;
68 HANDLE err_write = NULL;
69 if (!CreatePipe(&err_read, &err_write, &sa_attr, 0)) {
70 NOTREACHED() << "Failed to create pipe";
71 return false;
72 }
73 base::win::ScopedHandle scoped_err_read(err_read);
74 base::win::ScopedHandle scoped_err_write(err_write);
75
76 // Ensure the read handle to the pipe for STDOUT/STDERR is not inherited.
77 if (!SetHandleInformation(out_read, HANDLE_FLAG_INHERIT, 0)) {
78 NOTREACHED() << "Failed to disabled pipe inheritance";
79 return false;
80 }
81 if (!SetHandleInformation(err_read, HANDLE_FLAG_INHERIT, 0)) {
82 NOTREACHED() << "Failed to disabled pipe inheritance";
83 return false;
84 }
85
86 base::FilePath::StringType cmdline_str(cmdline.GetCommandLineString());
87
88 STARTUPINFO start_info = {};
89
90 start_info.cb = sizeof(STARTUPINFO);
91 start_info.hStdOutput = out_write;
92 // Keep the normal stdin.
93 start_info.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
94 // FIXME(brettw) set stderr here when we actually read it below.
95 //start_info.hStdError = err_write;
96 start_info.hStdError = GetStdHandle(STD_ERROR_HANDLE);
97 start_info.dwFlags |= STARTF_USESTDHANDLES;
98
99 // Create the child process.
100 PROCESS_INFORMATION temp_process_info = {};
101 if (!CreateProcess(NULL,
102 &cmdline_str[0],
103 NULL, NULL,
104 TRUE, // Handles are inherited.
105 0, NULL,
106 startup_dir.value().c_str(),
107 &start_info, &temp_process_info)) {
108 return false;
109 }
110 base::win::ScopedProcessInformation proc_info(temp_process_info);
111
112 // Close our writing end of pipes now. Otherwise later read would not be able
113 // to detect end of child's output.
114 scoped_out_write.Close();
115 scoped_err_write.Close();
116
117 // Read output from the child process's pipe for STDOUT
118 const int kBufferSize = 1024;
119 char buffer[kBufferSize];
120
121 // FIXME(brettw) read from stderr here! This is complicated because we want
122 // to read both of them at the same time, probably need overlapped I/O.
123 // Also uncomment start_info code above.
124 for (;;) {
125 DWORD bytes_read = 0;
126 BOOL success = ReadFile(out_read, buffer, kBufferSize, &bytes_read, NULL);
127 if (!success || bytes_read == 0)
128 break;
129 std_out->append(buffer, bytes_read);
130 }
131
132 // Let's wait for the process to finish.
133 WaitForSingleObject(proc_info.process_handle(), INFINITE);
134
135 DWORD dw_exit_code;
136 GetExitCodeProcess(proc_info.process_handle(), &dw_exit_code);
137 *exit_code = static_cast<int>(dw_exit_code);
138
139 return true;
140 }
141 #else
ExecProcess(const CommandLine & cmdline,const base::FilePath & startup_dir,std::string * std_out,std::string * std_err,int * exit_code)142 bool ExecProcess(const CommandLine& cmdline,
143 const base::FilePath& startup_dir,
144 std::string* std_out,
145 std::string* std_err,
146 int* exit_code) {
147 *exit_code = EXIT_FAILURE;
148
149 std::vector<std::string> argv = cmdline.argv();
150
151 int pipe_fd[2];
152 pid_t pid;
153 base::InjectiveMultimap fd_shuffle1, fd_shuffle2;
154 scoped_ptr<char*[]> argv_cstr(new char*[argv.size() + 1]);
155
156 fd_shuffle1.reserve(3);
157 fd_shuffle2.reserve(3);
158
159 if (pipe(pipe_fd) < 0)
160 return false;
161
162 switch (pid = fork()) {
163 case -1: // error
164 close(pipe_fd[0]);
165 close(pipe_fd[1]);
166 return false;
167 case 0: // child
168 {
169 // DANGER: no calls to malloc are allowed from now on:
170 // http://crbug.com/36678
171
172 // Obscure fork() rule: in the child, if you don't end up doing exec*(),
173 // you call _exit() instead of exit(). This is because _exit() does not
174 // call any previously-registered (in the parent) exit handlers, which
175 // might do things like block waiting for threads that don't even exist
176 // in the child.
177 int dev_null = open("/dev/null", O_WRONLY);
178 if (dev_null < 0)
179 _exit(127);
180
181 fd_shuffle1.push_back(
182 base::InjectionArc(pipe_fd[1], STDOUT_FILENO, true));
183 fd_shuffle1.push_back(
184 base::InjectionArc(dev_null, STDERR_FILENO, true));
185 fd_shuffle1.push_back(
186 base::InjectionArc(dev_null, STDIN_FILENO, true));
187 // Adding another element here? Remeber to increase the argument to
188 // reserve(), above.
189
190 for (size_t i = 0; i < fd_shuffle1.size(); ++i)
191 fd_shuffle2.push_back(fd_shuffle1[i]);
192
193 if (!ShuffleFileDescriptors(&fd_shuffle1))
194 _exit(127);
195
196 base::SetCurrentDirectory(startup_dir);
197
198 // TODO(brettw) the base version GetAppOutput does a
199 // CloseSuperfluousFds call here. Do we need this?
200
201 for (size_t i = 0; i < argv.size(); i++)
202 argv_cstr[i] = const_cast<char*>(argv[i].c_str());
203 argv_cstr[argv.size()] = NULL;
204 execvp(argv_cstr[0], argv_cstr.get());
205 _exit(127);
206 }
207 default: // parent
208 {
209 // Close our writing end of pipe now. Otherwise later read would not
210 // be able to detect end of child's output (in theory we could still
211 // write to the pipe).
212 close(pipe_fd[1]);
213
214 char buffer[256];
215 ssize_t bytes_read = 0;
216
217 while (true) {
218 bytes_read = HANDLE_EINTR(read(pipe_fd[0], buffer, sizeof(buffer)));
219 if (bytes_read <= 0)
220 break;
221 std_out->append(buffer, bytes_read);
222 }
223 close(pipe_fd[0]);
224
225 return base::WaitForExitCode(pid, exit_code);
226 }
227 }
228
229 return false;
230 }
231 #endif
232
233 } // namespace
234
235 const char kExecScript[] = "exec_script";
236 const char kExecScript_HelpShort[] =
237 "exec_script: Synchronously run a script and return the output.";
238 const char kExecScript_Help[] =
239 "exec_script: Synchronously run a script and return the output.\n"
240 "\n"
241 " exec_script(filename,\n"
242 " arguments = [],\n"
243 " input_conversion = \"\",\n"
244 " file_dependencies = [])\n"
245 "\n"
246 " Runs the given script, returning the stdout of the script. The build\n"
247 " generation will fail if the script does not exist or returns a nonzero\n"
248 " exit code.\n"
249 "\n"
250 " The current directory when executing the script will be the root\n"
251 " build directory. If you are passing file names, you will want to use\n"
252 " the rebase_path() function to make file names relative to this\n"
253 " path (see \"gn help rebase_path\").\n"
254 "\n"
255 "Arguments:\n"
256 "\n"
257 " filename:\n"
258 " File name of python script to execute. Non-absolute names will\n"
259 " be treated as relative to the current build file.\n"
260 "\n"
261 " arguments:\n"
262 " A list of strings to be passed to the script as arguments.\n"
263 " May be unspecified or the empty list which means no arguments.\n"
264 "\n"
265 " input_conversion:\n"
266 " Controls how the file is read and parsed.\n"
267 " See \"gn help input_conversion\".\n"
268 "\n"
269 " If unspecified, defaults to the empty string which causes the\n"
270 " script result to be discarded. exec script will return None.\n"
271 "\n"
272 " dependencies:\n"
273 " (Optional) A list of files that this script reads or otherwise\n"
274 " depends on. These dependencies will be added to the build result\n"
275 " such that if any of them change, the build will be regenerated and\n"
276 " the script will be re-run.\n"
277 "\n"
278 " The script itself will be an implicit dependency so you do not\n"
279 " need to list it.\n"
280 "\n"
281 "Example:\n"
282 "\n"
283 " all_lines = exec_script(\n"
284 " \"myscript.py\", [some_input], \"list lines\",\n"
285 " [ rebase_path(\"data_file.txt\", root_build_dir) ])\n"
286 "\n"
287 " # This example just calls the script with no arguments and discards\n"
288 " # the result.\n"
289 " exec_script(\"//foo/bar/myscript.py\")\n";
290
RunExecScript(Scope * scope,const FunctionCallNode * function,const std::vector<Value> & args,Err * err)291 Value RunExecScript(Scope* scope,
292 const FunctionCallNode* function,
293 const std::vector<Value>& args,
294 Err* err) {
295 if (args.size() < 1 || args.size() > 4) {
296 *err = Err(function->function(), "Wrong number of arguments to exec_script",
297 "I expected between one and four arguments.");
298 return Value();
299 }
300
301 const Settings* settings = scope->settings();
302 const BuildSettings* build_settings = settings->build_settings();
303 const SourceDir& cur_dir = scope->GetSourceDir();
304
305 // Find the python script to run.
306 if (!args[0].VerifyTypeIs(Value::STRING, err))
307 return Value();
308 SourceFile script_source =
309 cur_dir.ResolveRelativeFile(args[0].string_value());
310 base::FilePath script_path = build_settings->GetFullPath(script_source);
311 if (!build_settings->secondary_source_path().empty() &&
312 !base::PathExists(script_path)) {
313 // Fall back to secondary source root when the file doesn't exist.
314 script_path = build_settings->GetFullPathSecondary(script_source);
315 }
316
317 ScopedTrace trace(TraceItem::TRACE_SCRIPT_EXECUTE, script_source.value());
318 trace.SetToolchain(settings->toolchain_label());
319
320 // Add all dependencies of this script, including the script itself, to the
321 // build deps.
322 g_scheduler->AddGenDependency(script_path);
323 if (args.size() == 4) {
324 const Value& deps_value = args[3];
325 if (!deps_value.VerifyTypeIs(Value::LIST, err))
326 return Value();
327
328 for (size_t i = 0; i < deps_value.list_value().size(); i++) {
329 if (!deps_value.list_value()[0].VerifyTypeIs(Value::STRING, err))
330 return Value();
331 g_scheduler->AddGenDependency(
332 build_settings->GetFullPath(cur_dir.ResolveRelativeFile(
333 deps_value.list_value()[0].string_value())));
334 }
335 }
336
337 // Make the command line.
338 const base::FilePath& python_path = build_settings->python_path();
339 CommandLine cmdline(python_path);
340 cmdline.AppendArgPath(script_path);
341
342 if (args.size() >= 2) {
343 // Optional command-line arguments to the script.
344 const Value& script_args = args[1];
345 if (!script_args.VerifyTypeIs(Value::LIST, err))
346 return Value();
347 for (size_t i = 0; i < script_args.list_value().size(); i++) {
348 if (!script_args.list_value()[i].VerifyTypeIs(Value::STRING, err))
349 return Value();
350 cmdline.AppendArg(script_args.list_value()[i].string_value());
351 }
352 }
353
354 // Log command line for debugging help.
355 trace.SetCommandLine(cmdline);
356 base::TimeTicks begin_exec;
357 if (g_scheduler->verbose_logging()) {
358 #if defined(OS_WIN)
359 g_scheduler->Log("Pythoning",
360 base::UTF16ToUTF8(cmdline.GetCommandLineString()));
361 #else
362 g_scheduler->Log("Pythoning", cmdline.GetCommandLineString());
363 #endif
364 begin_exec = base::TimeTicks::Now();
365 }
366
367 base::FilePath startup_dir =
368 build_settings->GetFullPath(build_settings->build_dir());
369 // The first time a build is run, no targets will have been written so the
370 // build output directory won't exist. We need to make sure it does before
371 // running any scripts with this as its startup directory, although it will
372 // be relatively rare that the directory won't exist by the time we get here.
373 //
374 // If this shows up on benchmarks, we can cache whether we've done this
375 // or not and skip creating the directory.
376 base::CreateDirectory(startup_dir);
377
378 // Execute the process.
379 // TODO(brettw) set the environment block.
380 std::string output;
381 std::string stderr_output; // TODO(brettw) not hooked up, see above.
382 int exit_code = 0;
383 if (!CommandLine::ForCurrentProcess()->HasSwitch(kNoExecSwitch)) {
384 if (!ExecProcess(cmdline, startup_dir,
385 &output, &stderr_output, &exit_code)) {
386 *err = Err(function->function(), "Could not execute python.",
387 "I was trying to execute \"" + FilePathToUTF8(python_path) + "\".");
388 return Value();
389 }
390 }
391 if (g_scheduler->verbose_logging()) {
392 g_scheduler->Log("Pythoning", script_source.value() + " took " +
393 base::Int64ToString(
394 (base::TimeTicks::Now() - begin_exec).InMilliseconds()) +
395 "ms");
396 }
397
398 // TODO(brettw) maybe we need stderr also for reasonable stack dumps.
399 if (exit_code != 0) {
400 std::string msg = "Current dir: " + FilePathToUTF8(startup_dir) +
401 "\nCommand: " + FilePathToUTF8(cmdline.GetCommandLineString()) +
402 "\nReturned " + base::IntToString(exit_code);
403 if (!output.empty())
404 msg += " and printed out:\n\n" + output;
405 else
406 msg += ".";
407 *err = Err(function->function(), "Script returned non-zero exit code.",
408 msg);
409 return Value();
410 }
411
412 // Default to None value for the input conversion if unspecified.
413 return ConvertInputToValue(scope->settings(), output, function,
414 args.size() >= 3 ? args[2] : Value(), err);
415 }
416
417 } // namespace functions
418