1 // Copyright 2011 Google Inc. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "util.h"
16
17 #ifdef __CYGWIN__
18 #include <windows.h>
19 #include <io.h>
20 #elif defined( _WIN32)
21 #include <windows.h>
22 #include <io.h>
23 #include <share.h>
24 #endif
25
26 #include <assert.h>
27 #include <errno.h>
28 #include <fcntl.h>
29 #include <stdarg.h>
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <sys/stat.h>
34 #include <sys/types.h>
35
36 #ifndef _WIN32
37 #include <unistd.h>
38 #include <sys/time.h>
39 #endif
40
41 #include <vector>
42
43 #if defined(__APPLE__) || defined(__FreeBSD__)
44 #include <sys/sysctl.h>
45 #elif defined(__SVR4) && defined(__sun)
46 #include <unistd.h>
47 #include <sys/loadavg.h>
48 #elif defined(_AIX) && !defined(__PASE__)
49 #include <libperfstat.h>
50 #elif defined(linux) || defined(__GLIBC__)
51 #include <sys/sysinfo.h>
52 #endif
53
54 #include "edit_distance.h"
55 #include "metrics.h"
56
Fatal(const char * msg,...)57 void Fatal(const char* msg, ...) {
58 va_list ap;
59 fprintf(stderr, "ninja: fatal: ");
60 va_start(ap, msg);
61 vfprintf(stderr, msg, ap);
62 va_end(ap);
63 fprintf(stderr, "\n");
64 #ifdef _WIN32
65 // On Windows, some tools may inject extra threads.
66 // exit() may block on locks held by those threads, so forcibly exit.
67 fflush(stderr);
68 fflush(stdout);
69 ExitProcess(1);
70 #else
71 exit(1);
72 #endif
73 }
74
Warning(const char * msg,...)75 void Warning(const char* msg, ...) {
76 va_list ap;
77 fprintf(stderr, "ninja: warning: ");
78 va_start(ap, msg);
79 vfprintf(stderr, msg, ap);
80 va_end(ap);
81 fprintf(stderr, "\n");
82 }
83
Error(const char * msg,...)84 void Error(const char* msg, ...) {
85 va_list ap;
86 fprintf(stderr, "ninja: error: ");
87 va_start(ap, msg);
88 vfprintf(stderr, msg, ap);
89 va_end(ap);
90 fprintf(stderr, "\n");
91 }
92
CanonicalizePath(string * path,uint64_t * slash_bits,string * err)93 bool CanonicalizePath(string* path, uint64_t* slash_bits, string* err) {
94 METRIC_RECORD("canonicalize str");
95 size_t len = path->size();
96 char* str = 0;
97 if (len > 0)
98 str = &(*path)[0];
99 if (!CanonicalizePath(str, &len, slash_bits, err))
100 return false;
101 path->resize(len);
102 return true;
103 }
104
IsPathSeparator(char c)105 static bool IsPathSeparator(char c) {
106 #ifdef _WIN32
107 return c == '/' || c == '\\';
108 #else
109 return c == '/';
110 #endif
111 }
112
CanonicalizePath(char * path,size_t * len,uint64_t * slash_bits,string * err)113 bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits,
114 string* err) {
115 // WARNING: this function is performance-critical; please benchmark
116 // any changes you make to it.
117 METRIC_RECORD("canonicalize path");
118 if (*len == 0) {
119 *err = "empty path";
120 return false;
121 }
122
123 const int kMaxPathComponents = 60;
124 char* components[kMaxPathComponents];
125 int component_count = 0;
126
127 char* start = path;
128 char* dst = start;
129 const char* src = start;
130 const char* end = start + *len;
131
132 if (IsPathSeparator(*src)) {
133 #ifdef _WIN32
134
135 // network path starts with //
136 if (*len > 1 && IsPathSeparator(*(src + 1))) {
137 src += 2;
138 dst += 2;
139 } else {
140 ++src;
141 ++dst;
142 }
143 #else
144 ++src;
145 ++dst;
146 #endif
147 }
148
149 while (src < end) {
150 if (*src == '.') {
151 if (src + 1 == end || IsPathSeparator(src[1])) {
152 // '.' component; eliminate.
153 src += 2;
154 continue;
155 } else if (src[1] == '.' && (src + 2 == end || IsPathSeparator(src[2]))) {
156 // '..' component. Back up if possible.
157 if (component_count > 0) {
158 dst = components[component_count - 1];
159 src += 3;
160 --component_count;
161 } else {
162 *dst++ = *src++;
163 *dst++ = *src++;
164 *dst++ = *src++;
165 }
166 continue;
167 }
168 }
169
170 if (IsPathSeparator(*src)) {
171 src++;
172 continue;
173 }
174
175 if (component_count == kMaxPathComponents)
176 Fatal("path has too many components : %s", path);
177 components[component_count] = dst;
178 ++component_count;
179
180 while (src != end && !IsPathSeparator(*src))
181 *dst++ = *src++;
182 *dst++ = *src++; // Copy '/' or final \0 character as well.
183 }
184
185 if (dst == start) {
186 *dst++ = '.';
187 *dst++ = '\0';
188 }
189
190 *len = dst - start - 1;
191 #ifdef _WIN32
192 uint64_t bits = 0;
193 uint64_t bits_mask = 1;
194
195 for (char* c = start; c < start + *len; ++c) {
196 switch (*c) {
197 case '\\':
198 bits |= bits_mask;
199 *c = '/';
200 NINJA_FALLTHROUGH;
201 case '/':
202 bits_mask <<= 1;
203 }
204 }
205
206 *slash_bits = bits;
207 #else
208 *slash_bits = 0;
209 #endif
210 return true;
211 }
212
IsKnownShellSafeCharacter(char ch)213 static inline bool IsKnownShellSafeCharacter(char ch) {
214 if ('A' <= ch && ch <= 'Z') return true;
215 if ('a' <= ch && ch <= 'z') return true;
216 if ('0' <= ch && ch <= '9') return true;
217
218 switch (ch) {
219 case '_':
220 case '+':
221 case '-':
222 case '.':
223 case '/':
224 return true;
225 default:
226 return false;
227 }
228 }
229
IsKnownWin32SafeCharacter(char ch)230 static inline bool IsKnownWin32SafeCharacter(char ch) {
231 switch (ch) {
232 case ' ':
233 case '"':
234 return false;
235 default:
236 return true;
237 }
238 }
239
StringNeedsShellEscaping(const string & input)240 static inline bool StringNeedsShellEscaping(const string& input) {
241 for (size_t i = 0; i < input.size(); ++i) {
242 if (!IsKnownShellSafeCharacter(input[i])) return true;
243 }
244 return false;
245 }
246
StringNeedsWin32Escaping(const string & input)247 static inline bool StringNeedsWin32Escaping(const string& input) {
248 for (size_t i = 0; i < input.size(); ++i) {
249 if (!IsKnownWin32SafeCharacter(input[i])) return true;
250 }
251 return false;
252 }
253
GetShellEscapedString(const string & input,string * result)254 void GetShellEscapedString(const string& input, string* result) {
255 assert(result);
256
257 if (!StringNeedsShellEscaping(input)) {
258 result->append(input);
259 return;
260 }
261
262 const char kQuote = '\'';
263 const char kEscapeSequence[] = "'\\'";
264
265 result->push_back(kQuote);
266
267 string::const_iterator span_begin = input.begin();
268 for (string::const_iterator it = input.begin(), end = input.end(); it != end;
269 ++it) {
270 if (*it == kQuote) {
271 result->append(span_begin, it);
272 result->append(kEscapeSequence);
273 span_begin = it;
274 }
275 }
276 result->append(span_begin, input.end());
277 result->push_back(kQuote);
278 }
279
280
GetWin32EscapedString(const string & input,string * result)281 void GetWin32EscapedString(const string& input, string* result) {
282 assert(result);
283 if (!StringNeedsWin32Escaping(input)) {
284 result->append(input);
285 return;
286 }
287
288 const char kQuote = '"';
289 const char kBackslash = '\\';
290
291 result->push_back(kQuote);
292 size_t consecutive_backslash_count = 0;
293 string::const_iterator span_begin = input.begin();
294 for (string::const_iterator it = input.begin(), end = input.end(); it != end;
295 ++it) {
296 switch (*it) {
297 case kBackslash:
298 ++consecutive_backslash_count;
299 break;
300 case kQuote:
301 result->append(span_begin, it);
302 result->append(consecutive_backslash_count + 1, kBackslash);
303 span_begin = it;
304 consecutive_backslash_count = 0;
305 break;
306 default:
307 consecutive_backslash_count = 0;
308 break;
309 }
310 }
311 result->append(span_begin, input.end());
312 result->append(consecutive_backslash_count, kBackslash);
313 result->push_back(kQuote);
314 }
315
ReadFile(const string & path,string * contents,string * err)316 int ReadFile(const string& path, string* contents, string* err) {
317 #ifdef _WIN32
318 // This makes a ninja run on a set of 1500 manifest files about 4% faster
319 // than using the generic fopen code below.
320 err->clear();
321 HANDLE f = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL,
322 OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL);
323 if (f == INVALID_HANDLE_VALUE) {
324 err->assign(GetLastErrorString());
325 return -ENOENT;
326 }
327
328 for (;;) {
329 DWORD len;
330 char buf[64 << 10];
331 if (!::ReadFile(f, buf, sizeof(buf), &len, NULL)) {
332 err->assign(GetLastErrorString());
333 contents->clear();
334 return -1;
335 }
336 if (len == 0)
337 break;
338 contents->append(buf, len);
339 }
340 ::CloseHandle(f);
341 return 0;
342 #else
343 FILE* f = fopen(path.c_str(), "rb");
344 if (!f) {
345 err->assign(strerror(errno));
346 return -errno;
347 }
348
349 struct stat st;
350 if (fstat(fileno(f), &st) < 0) {
351 err->assign(strerror(errno));
352 fclose(f);
353 return -errno;
354 }
355
356 // +1 is for the resize in ManifestParser::Load
357 contents->reserve(st.st_size + 1);
358
359 char buf[64 << 10];
360 size_t len;
361 while (!feof(f) && (len = fread(buf, 1, sizeof(buf), f)) > 0) {
362 contents->append(buf, len);
363 }
364 if (ferror(f)) {
365 err->assign(strerror(errno)); // XXX errno?
366 contents->clear();
367 fclose(f);
368 return -errno;
369 }
370 fclose(f);
371 return 0;
372 #endif
373 }
374
SetCloseOnExec(int fd)375 void SetCloseOnExec(int fd) {
376 #ifndef _WIN32
377 int flags = fcntl(fd, F_GETFD);
378 if (flags < 0) {
379 perror("fcntl(F_GETFD)");
380 } else {
381 if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0)
382 perror("fcntl(F_SETFD)");
383 }
384 #else
385 HANDLE hd = (HANDLE) _get_osfhandle(fd);
386 if (! SetHandleInformation(hd, HANDLE_FLAG_INHERIT, 0)) {
387 fprintf(stderr, "SetHandleInformation(): %s", GetLastErrorString().c_str());
388 }
389 #endif // ! _WIN32
390 }
391
392
SpellcheckStringV(const string & text,const vector<const char * > & words)393 const char* SpellcheckStringV(const string& text,
394 const vector<const char*>& words) {
395 const bool kAllowReplacements = true;
396 const int kMaxValidEditDistance = 3;
397
398 int min_distance = kMaxValidEditDistance + 1;
399 const char* result = NULL;
400 for (vector<const char*>::const_iterator i = words.begin();
401 i != words.end(); ++i) {
402 int distance = EditDistance(*i, text, kAllowReplacements,
403 kMaxValidEditDistance);
404 if (distance < min_distance) {
405 min_distance = distance;
406 result = *i;
407 }
408 }
409 return result;
410 }
411
SpellcheckString(const char * text,...)412 const char* SpellcheckString(const char* text, ...) {
413 // Note: This takes a const char* instead of a string& because using
414 // va_start() with a reference parameter is undefined behavior.
415 va_list ap;
416 va_start(ap, text);
417 vector<const char*> words;
418 const char* word;
419 while ((word = va_arg(ap, const char*)))
420 words.push_back(word);
421 va_end(ap);
422 return SpellcheckStringV(text, words);
423 }
424
425 #ifdef _WIN32
GetLastErrorString()426 string GetLastErrorString() {
427 DWORD err = GetLastError();
428
429 char* msg_buf;
430 FormatMessageA(
431 FORMAT_MESSAGE_ALLOCATE_BUFFER |
432 FORMAT_MESSAGE_FROM_SYSTEM |
433 FORMAT_MESSAGE_IGNORE_INSERTS,
434 NULL,
435 err,
436 MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
437 (char*)&msg_buf,
438 0,
439 NULL);
440 string msg = msg_buf;
441 LocalFree(msg_buf);
442 return msg;
443 }
444
Win32Fatal(const char * function,const char * hint)445 void Win32Fatal(const char* function, const char* hint) {
446 if (hint) {
447 Fatal("%s: %s (%s)", function, GetLastErrorString().c_str(), hint);
448 } else {
449 Fatal("%s: %s", function, GetLastErrorString().c_str());
450 }
451 }
452 #endif
453
islatinalpha(int c)454 bool islatinalpha(int c) {
455 // isalpha() is locale-dependent.
456 return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
457 }
458
StripAnsiEscapeCodes(const string & in)459 string StripAnsiEscapeCodes(const string& in) {
460 string stripped;
461 stripped.reserve(in.size());
462
463 for (size_t i = 0; i < in.size(); ++i) {
464 if (in[i] != '\33') {
465 // Not an escape code.
466 stripped.push_back(in[i]);
467 continue;
468 }
469
470 // Only strip CSIs for now.
471 if (i + 1 >= in.size()) break;
472 if (in[i + 1] != '[') continue; // Not a CSI.
473 i += 2;
474
475 // Skip everything up to and including the next [a-zA-Z].
476 while (i < in.size() && !islatinalpha(in[i]))
477 ++i;
478 }
479 return stripped;
480 }
481
GetProcessorCount()482 int GetProcessorCount() {
483 #ifdef _WIN32
484 return GetActiveProcessorCount(ALL_PROCESSOR_GROUPS);
485 #else
486 #ifdef CPU_COUNT
487 // The number of exposed processors might not represent the actual number of
488 // processors threads can run on. This happens when a CPU set limitation is
489 // active, see https://github.com/ninja-build/ninja/issues/1278
490 cpu_set_t set;
491 if (sched_getaffinity(getpid(), sizeof(set), &set) == 0) {
492 return CPU_COUNT(&set);
493 }
494 #endif
495 return sysconf(_SC_NPROCESSORS_ONLN);
496 #endif
497 }
498
499 #if defined(_WIN32) || defined(__CYGWIN__)
CalculateProcessorLoad(uint64_t idle_ticks,uint64_t total_ticks)500 static double CalculateProcessorLoad(uint64_t idle_ticks, uint64_t total_ticks)
501 {
502 static uint64_t previous_idle_ticks = 0;
503 static uint64_t previous_total_ticks = 0;
504 static double previous_load = -0.0;
505
506 uint64_t idle_ticks_since_last_time = idle_ticks - previous_idle_ticks;
507 uint64_t total_ticks_since_last_time = total_ticks - previous_total_ticks;
508
509 bool first_call = (previous_total_ticks == 0);
510 bool ticks_not_updated_since_last_call = (total_ticks_since_last_time == 0);
511
512 double load;
513 if (first_call || ticks_not_updated_since_last_call) {
514 load = previous_load;
515 } else {
516 // Calculate load.
517 double idle_to_total_ratio =
518 ((double)idle_ticks_since_last_time) / total_ticks_since_last_time;
519 double load_since_last_call = 1.0 - idle_to_total_ratio;
520
521 // Filter/smooth result when possible.
522 if(previous_load > 0) {
523 load = 0.9 * previous_load + 0.1 * load_since_last_call;
524 } else {
525 load = load_since_last_call;
526 }
527 }
528
529 previous_load = load;
530 previous_total_ticks = total_ticks;
531 previous_idle_ticks = idle_ticks;
532
533 return load;
534 }
535
FileTimeToTickCount(const FILETIME & ft)536 static uint64_t FileTimeToTickCount(const FILETIME & ft)
537 {
538 uint64_t high = (((uint64_t)(ft.dwHighDateTime)) << 32);
539 uint64_t low = ft.dwLowDateTime;
540 return (high | low);
541 }
542
GetLoadAverage()543 double GetLoadAverage() {
544 FILETIME idle_time, kernel_time, user_time;
545 BOOL get_system_time_succeeded =
546 GetSystemTimes(&idle_time, &kernel_time, &user_time);
547
548 double posix_compatible_load;
549 if (get_system_time_succeeded) {
550 uint64_t idle_ticks = FileTimeToTickCount(idle_time);
551
552 // kernel_time from GetSystemTimes already includes idle_time.
553 uint64_t total_ticks =
554 FileTimeToTickCount(kernel_time) + FileTimeToTickCount(user_time);
555
556 double processor_load = CalculateProcessorLoad(idle_ticks, total_ticks);
557 posix_compatible_load = processor_load * GetProcessorCount();
558
559 } else {
560 posix_compatible_load = -0.0;
561 }
562
563 return posix_compatible_load;
564 }
565 #elif defined(__PASE__)
GetLoadAverage()566 double GetLoadAverage() {
567 return -0.0f;
568 }
569 #elif defined(_AIX)
GetLoadAverage()570 double GetLoadAverage() {
571 perfstat_cpu_total_t cpu_stats;
572 if (perfstat_cpu_total(NULL, &cpu_stats, sizeof(cpu_stats), 1) < 0) {
573 return -0.0f;
574 }
575
576 // Calculation taken from comment in libperfstats.h
577 return double(cpu_stats.loadavg[0]) / double(1 << SBITS);
578 }
579 #elif defined(__UCLIBC__) || (defined(__BIONIC__) && __ANDROID_API__ < 29)
GetLoadAverage()580 double GetLoadAverage() {
581 struct sysinfo si;
582 if (sysinfo(&si) != 0)
583 return -0.0f;
584 return 1.0 / (1 << SI_LOAD_SHIFT) * si.loads[0];
585 }
586 #else
GetLoadAverage()587 double GetLoadAverage() {
588 double loadavg[3] = { 0.0f, 0.0f, 0.0f };
589 if (getloadavg(loadavg, 3) < 0) {
590 // Maybe we should return an error here or the availability of
591 // getloadavg(3) should be checked when ninja is configured.
592 return -0.0f;
593 }
594 return loadavg[0];
595 }
596 #endif // _WIN32
597
ElideMiddle(const string & str,size_t width)598 string ElideMiddle(const string& str, size_t width) {
599 switch (width) {
600 case 0: return "";
601 case 1: return ".";
602 case 2: return "..";
603 case 3: return "...";
604 }
605 const int kMargin = 3; // Space for "...".
606 string result = str;
607 if (result.size() > width) {
608 size_t elide_size = (width - kMargin) / 2;
609 result = result.substr(0, elide_size)
610 + "..."
611 + result.substr(result.size() - elide_size, elide_size);
612 }
613 return result;
614 }
615
Truncate(const string & path,size_t size,string * err)616 bool Truncate(const string& path, size_t size, string* err) {
617 #ifdef _WIN32
618 int fh = _sopen(path.c_str(), _O_RDWR | _O_CREAT, _SH_DENYNO,
619 _S_IREAD | _S_IWRITE);
620 int success = _chsize(fh, size);
621 _close(fh);
622 #else
623 int success = truncate(path.c_str(), size);
624 #endif
625 // Both truncate() and _chsize() return 0 on success and set errno and return
626 // -1 on failure.
627 if (success < 0) {
628 *err = strerror(errno);
629 return false;
630 }
631 return true;
632 }
633