1 // Copyright 2016 Google Inc. All rights reserved
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 // +build ignore
16
17 #include "regen.h"
18
19 #include <sys/stat.h>
20
21 #include <algorithm>
22 #include <memory>
23 #include <mutex>
24 #include <vector>
25
26 #include "affinity.h"
27 #include "fileutil.h"
28 #include "find.h"
29 #include "func.h"
30 #include "io.h"
31 #include "log.h"
32 #include "ninja.h"
33 #include "stats.h"
34 #include "strutil.h"
35 #include "thread_pool.h"
36
37 namespace {
38
39 #define RETURN_TRUE \
40 do { \
41 if (g_flags.dump_kati_stamp) \
42 needs_regen_ = true; \
43 else \
44 return true; \
45 } while (0)
46
ShouldIgnoreDirty(StringPiece s)47 bool ShouldIgnoreDirty(StringPiece s) {
48 Pattern pat(g_flags.ignore_dirty_pattern);
49 Pattern nopat(g_flags.no_ignore_dirty_pattern);
50 return pat.Match(s) && !nopat.Match(s);
51 }
52
53 class StampChecker {
54 struct GlobResult {
55 string pat;
56 vector<string> result;
57 };
58
59 struct ShellResult {
60 CommandOp op;
61 string shell;
62 string shellflag;
63 string cmd;
64 string result;
65 vector<string> missing_dirs;
66 vector<string> files;
67 vector<string> read_dirs;
68 };
69
70 public:
StampChecker()71 StampChecker() : needs_regen_(false) {}
72
~StampChecker()73 ~StampChecker() {
74 for (GlobResult* gr : globs_) {
75 delete gr;
76 }
77 for (ShellResult* sr : commands_) {
78 delete sr;
79 }
80 }
81
NeedsRegen(double start_time,const string & orig_args)82 bool NeedsRegen(double start_time, const string& orig_args) {
83 if (IsMissingOutputs())
84 RETURN_TRUE;
85
86 if (CheckStep1(orig_args))
87 RETURN_TRUE;
88
89 if (CheckStep2())
90 RETURN_TRUE;
91
92 if (!needs_regen_) {
93 FILE* fp = fopen(GetNinjaStampFilename().c_str(), "rb+");
94 if (!fp)
95 return true;
96 ScopedFile sfp(fp);
97 if (fseek(fp, 0, SEEK_SET) < 0)
98 PERROR("fseek");
99 size_t r = fwrite(&start_time, sizeof(start_time), 1, fp);
100 CHECK(r == 1);
101 }
102 return needs_regen_;
103 }
104
105 private:
IsMissingOutputs()106 bool IsMissingOutputs() {
107 if (!Exists(GetNinjaFilename())) {
108 fprintf(stderr, "%s is missing, regenerating...\n",
109 GetNinjaFilename().c_str());
110 return true;
111 }
112 if (!Exists(GetNinjaShellScriptFilename())) {
113 fprintf(stderr, "%s is missing, regenerating...\n",
114 GetNinjaShellScriptFilename().c_str());
115 return true;
116 }
117 return false;
118 }
119
CheckStep1(const string & orig_args)120 bool CheckStep1(const string& orig_args) {
121 #define LOAD_INT(fp) \
122 ({ \
123 int v = LoadInt(fp); \
124 if (v < 0) { \
125 fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \
126 RETURN_TRUE; \
127 } \
128 v; \
129 })
130
131 #define LOAD_STRING(fp, s) \
132 ({ \
133 if (!LoadString(fp, s)) { \
134 fprintf(stderr, "incomplete kati_stamp, regenerating...\n"); \
135 RETURN_TRUE; \
136 } \
137 })
138
139 const string& stamp_filename = GetNinjaStampFilename();
140 FILE* fp = fopen(stamp_filename.c_str(), "rb");
141 if (!fp) {
142 if (g_flags.regen_debug)
143 printf("%s: %s\n", stamp_filename.c_str(), strerror(errno));
144 return true;
145 }
146 ScopedFile sfp(fp);
147
148 double gen_time;
149 size_t r = fread(&gen_time, sizeof(gen_time), 1, fp);
150 gen_time_ = gen_time;
151 if (r != 1) {
152 fprintf(stderr, "incomplete kati_stamp, regenerating...\n");
153 RETURN_TRUE;
154 }
155 if (g_flags.regen_debug)
156 printf("Generated time: %f\n", gen_time);
157
158 string s, s2;
159 int num_files = LOAD_INT(fp);
160 for (int i = 0; i < num_files; i++) {
161 LOAD_STRING(fp, &s);
162 double ts = GetTimestamp(s);
163 if (gen_time < ts) {
164 if (g_flags.regen_ignoring_kati_binary) {
165 string kati_binary;
166 GetExecutablePath(&kati_binary);
167 if (s == kati_binary) {
168 fprintf(stderr, "%s was modified, ignored.\n", s.c_str());
169 continue;
170 }
171 }
172 if (ShouldIgnoreDirty(s)) {
173 if (g_flags.regen_debug)
174 printf("file %s: ignored (%f)\n", s.c_str(), ts);
175 continue;
176 }
177 if (g_flags.dump_kati_stamp)
178 printf("file %s: dirty (%f)\n", s.c_str(), ts);
179 else
180 fprintf(stderr, "%s was modified, regenerating...\n", s.c_str());
181 RETURN_TRUE;
182 } else if (g_flags.dump_kati_stamp) {
183 printf("file %s: clean (%f)\n", s.c_str(), ts);
184 }
185 }
186
187 int num_undefineds = LOAD_INT(fp);
188 for (int i = 0; i < num_undefineds; i++) {
189 LOAD_STRING(fp, &s);
190 if (getenv(s.c_str())) {
191 if (g_flags.dump_kati_stamp) {
192 printf("env %s: dirty (unset => %s)\n", s.c_str(), getenv(s.c_str()));
193 } else {
194 fprintf(stderr, "Environment variable %s was set, regenerating...\n",
195 s.c_str());
196 }
197 RETURN_TRUE;
198 } else if (g_flags.dump_kati_stamp) {
199 printf("env %s: clean (unset)\n", s.c_str());
200 }
201 }
202
203 int num_envs = LOAD_INT(fp);
204 for (int i = 0; i < num_envs; i++) {
205 LOAD_STRING(fp, &s);
206 StringPiece val(getenv(s.c_str()));
207 LOAD_STRING(fp, &s2);
208 if (val != s2) {
209 if (g_flags.dump_kati_stamp) {
210 printf("env %s: dirty (%s => %.*s)\n", s.c_str(), s2.c_str(),
211 SPF(val));
212 } else {
213 fprintf(stderr,
214 "Environment variable %s was modified (%s => %.*s), "
215 "regenerating...\n",
216 s.c_str(), s2.c_str(), SPF(val));
217 }
218 RETURN_TRUE;
219 } else if (g_flags.dump_kati_stamp) {
220 printf("env %s: clean (%.*s)\n", s.c_str(), SPF(val));
221 }
222 }
223
224 int num_globs = LOAD_INT(fp);
225 string pat;
226 for (int i = 0; i < num_globs; i++) {
227 GlobResult* gr = new GlobResult;
228 globs_.push_back(gr);
229
230 LOAD_STRING(fp, &gr->pat);
231 int num_files = LOAD_INT(fp);
232 gr->result.resize(num_files);
233 for (int j = 0; j < num_files; j++) {
234 LOAD_STRING(fp, &gr->result[j]);
235 }
236 }
237
238 int num_crs = LOAD_INT(fp);
239 for (int i = 0; i < num_crs; i++) {
240 ShellResult* sr = new ShellResult;
241 commands_.push_back(sr);
242 sr->op = static_cast<CommandOp>(LOAD_INT(fp));
243 LOAD_STRING(fp, &sr->shell);
244 LOAD_STRING(fp, &sr->shellflag);
245 LOAD_STRING(fp, &sr->cmd);
246 LOAD_STRING(fp, &sr->result);
247
248 if (sr->op == CommandOp::FIND) {
249 int num_missing_dirs = LOAD_INT(fp);
250 for (int j = 0; j < num_missing_dirs; j++) {
251 LOAD_STRING(fp, &s);
252 sr->missing_dirs.push_back(s);
253 }
254 int num_files = LOAD_INT(fp);
255 for (int j = 0; j < num_files; j++) {
256 LOAD_STRING(fp, &s);
257 sr->files.push_back(s);
258 }
259 int num_read_dirs = LOAD_INT(fp);
260 for (int j = 0; j < num_read_dirs; j++) {
261 LOAD_STRING(fp, &s);
262 sr->read_dirs.push_back(s);
263 }
264 }
265 }
266
267 LoadString(fp, &s);
268 if (orig_args != s) {
269 fprintf(stderr, "arguments changed, regenerating...\n");
270 RETURN_TRUE;
271 }
272
273 return needs_regen_;
274 }
275
CheckGlobResult(const GlobResult * gr,string * err)276 bool CheckGlobResult(const GlobResult* gr, string* err) {
277 COLLECT_STATS("glob time (regen)");
278 vector<string>* files;
279 Glob(gr->pat.c_str(), &files);
280 bool needs_regen = files->size() != gr->result.size();
281 for (size_t i = 0; i < gr->result.size(); i++) {
282 if (!needs_regen) {
283 if ((*files)[i] != gr->result[i]) {
284 needs_regen = true;
285 break;
286 }
287 }
288 }
289 if (needs_regen) {
290 if (ShouldIgnoreDirty(gr->pat)) {
291 if (g_flags.dump_kati_stamp) {
292 printf("wildcard %s: ignored\n", gr->pat.c_str());
293 }
294 return false;
295 }
296 if (g_flags.dump_kati_stamp) {
297 printf("wildcard %s: dirty\n", gr->pat.c_str());
298 } else {
299 *err = StringPrintf("wildcard(%s) was changed, regenerating...\n",
300 gr->pat.c_str());
301 }
302 } else if (g_flags.dump_kati_stamp) {
303 printf("wildcard %s: clean\n", gr->pat.c_str());
304 }
305 return needs_regen;
306 }
307
ShouldRunCommand(const ShellResult * sr)308 bool ShouldRunCommand(const ShellResult* sr) {
309 if (sr->op != CommandOp::FIND)
310 return true;
311
312 COLLECT_STATS("stat time (regen)");
313 for (const string& dir : sr->missing_dirs) {
314 if (Exists(dir))
315 return true;
316 }
317 for (const string& file : sr->files) {
318 if (!Exists(file))
319 return true;
320 }
321 for (const string& dir : sr->read_dirs) {
322 // We assume we rarely do a significant change for the top
323 // directory which affects the results of find command.
324 if (dir == "" || dir == "." || ShouldIgnoreDirty(dir))
325 continue;
326
327 struct stat st;
328 if (lstat(dir.c_str(), &st) != 0) {
329 return true;
330 }
331 double ts = GetTimestampFromStat(st);
332 if (gen_time_ < ts) {
333 return true;
334 }
335 if (S_ISLNK(st.st_mode)) {
336 ts = GetTimestamp(dir);
337 if (ts < 0 || gen_time_ < ts)
338 return true;
339 }
340 }
341 return false;
342 }
343
CheckShellResult(const ShellResult * sr,string * err)344 bool CheckShellResult(const ShellResult* sr, string* err) {
345 if (sr->op == CommandOp::READ_MISSING) {
346 if (Exists(sr->cmd)) {
347 if (g_flags.dump_kati_stamp)
348 printf("file %s: dirty\n", sr->cmd.c_str());
349 else
350 *err = StringPrintf("$(file <%s) was changed, regenerating...\n",
351 sr->cmd.c_str());
352 return true;
353 }
354 if (g_flags.dump_kati_stamp)
355 printf("file %s: clean\n", sr->cmd.c_str());
356 return false;
357 }
358
359 if (sr->op == CommandOp::READ) {
360 double ts = GetTimestamp(sr->cmd);
361 if (gen_time_ < ts) {
362 if (g_flags.dump_kati_stamp)
363 printf("file %s: dirty\n", sr->cmd.c_str());
364 else
365 *err = StringPrintf("$(file <%s) was changed, regenerating...\n",
366 sr->cmd.c_str());
367 return true;
368 }
369 if (g_flags.dump_kati_stamp)
370 printf("file %s: clean\n", sr->cmd.c_str());
371 return false;
372 }
373
374 if (sr->op == CommandOp::WRITE || sr->op == CommandOp::APPEND) {
375 FILE* f =
376 fopen(sr->cmd.c_str(), (sr->op == CommandOp::WRITE) ? "wb" : "ab");
377 if (f == NULL) {
378 PERROR("fopen");
379 }
380
381 if (fwrite(&sr->result[0], sr->result.size(), 1, f) != 1) {
382 PERROR("fwrite");
383 }
384
385 if (fclose(f) != 0) {
386 PERROR("fclose");
387 }
388
389 if (g_flags.dump_kati_stamp)
390 printf("file %s: clean (write)\n", sr->cmd.c_str());
391 return false;
392 }
393
394 if (!ShouldRunCommand(sr)) {
395 if (g_flags.regen_debug)
396 printf("shell %s: clean (no rerun)\n", sr->cmd.c_str());
397 return false;
398 }
399
400 FindCommand fc;
401 if (fc.Parse(sr->cmd) && !fc.chdir.empty() && ShouldIgnoreDirty(fc.chdir)) {
402 if (g_flags.dump_kati_stamp)
403 printf("shell %s: ignored\n", sr->cmd.c_str());
404 return false;
405 }
406
407 COLLECT_STATS_WITH_SLOW_REPORT("shell time (regen)", sr->cmd.c_str());
408 string result;
409 RunCommand(sr->shell, sr->shellflag, sr->cmd, RedirectStderr::DEV_NULL,
410 &result);
411 FormatForCommandSubstitution(&result);
412 if (sr->result != result) {
413 if (g_flags.dump_kati_stamp) {
414 printf("shell %s: dirty\n", sr->cmd.c_str());
415 } else {
416 *err = StringPrintf("$(shell %s) was changed, regenerating...\n",
417 sr->cmd.c_str());
418 //*err += StringPrintf("%s => %s\n", expected.c_str(), result.c_str());
419 }
420 return true;
421 } else if (g_flags.regen_debug) {
422 printf("shell %s: clean (rerun)\n", sr->cmd.c_str());
423 }
424 return false;
425 }
426
CheckStep2()427 bool CheckStep2() {
428 unique_ptr<ThreadPool> tp(NewThreadPool(g_flags.num_jobs));
429
430 tp->Submit([this]() {
431 string err;
432 // TODO: Make glob cache thread safe and create a task for each glob.
433 SetAffinityForSingleThread();
434 for (GlobResult* gr : globs_) {
435 if (CheckGlobResult(gr, &err)) {
436 unique_lock<mutex> lock(mu_);
437 if (!needs_regen_) {
438 needs_regen_ = true;
439 msg_ = err;
440 }
441 break;
442 }
443 }
444 });
445
446 tp->Submit([this]() {
447 SetAffinityForSingleThread();
448 for (ShellResult* sr : commands_) {
449 string err;
450 if (CheckShellResult(sr, &err)) {
451 unique_lock<mutex> lock(mu_);
452 if (!needs_regen_) {
453 needs_regen_ = true;
454 msg_ = err;
455 }
456 }
457 }
458 });
459
460 tp->Wait();
461 if (needs_regen_) {
462 fprintf(stderr, "%s", msg_.c_str());
463 }
464 return needs_regen_;
465 }
466
467 private:
468 double gen_time_;
469 vector<GlobResult*> globs_;
470 vector<ShellResult*> commands_;
471 mutex mu_;
472 bool needs_regen_;
473 string msg_;
474 };
475
476 } // namespace
477
NeedsRegen(double start_time,const string & orig_args)478 bool NeedsRegen(double start_time, const string& orig_args) {
479 return StampChecker().NeedsRegen(start_time, orig_args);
480 }
481