1 // Copyright 2012 Google Inc. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "deps_log.h"
16
17 #include <sys/stat.h>
18 #ifndef _WIN32
19 #include <unistd.h>
20 #endif
21
22 #include "graph.h"
23 #include "util.h"
24 #include "test.h"
25
26 using namespace std;
27
28 namespace {
29
30 const char kTestFilename[] = "DepsLogTest-tempfile";
31
32 struct DepsLogTest : public testing::Test {
SetUp__anona94034180111::DepsLogTest33 virtual void SetUp() {
34 // In case a crashing test left a stale file behind.
35 unlink(kTestFilename);
36 }
TearDown__anona94034180111::DepsLogTest37 virtual void TearDown() {
38 unlink(kTestFilename);
39 }
40 };
41
TEST_F(DepsLogTest,WriteRead)42 TEST_F(DepsLogTest, WriteRead) {
43 State state1;
44 DepsLog log1;
45 string err;
46 EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
47 ASSERT_EQ("", err);
48
49 {
50 vector<Node*> deps;
51 deps.push_back(state1.GetNode("foo.h", 0));
52 deps.push_back(state1.GetNode("bar.h", 0));
53 log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps);
54
55 deps.clear();
56 deps.push_back(state1.GetNode("foo.h", 0));
57 deps.push_back(state1.GetNode("bar2.h", 0));
58 log1.RecordDeps(state1.GetNode("out2.o", 0), 2, deps);
59
60 DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0));
61 ASSERT_TRUE(log_deps);
62 ASSERT_EQ(1, log_deps->mtime);
63 ASSERT_EQ(2, log_deps->node_count);
64 ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
65 ASSERT_EQ("bar.h", log_deps->nodes[1]->path());
66 }
67
68 log1.Close();
69
70 State state2;
71 DepsLog log2;
72 EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
73 ASSERT_EQ("", err);
74
75 ASSERT_EQ(log1.nodes().size(), log2.nodes().size());
76 for (int i = 0; i < (int)log1.nodes().size(); ++i) {
77 Node* node1 = log1.nodes()[i];
78 Node* node2 = log2.nodes()[i];
79 ASSERT_EQ(i, node1->id());
80 ASSERT_EQ(node1->id(), node2->id());
81 }
82
83 // Spot-check the entries in log2.
84 DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o", 0));
85 ASSERT_TRUE(log_deps);
86 ASSERT_EQ(2, log_deps->mtime);
87 ASSERT_EQ(2, log_deps->node_count);
88 ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
89 ASSERT_EQ("bar2.h", log_deps->nodes[1]->path());
90 }
91
TEST_F(DepsLogTest,LotsOfDeps)92 TEST_F(DepsLogTest, LotsOfDeps) {
93 const int kNumDeps = 100000; // More than 64k.
94
95 State state1;
96 DepsLog log1;
97 string err;
98 EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
99 ASSERT_EQ("", err);
100
101 {
102 vector<Node*> deps;
103 for (int i = 0; i < kNumDeps; ++i) {
104 char buf[32];
105 sprintf(buf, "file%d.h", i);
106 deps.push_back(state1.GetNode(buf, 0));
107 }
108 log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps);
109
110 DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0));
111 ASSERT_EQ(kNumDeps, log_deps->node_count);
112 }
113
114 log1.Close();
115
116 State state2;
117 DepsLog log2;
118 EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
119 ASSERT_EQ("", err);
120
121 DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o", 0));
122 ASSERT_EQ(kNumDeps, log_deps->node_count);
123 }
124
125 // Verify that adding the same deps twice doesn't grow the file.
TEST_F(DepsLogTest,DoubleEntry)126 TEST_F(DepsLogTest, DoubleEntry) {
127 // Write some deps to the file and grab its size.
128 int file_size;
129 {
130 State state;
131 DepsLog log;
132 string err;
133 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
134 ASSERT_EQ("", err);
135
136 vector<Node*> deps;
137 deps.push_back(state.GetNode("foo.h", 0));
138 deps.push_back(state.GetNode("bar.h", 0));
139 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
140 log.Close();
141 #ifdef __USE_LARGEFILE64
142 struct stat64 st;
143 ASSERT_EQ(0, stat64(kTestFilename, &st));
144 #else
145 struct stat st;
146 ASSERT_EQ(0, stat(kTestFilename, &st));
147 #endif
148 file_size = (int)st.st_size;
149 ASSERT_GT(file_size, 0);
150 }
151
152 // Now reload the file, and read the same deps.
153 {
154 State state;
155 DepsLog log;
156 string err;
157 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
158
159 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
160 ASSERT_EQ("", err);
161
162 vector<Node*> deps;
163 deps.push_back(state.GetNode("foo.h", 0));
164 deps.push_back(state.GetNode("bar.h", 0));
165 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
166 log.Close();
167 #ifdef __USE_LARGEFILE64
168 struct stat64 st;
169 ASSERT_EQ(0, stat64(kTestFilename, &st));
170 #else
171 struct stat st;
172 ASSERT_EQ(0, stat(kTestFilename, &st));
173 #endif
174 int file_size_2 = (int)st.st_size;
175 ASSERT_EQ(file_size, file_size_2);
176 }
177 }
178
179 // Verify that adding the new deps works and can be compacted away.
TEST_F(DepsLogTest,Recompact)180 TEST_F(DepsLogTest, Recompact) {
181 const char kManifest[] =
182 "rule cc\n"
183 " command = cc\n"
184 " deps = gcc\n"
185 "build out.o: cc\n"
186 "build other_out.o: cc\n";
187
188 // Write some deps to the file and grab its size.
189 int file_size;
190 {
191 State state;
192 ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
193 DepsLog log;
194 string err;
195 ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
196 ASSERT_EQ("", err);
197
198 vector<Node*> deps;
199 deps.push_back(state.GetNode("foo.h", 0));
200 deps.push_back(state.GetNode("bar.h", 0));
201 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
202
203 deps.clear();
204 deps.push_back(state.GetNode("foo.h", 0));
205 deps.push_back(state.GetNode("baz.h", 0));
206 log.RecordDeps(state.GetNode("other_out.o", 0), 1, deps);
207
208 log.Close();
209 #ifdef __USE_LARGEFILE64
210 struct stat64 st;
211 ASSERT_EQ(0, stat64(kTestFilename, &st));
212 #else
213 struct stat st;
214 ASSERT_EQ(0, stat(kTestFilename, &st));
215 #endif
216 file_size = (int)st.st_size;
217 ASSERT_GT(file_size, 0);
218 }
219
220 // Now reload the file, and add slightly different deps.
221 int file_size_2;
222 {
223 State state;
224 ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
225 DepsLog log;
226 string err;
227 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
228
229 ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
230 ASSERT_EQ("", err);
231
232 vector<Node*> deps;
233 deps.push_back(state.GetNode("foo.h", 0));
234 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
235 log.Close();
236
237 #ifdef __USE_LARGEFILE64
238 struct stat64 st;
239 ASSERT_EQ(0, stat64(kTestFilename, &st));
240 #else
241 struct stat st;
242 ASSERT_EQ(0, stat(kTestFilename, &st));
243 #endif
244 file_size_2 = (int)st.st_size;
245 // The file should grow to record the new deps.
246 ASSERT_GT(file_size_2, file_size);
247 }
248
249 // Now reload the file, verify the new deps have replaced the old, then
250 // recompact.
251 int file_size_3;
252 {
253 State state;
254 ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
255 DepsLog log;
256 string err;
257 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
258
259 Node* out = state.GetNode("out.o", 0);
260 DepsLog::Deps* deps = log.GetDeps(out);
261 ASSERT_TRUE(deps);
262 ASSERT_EQ(1, deps->mtime);
263 ASSERT_EQ(1, deps->node_count);
264 ASSERT_EQ("foo.h", deps->nodes[0]->path());
265
266 Node* other_out = state.GetNode("other_out.o", 0);
267 deps = log.GetDeps(other_out);
268 ASSERT_TRUE(deps);
269 ASSERT_EQ(1, deps->mtime);
270 ASSERT_EQ(2, deps->node_count);
271 ASSERT_EQ("foo.h", deps->nodes[0]->path());
272 ASSERT_EQ("baz.h", deps->nodes[1]->path());
273
274 ASSERT_TRUE(log.Recompact(kTestFilename, &err));
275
276 // The in-memory deps graph should still be valid after recompaction.
277 deps = log.GetDeps(out);
278 ASSERT_TRUE(deps);
279 ASSERT_EQ(1, deps->mtime);
280 ASSERT_EQ(1, deps->node_count);
281 ASSERT_EQ("foo.h", deps->nodes[0]->path());
282 ASSERT_EQ(out, log.nodes()[out->id()]);
283
284 deps = log.GetDeps(other_out);
285 ASSERT_TRUE(deps);
286 ASSERT_EQ(1, deps->mtime);
287 ASSERT_EQ(2, deps->node_count);
288 ASSERT_EQ("foo.h", deps->nodes[0]->path());
289 ASSERT_EQ("baz.h", deps->nodes[1]->path());
290 ASSERT_EQ(other_out, log.nodes()[other_out->id()]);
291
292 // The file should have shrunk a bit for the smaller deps.
293 #ifdef __USE_LARGEFILE64
294 struct stat64 st;
295 ASSERT_EQ(0, stat64(kTestFilename, &st));
296 #else
297 struct stat st;
298 ASSERT_EQ(0, stat(kTestFilename, &st));
299 #endif
300 file_size_3 = (int)st.st_size;
301 ASSERT_LT(file_size_3, file_size_2);
302 }
303
304 // Now reload the file and recompact with an empty manifest. The previous
305 // entries should be removed.
306 {
307 State state;
308 // Intentionally not parsing kManifest here.
309 DepsLog log;
310 string err;
311 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
312
313 Node* out = state.GetNode("out.o", 0);
314 DepsLog::Deps* deps = log.GetDeps(out);
315 ASSERT_TRUE(deps);
316 ASSERT_EQ(1, deps->mtime);
317 ASSERT_EQ(1, deps->node_count);
318 ASSERT_EQ("foo.h", deps->nodes[0]->path());
319
320 Node* other_out = state.GetNode("other_out.o", 0);
321 deps = log.GetDeps(other_out);
322 ASSERT_TRUE(deps);
323 ASSERT_EQ(1, deps->mtime);
324 ASSERT_EQ(2, deps->node_count);
325 ASSERT_EQ("foo.h", deps->nodes[0]->path());
326 ASSERT_EQ("baz.h", deps->nodes[1]->path());
327
328 ASSERT_TRUE(log.Recompact(kTestFilename, &err));
329
330 // The previous entries should have been removed.
331 deps = log.GetDeps(out);
332 ASSERT_FALSE(deps);
333
334 deps = log.GetDeps(other_out);
335 ASSERT_FALSE(deps);
336
337 // The .h files pulled in via deps should no longer have ids either.
338 ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
339 ASSERT_EQ(-1, state.LookupNode("baz.h")->id());
340
341 // The file should have shrunk more.
342 #ifdef __USE_LARGEFILE64
343 struct stat64 st;
344 ASSERT_EQ(0, stat64(kTestFilename, &st));
345 #else
346 struct stat st;
347 ASSERT_EQ(0, stat(kTestFilename, &st));
348 #endif
349 int file_size_4 = (int)st.st_size;
350 ASSERT_LT(file_size_4, file_size_3);
351 }
352 }
353
354 // Verify that invalid file headers cause a new build.
TEST_F(DepsLogTest,InvalidHeader)355 TEST_F(DepsLogTest, InvalidHeader) {
356 const char *kInvalidHeaders[] = {
357 "", // Empty file.
358 "# ninjad", // Truncated first line.
359 "# ninjadeps\n", // No version int.
360 "# ninjadeps\n\001\002", // Truncated version int.
361 "# ninjadeps\n\001\002\003\004" // Invalid version int.
362 };
363 for (size_t i = 0; i < sizeof(kInvalidHeaders) / sizeof(kInvalidHeaders[0]);
364 ++i) {
365 FILE* deps_log = fopen(kTestFilename, "wb");
366 ASSERT_TRUE(deps_log != NULL);
367 ASSERT_EQ(
368 strlen(kInvalidHeaders[i]),
369 fwrite(kInvalidHeaders[i], 1, strlen(kInvalidHeaders[i]), deps_log));
370 ASSERT_EQ(0 ,fclose(deps_log));
371
372 string err;
373 DepsLog log;
374 State state;
375 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
376 EXPECT_EQ("bad deps log signature or version; starting over", err);
377 }
378 }
379
380 // Simulate what happens when loading a truncated log file.
TEST_F(DepsLogTest,Truncated)381 TEST_F(DepsLogTest, Truncated) {
382 // Create a file with some entries.
383 {
384 State state;
385 DepsLog log;
386 string err;
387 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
388 ASSERT_EQ("", err);
389
390 vector<Node*> deps;
391 deps.push_back(state.GetNode("foo.h", 0));
392 deps.push_back(state.GetNode("bar.h", 0));
393 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
394
395 deps.clear();
396 deps.push_back(state.GetNode("foo.h", 0));
397 deps.push_back(state.GetNode("bar2.h", 0));
398 log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
399
400 log.Close();
401 }
402
403 // Get the file size.
404 #ifdef __USE_LARGEFILE64
405 struct stat64 st;
406 ASSERT_EQ(0, stat64(kTestFilename, &st));
407 #else
408 struct stat st;
409 ASSERT_EQ(0, stat(kTestFilename, &st));
410 #endif
411
412 // Try reloading at truncated sizes.
413 // Track how many nodes/deps were found; they should decrease with
414 // smaller sizes.
415 int node_count = 5;
416 int deps_count = 2;
417 for (int size = (int)st.st_size; size > 0; --size) {
418 string err;
419 ASSERT_TRUE(Truncate(kTestFilename, size, &err));
420
421 State state;
422 DepsLog log;
423 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
424 if (!err.empty()) {
425 // At some point the log will be so short as to be unparsable.
426 break;
427 }
428
429 ASSERT_GE(node_count, (int)log.nodes().size());
430 node_count = log.nodes().size();
431
432 // Count how many non-NULL deps entries there are.
433 int new_deps_count = 0;
434 for (vector<DepsLog::Deps*>::const_iterator i = log.deps().begin();
435 i != log.deps().end(); ++i) {
436 if (*i)
437 ++new_deps_count;
438 }
439 ASSERT_GE(deps_count, new_deps_count);
440 deps_count = new_deps_count;
441 }
442 }
443
444 // Run the truncation-recovery logic.
TEST_F(DepsLogTest,TruncatedRecovery)445 TEST_F(DepsLogTest, TruncatedRecovery) {
446 // Create a file with some entries.
447 {
448 State state;
449 DepsLog log;
450 string err;
451 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
452 ASSERT_EQ("", err);
453
454 vector<Node*> deps;
455 deps.push_back(state.GetNode("foo.h", 0));
456 deps.push_back(state.GetNode("bar.h", 0));
457 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
458
459 deps.clear();
460 deps.push_back(state.GetNode("foo.h", 0));
461 deps.push_back(state.GetNode("bar2.h", 0));
462 log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
463
464 log.Close();
465 }
466
467 // Shorten the file, corrupting the last record.
468 {
469 #ifdef __USE_LARGEFILE64
470 struct stat64 st;
471 ASSERT_EQ(0, stat64(kTestFilename, &st));
472 #else
473 struct stat st;
474 ASSERT_EQ(0, stat(kTestFilename, &st));
475 #endif
476 string err;
477 ASSERT_TRUE(Truncate(kTestFilename, st.st_size - 2, &err));
478 }
479
480 // Load the file again, add an entry.
481 {
482 State state;
483 DepsLog log;
484 string err;
485 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
486 ASSERT_EQ("premature end of file; recovering", err);
487 err.clear();
488
489 // The truncated entry should've been discarded.
490 EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o", 0)));
491
492 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
493 ASSERT_EQ("", err);
494
495 // Add a new entry.
496 vector<Node*> deps;
497 deps.push_back(state.GetNode("foo.h", 0));
498 deps.push_back(state.GetNode("bar2.h", 0));
499 log.RecordDeps(state.GetNode("out2.o", 0), 3, deps);
500
501 log.Close();
502 }
503
504 // Load the file a third time to verify appending after a mangled
505 // entry doesn't break things.
506 {
507 State state;
508 DepsLog log;
509 string err;
510 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
511
512 // The truncated entry should exist.
513 DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o", 0));
514 ASSERT_TRUE(deps);
515 }
516 }
517
TEST_F(DepsLogTest,ReverseDepsNodes)518 TEST_F(DepsLogTest, ReverseDepsNodes) {
519 State state;
520 DepsLog log;
521 string err;
522 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
523 ASSERT_EQ("", err);
524
525 vector<Node*> deps;
526 deps.push_back(state.GetNode("foo.h", 0));
527 deps.push_back(state.GetNode("bar.h", 0));
528 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
529
530 deps.clear();
531 deps.push_back(state.GetNode("foo.h", 0));
532 deps.push_back(state.GetNode("bar2.h", 0));
533 log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
534
535 log.Close();
536
537 Node* rev_deps = log.GetFirstReverseDepsNode(state.GetNode("foo.h", 0));
538 EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0) ||
539 rev_deps == state.GetNode("out2.o", 0));
540
541 rev_deps = log.GetFirstReverseDepsNode(state.GetNode("bar.h", 0));
542 EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0));
543 }
544
545 } // anonymous namespace
546