1 // Copyright 2012 Google Inc. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "deps_log.h"
16
17 #include <sys/stat.h>
18 #ifndef _WIN32
19 #include <unistd.h>
20 #endif
21
22 #include "graph.h"
23 #include "util.h"
24 #include "test.h"
25
26 namespace {
27
28 const char kTestFilename[] = "DepsLogTest-tempfile";
29
30 struct DepsLogTest : public testing::Test {
SetUp__anonfd16347d0111::DepsLogTest31 virtual void SetUp() {
32 // In case a crashing test left a stale file behind.
33 unlink(kTestFilename);
34 }
TearDown__anonfd16347d0111::DepsLogTest35 virtual void TearDown() {
36 unlink(kTestFilename);
37 }
38 };
39
TEST_F(DepsLogTest,WriteRead)40 TEST_F(DepsLogTest, WriteRead) {
41 State state1;
42 DepsLog log1;
43 string err;
44 EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
45 ASSERT_EQ("", err);
46
47 {
48 vector<Node*> deps;
49 deps.push_back(state1.GetNode("foo.h", 0));
50 deps.push_back(state1.GetNode("bar.h", 0));
51 log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps);
52
53 deps.clear();
54 deps.push_back(state1.GetNode("foo.h", 0));
55 deps.push_back(state1.GetNode("bar2.h", 0));
56 log1.RecordDeps(state1.GetNode("out2.o", 0), 2, deps);
57
58 DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0));
59 ASSERT_TRUE(log_deps);
60 ASSERT_EQ(1, log_deps->mtime);
61 ASSERT_EQ(2, log_deps->node_count);
62 ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
63 ASSERT_EQ("bar.h", log_deps->nodes[1]->path());
64 }
65
66 log1.Close();
67
68 State state2;
69 DepsLog log2;
70 EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
71 ASSERT_EQ("", err);
72
73 ASSERT_EQ(log1.nodes().size(), log2.nodes().size());
74 for (int i = 0; i < (int)log1.nodes().size(); ++i) {
75 Node* node1 = log1.nodes()[i];
76 Node* node2 = log2.nodes()[i];
77 ASSERT_EQ(i, node1->id());
78 ASSERT_EQ(node1->id(), node2->id());
79 }
80
81 // Spot-check the entries in log2.
82 DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out2.o", 0));
83 ASSERT_TRUE(log_deps);
84 ASSERT_EQ(2, log_deps->mtime);
85 ASSERT_EQ(2, log_deps->node_count);
86 ASSERT_EQ("foo.h", log_deps->nodes[0]->path());
87 ASSERT_EQ("bar2.h", log_deps->nodes[1]->path());
88 }
89
TEST_F(DepsLogTest,LotsOfDeps)90 TEST_F(DepsLogTest, LotsOfDeps) {
91 const int kNumDeps = 100000; // More than 64k.
92
93 State state1;
94 DepsLog log1;
95 string err;
96 EXPECT_TRUE(log1.OpenForWrite(kTestFilename, &err));
97 ASSERT_EQ("", err);
98
99 {
100 vector<Node*> deps;
101 for (int i = 0; i < kNumDeps; ++i) {
102 char buf[32];
103 sprintf(buf, "file%d.h", i);
104 deps.push_back(state1.GetNode(buf, 0));
105 }
106 log1.RecordDeps(state1.GetNode("out.o", 0), 1, deps);
107
108 DepsLog::Deps* log_deps = log1.GetDeps(state1.GetNode("out.o", 0));
109 ASSERT_EQ(kNumDeps, log_deps->node_count);
110 }
111
112 log1.Close();
113
114 State state2;
115 DepsLog log2;
116 EXPECT_TRUE(log2.Load(kTestFilename, &state2, &err));
117 ASSERT_EQ("", err);
118
119 DepsLog::Deps* log_deps = log2.GetDeps(state2.GetNode("out.o", 0));
120 ASSERT_EQ(kNumDeps, log_deps->node_count);
121 }
122
123 // Verify that adding the same deps twice doesn't grow the file.
TEST_F(DepsLogTest,DoubleEntry)124 TEST_F(DepsLogTest, DoubleEntry) {
125 // Write some deps to the file and grab its size.
126 int file_size;
127 {
128 State state;
129 DepsLog log;
130 string err;
131 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
132 ASSERT_EQ("", err);
133
134 vector<Node*> deps;
135 deps.push_back(state.GetNode("foo.h", 0));
136 deps.push_back(state.GetNode("bar.h", 0));
137 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
138 log.Close();
139
140 struct stat st;
141 ASSERT_EQ(0, stat(kTestFilename, &st));
142 file_size = (int)st.st_size;
143 ASSERT_GT(file_size, 0);
144 }
145
146 // Now reload the file, and read the same deps.
147 {
148 State state;
149 DepsLog log;
150 string err;
151 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
152
153 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
154 ASSERT_EQ("", err);
155
156 vector<Node*> deps;
157 deps.push_back(state.GetNode("foo.h", 0));
158 deps.push_back(state.GetNode("bar.h", 0));
159 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
160 log.Close();
161
162 struct stat st;
163 ASSERT_EQ(0, stat(kTestFilename, &st));
164 int file_size_2 = (int)st.st_size;
165 ASSERT_EQ(file_size, file_size_2);
166 }
167 }
168
169 // Verify that adding the new deps works and can be compacted away.
TEST_F(DepsLogTest,Recompact)170 TEST_F(DepsLogTest, Recompact) {
171 const char kManifest[] =
172 "rule cc\n"
173 " command = cc\n"
174 " deps = gcc\n"
175 "build out.o: cc\n"
176 "build other_out.o: cc\n";
177
178 // Write some deps to the file and grab its size.
179 int file_size;
180 {
181 State state;
182 ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
183 DepsLog log;
184 string err;
185 ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
186 ASSERT_EQ("", err);
187
188 vector<Node*> deps;
189 deps.push_back(state.GetNode("foo.h", 0));
190 deps.push_back(state.GetNode("bar.h", 0));
191 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
192
193 deps.clear();
194 deps.push_back(state.GetNode("foo.h", 0));
195 deps.push_back(state.GetNode("baz.h", 0));
196 log.RecordDeps(state.GetNode("other_out.o", 0), 1, deps);
197
198 log.Close();
199
200 struct stat st;
201 ASSERT_EQ(0, stat(kTestFilename, &st));
202 file_size = (int)st.st_size;
203 ASSERT_GT(file_size, 0);
204 }
205
206 // Now reload the file, and add slightly different deps.
207 int file_size_2;
208 {
209 State state;
210 ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
211 DepsLog log;
212 string err;
213 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
214
215 ASSERT_TRUE(log.OpenForWrite(kTestFilename, &err));
216 ASSERT_EQ("", err);
217
218 vector<Node*> deps;
219 deps.push_back(state.GetNode("foo.h", 0));
220 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
221 log.Close();
222
223 struct stat st;
224 ASSERT_EQ(0, stat(kTestFilename, &st));
225 file_size_2 = (int)st.st_size;
226 // The file should grow to record the new deps.
227 ASSERT_GT(file_size_2, file_size);
228 }
229
230 // Now reload the file, verify the new deps have replaced the old, then
231 // recompact.
232 int file_size_3;
233 {
234 State state;
235 ASSERT_NO_FATAL_FAILURE(AssertParse(&state, kManifest));
236 DepsLog log;
237 string err;
238 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
239
240 Node* out = state.GetNode("out.o", 0);
241 DepsLog::Deps* deps = log.GetDeps(out);
242 ASSERT_TRUE(deps);
243 ASSERT_EQ(1, deps->mtime);
244 ASSERT_EQ(1, deps->node_count);
245 ASSERT_EQ("foo.h", deps->nodes[0]->path());
246
247 Node* other_out = state.GetNode("other_out.o", 0);
248 deps = log.GetDeps(other_out);
249 ASSERT_TRUE(deps);
250 ASSERT_EQ(1, deps->mtime);
251 ASSERT_EQ(2, deps->node_count);
252 ASSERT_EQ("foo.h", deps->nodes[0]->path());
253 ASSERT_EQ("baz.h", deps->nodes[1]->path());
254
255 ASSERT_TRUE(log.Recompact(kTestFilename, &err));
256
257 // The in-memory deps graph should still be valid after recompaction.
258 deps = log.GetDeps(out);
259 ASSERT_TRUE(deps);
260 ASSERT_EQ(1, deps->mtime);
261 ASSERT_EQ(1, deps->node_count);
262 ASSERT_EQ("foo.h", deps->nodes[0]->path());
263 ASSERT_EQ(out, log.nodes()[out->id()]);
264
265 deps = log.GetDeps(other_out);
266 ASSERT_TRUE(deps);
267 ASSERT_EQ(1, deps->mtime);
268 ASSERT_EQ(2, deps->node_count);
269 ASSERT_EQ("foo.h", deps->nodes[0]->path());
270 ASSERT_EQ("baz.h", deps->nodes[1]->path());
271 ASSERT_EQ(other_out, log.nodes()[other_out->id()]);
272
273 // The file should have shrunk a bit for the smaller deps.
274 struct stat st;
275 ASSERT_EQ(0, stat(kTestFilename, &st));
276 file_size_3 = (int)st.st_size;
277 ASSERT_LT(file_size_3, file_size_2);
278 }
279
280 // Now reload the file and recompact with an empty manifest. The previous
281 // entries should be removed.
282 {
283 State state;
284 // Intentionally not parsing kManifest here.
285 DepsLog log;
286 string err;
287 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
288
289 Node* out = state.GetNode("out.o", 0);
290 DepsLog::Deps* deps = log.GetDeps(out);
291 ASSERT_TRUE(deps);
292 ASSERT_EQ(1, deps->mtime);
293 ASSERT_EQ(1, deps->node_count);
294 ASSERT_EQ("foo.h", deps->nodes[0]->path());
295
296 Node* other_out = state.GetNode("other_out.o", 0);
297 deps = log.GetDeps(other_out);
298 ASSERT_TRUE(deps);
299 ASSERT_EQ(1, deps->mtime);
300 ASSERT_EQ(2, deps->node_count);
301 ASSERT_EQ("foo.h", deps->nodes[0]->path());
302 ASSERT_EQ("baz.h", deps->nodes[1]->path());
303
304 ASSERT_TRUE(log.Recompact(kTestFilename, &err));
305
306 // The previous entries should have been removed.
307 deps = log.GetDeps(out);
308 ASSERT_FALSE(deps);
309
310 deps = log.GetDeps(other_out);
311 ASSERT_FALSE(deps);
312
313 // The .h files pulled in via deps should no longer have ids either.
314 ASSERT_EQ(-1, state.LookupNode("foo.h")->id());
315 ASSERT_EQ(-1, state.LookupNode("baz.h")->id());
316
317 // The file should have shrunk more.
318 struct stat st;
319 ASSERT_EQ(0, stat(kTestFilename, &st));
320 int file_size_4 = (int)st.st_size;
321 ASSERT_LT(file_size_4, file_size_3);
322 }
323 }
324
325 // Verify that invalid file headers cause a new build.
TEST_F(DepsLogTest,InvalidHeader)326 TEST_F(DepsLogTest, InvalidHeader) {
327 const char *kInvalidHeaders[] = {
328 "", // Empty file.
329 "# ninjad", // Truncated first line.
330 "# ninjadeps\n", // No version int.
331 "# ninjadeps\n\001\002", // Truncated version int.
332 "# ninjadeps\n\001\002\003\004" // Invalid version int.
333 };
334 for (size_t i = 0; i < sizeof(kInvalidHeaders) / sizeof(kInvalidHeaders[0]);
335 ++i) {
336 FILE* deps_log = fopen(kTestFilename, "wb");
337 ASSERT_TRUE(deps_log != NULL);
338 ASSERT_EQ(
339 strlen(kInvalidHeaders[i]),
340 fwrite(kInvalidHeaders[i], 1, strlen(kInvalidHeaders[i]), deps_log));
341 ASSERT_EQ(0 ,fclose(deps_log));
342
343 string err;
344 DepsLog log;
345 State state;
346 ASSERT_TRUE(log.Load(kTestFilename, &state, &err));
347 EXPECT_EQ("bad deps log signature or version; starting over", err);
348 }
349 }
350
351 // Simulate what happens when loading a truncated log file.
TEST_F(DepsLogTest,Truncated)352 TEST_F(DepsLogTest, Truncated) {
353 // Create a file with some entries.
354 {
355 State state;
356 DepsLog log;
357 string err;
358 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
359 ASSERT_EQ("", err);
360
361 vector<Node*> deps;
362 deps.push_back(state.GetNode("foo.h", 0));
363 deps.push_back(state.GetNode("bar.h", 0));
364 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
365
366 deps.clear();
367 deps.push_back(state.GetNode("foo.h", 0));
368 deps.push_back(state.GetNode("bar2.h", 0));
369 log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
370
371 log.Close();
372 }
373
374 // Get the file size.
375 struct stat st;
376 ASSERT_EQ(0, stat(kTestFilename, &st));
377
378 // Try reloading at truncated sizes.
379 // Track how many nodes/deps were found; they should decrease with
380 // smaller sizes.
381 int node_count = 5;
382 int deps_count = 2;
383 for (int size = (int)st.st_size; size > 0; --size) {
384 string err;
385 ASSERT_TRUE(Truncate(kTestFilename, size, &err));
386
387 State state;
388 DepsLog log;
389 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
390 if (!err.empty()) {
391 // At some point the log will be so short as to be unparseable.
392 break;
393 }
394
395 ASSERT_GE(node_count, (int)log.nodes().size());
396 node_count = log.nodes().size();
397
398 // Count how many non-NULL deps entries there are.
399 int new_deps_count = 0;
400 for (vector<DepsLog::Deps*>::const_iterator i = log.deps().begin();
401 i != log.deps().end(); ++i) {
402 if (*i)
403 ++new_deps_count;
404 }
405 ASSERT_GE(deps_count, new_deps_count);
406 deps_count = new_deps_count;
407 }
408 }
409
410 // Run the truncation-recovery logic.
TEST_F(DepsLogTest,TruncatedRecovery)411 TEST_F(DepsLogTest, TruncatedRecovery) {
412 // Create a file with some entries.
413 {
414 State state;
415 DepsLog log;
416 string err;
417 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
418 ASSERT_EQ("", err);
419
420 vector<Node*> deps;
421 deps.push_back(state.GetNode("foo.h", 0));
422 deps.push_back(state.GetNode("bar.h", 0));
423 log.RecordDeps(state.GetNode("out.o", 0), 1, deps);
424
425 deps.clear();
426 deps.push_back(state.GetNode("foo.h", 0));
427 deps.push_back(state.GetNode("bar2.h", 0));
428 log.RecordDeps(state.GetNode("out2.o", 0), 2, deps);
429
430 log.Close();
431 }
432
433 // Shorten the file, corrupting the last record.
434 {
435 struct stat st;
436 ASSERT_EQ(0, stat(kTestFilename, &st));
437 string err;
438 ASSERT_TRUE(Truncate(kTestFilename, st.st_size - 2, &err));
439 }
440
441 // Load the file again, add an entry.
442 {
443 State state;
444 DepsLog log;
445 string err;
446 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
447 ASSERT_EQ("premature end of file; recovering", err);
448 err.clear();
449
450 // The truncated entry should've been discarded.
451 EXPECT_EQ(NULL, log.GetDeps(state.GetNode("out2.o", 0)));
452
453 EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err));
454 ASSERT_EQ("", err);
455
456 // Add a new entry.
457 vector<Node*> deps;
458 deps.push_back(state.GetNode("foo.h", 0));
459 deps.push_back(state.GetNode("bar2.h", 0));
460 log.RecordDeps(state.GetNode("out2.o", 0), 3, deps);
461
462 log.Close();
463 }
464
465 // Load the file a third time to verify appending after a mangled
466 // entry doesn't break things.
467 {
468 State state;
469 DepsLog log;
470 string err;
471 EXPECT_TRUE(log.Load(kTestFilename, &state, &err));
472
473 // The truncated entry should exist.
474 DepsLog::Deps* deps = log.GetDeps(state.GetNode("out2.o", 0));
475 ASSERT_TRUE(deps);
476 }
477 }
478
479 } // anonymous namespace
480