1 // Copyright 2011 Google Inc. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "manifest_parser.h"
16
17 #include <stdio.h>
18 #include <stdlib.h>
19 #include <vector>
20
21 #include "graph.h"
22 #include "state.h"
23 #include "util.h"
24 #include "version.h"
25
ManifestParser(State * state,FileReader * file_reader,ManifestParserOptions options)26 ManifestParser::ManifestParser(State* state, FileReader* file_reader,
27 ManifestParserOptions options)
28 : Parser(state, file_reader),
29 options_(options), quiet_(false) {
30 env_ = &state->bindings_;
31 }
32
Parse(const string & filename,const string & input,string * err)33 bool ManifestParser::Parse(const string& filename, const string& input,
34 string* err) {
35 lexer_.Start(filename, input);
36
37 for (;;) {
38 Lexer::Token token = lexer_.ReadToken();
39 switch (token) {
40 case Lexer::POOL:
41 if (!ParsePool(err))
42 return false;
43 break;
44 case Lexer::BUILD:
45 if (!ParseEdge(err))
46 return false;
47 break;
48 case Lexer::RULE:
49 if (!ParseRule(err))
50 return false;
51 break;
52 case Lexer::DEFAULT:
53 if (!ParseDefault(err))
54 return false;
55 break;
56 case Lexer::IDENT: {
57 lexer_.UnreadToken();
58 string name;
59 EvalString let_value;
60 if (!ParseLet(&name, &let_value, err))
61 return false;
62 string value = let_value.Evaluate(env_);
63 // Check ninja_required_version immediately so we can exit
64 // before encountering any syntactic surprises.
65 if (name == "ninja_required_version")
66 CheckNinjaVersion(value);
67 env_->AddBinding(name, value);
68 break;
69 }
70 case Lexer::INCLUDE:
71 if (!ParseFileInclude(false, err))
72 return false;
73 break;
74 case Lexer::SUBNINJA:
75 if (!ParseFileInclude(true, err))
76 return false;
77 break;
78 case Lexer::ERROR: {
79 return lexer_.Error(lexer_.DescribeLastError(), err);
80 }
81 case Lexer::TEOF:
82 return true;
83 case Lexer::NEWLINE:
84 break;
85 default:
86 return lexer_.Error(string("unexpected ") + Lexer::TokenName(token),
87 err);
88 }
89 }
90 return false; // not reached
91 }
92
93
ParsePool(string * err)94 bool ManifestParser::ParsePool(string* err) {
95 string name;
96 if (!lexer_.ReadIdent(&name))
97 return lexer_.Error("expected pool name", err);
98
99 if (!ExpectToken(Lexer::NEWLINE, err))
100 return false;
101
102 if (state_->LookupPool(name) != NULL)
103 return lexer_.Error("duplicate pool '" + name + "'", err);
104
105 int depth = -1;
106
107 while (lexer_.PeekToken(Lexer::INDENT)) {
108 string key;
109 EvalString value;
110 if (!ParseLet(&key, &value, err))
111 return false;
112
113 if (key == "depth") {
114 string depth_string = value.Evaluate(env_);
115 depth = atol(depth_string.c_str());
116 if (depth < 0)
117 return lexer_.Error("invalid pool depth", err);
118 } else {
119 return lexer_.Error("unexpected variable '" + key + "'", err);
120 }
121 }
122
123 if (depth < 0)
124 return lexer_.Error("expected 'depth =' line", err);
125
126 state_->AddPool(new Pool(name, depth));
127 return true;
128 }
129
130
ParseRule(string * err)131 bool ManifestParser::ParseRule(string* err) {
132 string name;
133 if (!lexer_.ReadIdent(&name))
134 return lexer_.Error("expected rule name", err);
135
136 if (!ExpectToken(Lexer::NEWLINE, err))
137 return false;
138
139 if (env_->LookupRuleCurrentScope(name) != NULL)
140 return lexer_.Error("duplicate rule '" + name + "'", err);
141
142 Rule* rule = new Rule(name); // XXX scoped_ptr
143
144 while (lexer_.PeekToken(Lexer::INDENT)) {
145 string key;
146 EvalString value;
147 if (!ParseLet(&key, &value, err))
148 return false;
149
150 if (Rule::IsReservedBinding(key)) {
151 rule->AddBinding(key, value);
152 } else {
153 // Die on other keyvals for now; revisit if we want to add a
154 // scope here.
155 return lexer_.Error("unexpected variable '" + key + "'", err);
156 }
157 }
158
159 if (rule->bindings_["rspfile"].empty() !=
160 rule->bindings_["rspfile_content"].empty()) {
161 return lexer_.Error("rspfile and rspfile_content need to be "
162 "both specified", err);
163 }
164
165 if (rule->bindings_["command"].empty())
166 return lexer_.Error("expected 'command =' line", err);
167
168 env_->AddRule(rule);
169 return true;
170 }
171
ParseLet(string * key,EvalString * value,string * err)172 bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) {
173 if (!lexer_.ReadIdent(key))
174 return lexer_.Error("expected variable name", err);
175 if (!ExpectToken(Lexer::EQUALS, err))
176 return false;
177 if (!lexer_.ReadVarValue(value, err))
178 return false;
179 return true;
180 }
181
ParseDefault(string * err)182 bool ManifestParser::ParseDefault(string* err) {
183 EvalString eval;
184 if (!lexer_.ReadPath(&eval, err))
185 return false;
186 if (eval.empty())
187 return lexer_.Error("expected target name", err);
188
189 do {
190 string path = eval.Evaluate(env_);
191 string path_err;
192 uint64_t slash_bits; // Unused because this only does lookup.
193 if (!CanonicalizePath(&path, &slash_bits, &path_err))
194 return lexer_.Error(path_err, err);
195 if (!state_->AddDefault(path, &path_err))
196 return lexer_.Error(path_err, err);
197
198 eval.Clear();
199 if (!lexer_.ReadPath(&eval, err))
200 return false;
201 } while (!eval.empty());
202
203 if (!ExpectToken(Lexer::NEWLINE, err))
204 return false;
205
206 return true;
207 }
208
ParseEdge(string * err)209 bool ManifestParser::ParseEdge(string* err) {
210 vector<EvalString> ins, outs;
211
212 {
213 EvalString out;
214 if (!lexer_.ReadPath(&out, err))
215 return false;
216 while (!out.empty()) {
217 outs.push_back(out);
218
219 out.Clear();
220 if (!lexer_.ReadPath(&out, err))
221 return false;
222 }
223 }
224
225 // Add all implicit outs, counting how many as we go.
226 int implicit_outs = 0;
227 if (lexer_.PeekToken(Lexer::PIPE)) {
228 for (;;) {
229 EvalString out;
230 if (!lexer_.ReadPath(&out, err))
231 return false;
232 if (out.empty())
233 break;
234 outs.push_back(out);
235 ++implicit_outs;
236 }
237 }
238
239 if (outs.empty())
240 return lexer_.Error("expected path", err);
241
242 if (!ExpectToken(Lexer::COLON, err))
243 return false;
244
245 string rule_name;
246 if (!lexer_.ReadIdent(&rule_name))
247 return lexer_.Error("expected build command name", err);
248
249 const Rule* rule = env_->LookupRule(rule_name);
250 if (!rule)
251 return lexer_.Error("unknown build rule '" + rule_name + "'", err);
252
253 for (;;) {
254 // XXX should we require one path here?
255 EvalString in;
256 if (!lexer_.ReadPath(&in, err))
257 return false;
258 if (in.empty())
259 break;
260 ins.push_back(in);
261 }
262
263 // Add all implicit deps, counting how many as we go.
264 int implicit = 0;
265 if (lexer_.PeekToken(Lexer::PIPE)) {
266 for (;;) {
267 EvalString in;
268 if (!lexer_.ReadPath(&in, err))
269 return false;
270 if (in.empty())
271 break;
272 ins.push_back(in);
273 ++implicit;
274 }
275 }
276
277 // Add all order-only deps, counting how many as we go.
278 int order_only = 0;
279 if (lexer_.PeekToken(Lexer::PIPE2)) {
280 for (;;) {
281 EvalString in;
282 if (!lexer_.ReadPath(&in, err))
283 return false;
284 if (in.empty())
285 break;
286 ins.push_back(in);
287 ++order_only;
288 }
289 }
290
291 if (!ExpectToken(Lexer::NEWLINE, err))
292 return false;
293
294 // Bindings on edges are rare, so allocate per-edge envs only when needed.
295 bool has_indent_token = lexer_.PeekToken(Lexer::INDENT);
296 BindingEnv* env = has_indent_token ? new BindingEnv(env_) : env_;
297 while (has_indent_token) {
298 string key;
299 EvalString val;
300 if (!ParseLet(&key, &val, err))
301 return false;
302
303 env->AddBinding(key, val.Evaluate(env_));
304 has_indent_token = lexer_.PeekToken(Lexer::INDENT);
305 }
306
307 Edge* edge = state_->AddEdge(rule);
308 edge->env_ = env;
309
310 string pool_name = edge->GetBinding("pool");
311 if (!pool_name.empty()) {
312 Pool* pool = state_->LookupPool(pool_name);
313 if (pool == NULL)
314 return lexer_.Error("unknown pool name '" + pool_name + "'", err);
315 edge->pool_ = pool;
316 }
317
318 edge->outputs_.reserve(outs.size());
319 for (size_t i = 0, e = outs.size(); i != e; ++i) {
320 string path = outs[i].Evaluate(env);
321 string path_err;
322 uint64_t slash_bits;
323 if (!CanonicalizePath(&path, &slash_bits, &path_err))
324 return lexer_.Error(path_err, err);
325 if (!state_->AddOut(edge, path, slash_bits)) {
326 if (options_.dupe_edge_action_ == kDupeEdgeActionError) {
327 lexer_.Error("multiple rules generate " + path + " [-w dupbuild=err]",
328 err);
329 return false;
330 } else {
331 if (!quiet_) {
332 Warning("multiple rules generate %s. "
333 "builds involving this target will not be correct; "
334 "continuing anyway [-w dupbuild=warn]",
335 path.c_str());
336 }
337 if (e - i <= static_cast<size_t>(implicit_outs))
338 --implicit_outs;
339 }
340 }
341 }
342 if (edge->outputs_.empty()) {
343 // All outputs of the edge are already created by other edges. Don't add
344 // this edge. Do this check before input nodes are connected to the edge.
345 state_->edges_.pop_back();
346 delete edge;
347 return true;
348 }
349 edge->implicit_outs_ = implicit_outs;
350
351 edge->inputs_.reserve(ins.size());
352 for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) {
353 string path = i->Evaluate(env);
354 string path_err;
355 uint64_t slash_bits;
356 if (!CanonicalizePath(&path, &slash_bits, &path_err))
357 return lexer_.Error(path_err, err);
358 state_->AddIn(edge, path, slash_bits);
359 }
360 edge->implicit_deps_ = implicit;
361 edge->order_only_deps_ = order_only;
362
363 if (options_.phony_cycle_action_ == kPhonyCycleActionWarn &&
364 edge->maybe_phonycycle_diagnostic()) {
365 // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements
366 // that reference themselves. Ninja used to tolerate these in the
367 // build graph but that has since been fixed. Filter them out to
368 // support users of those old CMake versions.
369 Node* out = edge->outputs_[0];
370 vector<Node*>::iterator new_end =
371 remove(edge->inputs_.begin(), edge->inputs_.end(), out);
372 if (new_end != edge->inputs_.end()) {
373 edge->inputs_.erase(new_end, edge->inputs_.end());
374 if (!quiet_) {
375 Warning("phony target '%s' names itself as an input; "
376 "ignoring [-w phonycycle=warn]",
377 out->path().c_str());
378 }
379 }
380 }
381
382 // Lookup, validate, and save any dyndep binding. It will be used later
383 // to load generated dependency information dynamically, but it must
384 // be one of our manifest-specified inputs.
385 string dyndep = edge->GetUnescapedDyndep();
386 if (!dyndep.empty()) {
387 uint64_t slash_bits;
388 if (!CanonicalizePath(&dyndep, &slash_bits, err))
389 return false;
390 edge->dyndep_ = state_->GetNode(dyndep, slash_bits);
391 edge->dyndep_->set_dyndep_pending(true);
392 vector<Node*>::iterator dgi =
393 std::find(edge->inputs_.begin(), edge->inputs_.end(), edge->dyndep_);
394 if (dgi == edge->inputs_.end()) {
395 return lexer_.Error("dyndep '" + dyndep + "' is not an input", err);
396 }
397 }
398
399 return true;
400 }
401
ParseFileInclude(bool new_scope,string * err)402 bool ManifestParser::ParseFileInclude(bool new_scope, string* err) {
403 EvalString eval;
404 if (!lexer_.ReadPath(&eval, err))
405 return false;
406 string path = eval.Evaluate(env_);
407
408 ManifestParser subparser(state_, file_reader_, options_);
409 if (new_scope) {
410 subparser.env_ = new BindingEnv(env_);
411 } else {
412 subparser.env_ = env_;
413 }
414
415 if (!subparser.Load(path, err, &lexer_))
416 return false;
417
418 if (!ExpectToken(Lexer::NEWLINE, err))
419 return false;
420
421 return true;
422 }
423