1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "gn/metadata.h"
6
7 #include "gn/filesystem_utils.h"
8
9 const char kMetadata_Help[] =
10 R"(Metadata Collection
11
12 Metadata is information attached to targets throughout the dependency tree. GN
13 allows for the collection of this data into files written during the generation
14 step, enabling users to expose and aggregate this data based on the dependency
15 tree.
16
17 generated_file targets
18
19 Similar to the write_file() function, the generated_file target type
20 creates a file in the specified location with the specified content. The
21 primary difference between write_file() and this target type is that the
22 write_file function does the file write at parse time, while the
23 generated_file target type writes at target resolution time. See
24 "gn help generated_file" for more detail.
25
26 When written at target resolution time, generated_file enables GN to
27 collect and write aggregated metadata from dependents.
28
29 A generated_file target can declare either 'contents' to write statically
30 known contents to a file or 'data_keys' to aggregate metadata and write the
31 result to a file. It can also specify 'walk_keys' (to restrict the metadata
32 collection), 'output_conversion', and 'rebase'.
33
34
35 Collection and Aggregation
36
37 Targets can declare a 'metadata' variable containing a scope, and this
38 metadata may be collected and written out to a file specified by
39 generated_file aggregation targets. The 'metadata' scope must contain
40 only list values since the aggregation step collects a list of these values.
41
42 During the target resolution, generated_file targets will walk their
43 dependencies recursively, collecting metadata based on the specified
44 'data_keys'. 'data_keys' is specified as a list of strings, used by the walk
45 to identify which variables in dependencies' 'metadata' scopes to collect.
46
47 The walk begins with the listed dependencies of the 'generated_file' target.
48 The 'metadata' scope for each dependency is inspected for matching elements
49 of the 'generated_file' target's 'data_keys' list. If a match is found, the
50 data from the dependent's matching key list is appended to the aggregate walk
51 list. Note that this means that if more than one walk key is specified, the
52 data in all of them will be aggregated into one list. From there, the walk
53 will then recurse into the dependencies of each target it encounters,
54 collecting the specified metadata for each.
55
56 For example:
57
58 group("a") {
59 metadata = {
60 doom_melon = [ "enable" ]
61 my_files = [ "foo.cpp" ]
62 my_extra_files = [ "bar.cpp" ]
63 }
64
65 deps = [ ":b" ]
66 }
67
68 group("b") {
69 metadata = {
70 my_files = [ "baz.cpp" ]
71 }
72 }
73
74 generated_file("metadata") {
75 outputs = [ "$root_build_dir/my_files.json" ]
76 data_keys = [ "my_files", "my_extra_files" ]
77
78 deps = [ ":a" ]
79 }
80
81 The above will produce the following file data:
82
83 foo.cpp
84 bar.cpp
85 baz.cpp
86
87 The dependency walk can be limited by using the 'walk_keys'. This is a list of
88 labels that should be included in the walk. All labels specified here should
89 also be in one of the deps lists. These keys act as barriers, where the walk
90 will only recurse into the targets listed. An empty list in all specified
91 barriers will end that portion of the walk.
92
93 group("a") {
94 metadata = {
95 my_files = [ "foo.cpp" ]
96 my_files_barrier [ ":b" ]
97 }
98
99 deps = [ ":b", ":c" ]
100 }
101
102 group("b") {
103 metadata = {
104 my_files = [ "bar.cpp" ]
105 }
106 }
107
108 group("c") {
109 metadata = {
110 my_files = [ "doom_melon.cpp" ]
111 }
112 }
113
114 generated_file("metadata") {
115 outputs = [ "$root_build_dir/my_files.json" ]
116 data_keys = [ "my_files", "my_extra_files" ]
117
118 deps = [ ":a" ]
119 }
120
121 The above will produce the following file data (note that `doom_melon.cpp` is
122 not included):
123
124 foo.cpp
125 bar.cpp
126
127 A common example of this sort of barrier is in builds that have host tools
128 built as part of the tree, but do not want the metadata from those host tools
129 to be collected with the target-side code.
130
131 Common Uses
132
133 Metadata can be used to collect information about the different targets in the
134 build, and so a common use is to provide post-build tooling with a set of data
135 necessary to do aggregation tasks. For example, if each test target specifies
136 the output location of its binary to run in a metadata field, that can be
137 collected into a single file listing the locations of all tests in the
138 dependency tree. A local build tool (or continuous integration infrastructure)
139 can then use that file to know which tests exist, and where, and run them
140 accordingly.
141
142 Another use is in image creation, where a post-build image tool needs to know
143 various pieces of information about the components it should include in order
144 to put together the correct image.
145 )";
146
WalkStep(const BuildSettings * settings,const std::vector<std::string> & keys_to_extract,const std::vector<std::string> & keys_to_walk,const SourceDir & rebase_dir,std::vector<Value> * next_walk_keys,std::vector<Value> * result,Err * err) const147 bool Metadata::WalkStep(const BuildSettings* settings,
148 const std::vector<std::string>& keys_to_extract,
149 const std::vector<std::string>& keys_to_walk,
150 const SourceDir& rebase_dir,
151 std::vector<Value>* next_walk_keys,
152 std::vector<Value>* result,
153 Err* err) const {
154 // If there's no metadata, there's nothing to find, so quick exit.
155 if (contents_.empty()) {
156 next_walk_keys->emplace_back(nullptr, "");
157 return true;
158 }
159
160 // Pull the data from each specified key.
161 for (const auto& key : keys_to_extract) {
162 auto iter = contents_.find(key);
163 if (iter == contents_.end())
164 continue;
165 assert(iter->second.type() == Value::LIST);
166
167 if (!rebase_dir.is_null()) {
168 for (const auto& val : iter->second.list_value()) {
169 std::pair<Value, bool> pair =
170 this->RebaseValue(settings, rebase_dir, val, err);
171 if (!pair.second)
172 return false;
173 result->push_back(pair.first);
174 }
175 } else {
176 result->insert(result->end(), iter->second.list_value().begin(),
177 iter->second.list_value().end());
178 }
179 }
180
181 // Get the targets to look at next. If no keys_to_walk are present, we push
182 // the empty string to the list so that the target knows to include its deps
183 // and data_deps. The values used here must be lists of strings.
184 bool found_walk_key = false;
185 for (const auto& key : keys_to_walk) {
186 auto iter = contents_.find(key);
187 if (iter != contents_.end()) {
188 found_walk_key = true;
189 assert(iter->second.type() == Value::LIST);
190 for (const auto& val : iter->second.list_value()) {
191 if (!val.VerifyTypeIs(Value::STRING, err))
192 return false;
193 next_walk_keys->emplace_back(val);
194 }
195 }
196 }
197
198 if (!found_walk_key)
199 next_walk_keys->emplace_back(nullptr, "");
200
201 return true;
202 }
203
RebaseValue(const BuildSettings * settings,const SourceDir & rebase_dir,const Value & value,Err * err) const204 std::pair<Value, bool> Metadata::RebaseValue(const BuildSettings* settings,
205 const SourceDir& rebase_dir,
206 const Value& value,
207 Err* err) const {
208 switch (value.type()) {
209 case Value::STRING:
210 return this->RebaseStringValue(settings, rebase_dir, value, err);
211 case Value::LIST:
212 return this->RebaseListValue(settings, rebase_dir, value, err);
213 case Value::SCOPE:
214 return this->RebaseScopeValue(settings, rebase_dir, value, err);
215 default:
216 return std::make_pair(value, true);
217 }
218 }
219
RebaseStringValue(const BuildSettings * settings,const SourceDir & rebase_dir,const Value & value,Err * err) const220 std::pair<Value, bool> Metadata::RebaseStringValue(
221 const BuildSettings* settings,
222 const SourceDir& rebase_dir,
223 const Value& value,
224 Err* err) const {
225 if (!value.VerifyTypeIs(Value::STRING, err))
226 return std::make_pair(value, false);
227 std::string filename = source_dir_.ResolveRelativeAs(
228 /*as_file = */ true, value, err, settings->root_path_utf8());
229 if (err->has_error())
230 return std::make_pair(value, false);
231 Value rebased_value(value.origin(), RebasePath(filename, rebase_dir,
232 settings->root_path_utf8()));
233 return std::make_pair(rebased_value, true);
234 }
235
RebaseListValue(const BuildSettings * settings,const SourceDir & rebase_dir,const Value & value,Err * err) const236 std::pair<Value, bool> Metadata::RebaseListValue(const BuildSettings* settings,
237 const SourceDir& rebase_dir,
238 const Value& value,
239 Err* err) const {
240 if (!value.VerifyTypeIs(Value::LIST, err))
241 return std::make_pair(value, false);
242
243 Value rebased_list_value(value.origin(), Value::LIST);
244 for (auto& val : value.list_value()) {
245 std::pair<Value, bool> pair = RebaseValue(settings, rebase_dir, val, err);
246 if (!pair.second)
247 return std::make_pair(value, false);
248 rebased_list_value.list_value().push_back(pair.first);
249 }
250 return std::make_pair(rebased_list_value, true);
251 }
252
RebaseScopeValue(const BuildSettings * settings,const SourceDir & rebase_dir,const Value & value,Err * err) const253 std::pair<Value, bool> Metadata::RebaseScopeValue(const BuildSettings* settings,
254 const SourceDir& rebase_dir,
255 const Value& value,
256 Err* err) const {
257 if (!value.VerifyTypeIs(Value::SCOPE, err))
258 return std::make_pair(value, false);
259 Value rebased_scope_value(value);
260 Scope::KeyValueMap scope_values;
261 value.scope_value()->GetCurrentScopeValues(&scope_values);
262 for (auto& value_pair : scope_values) {
263 std::pair<Value, bool> pair =
264 RebaseValue(settings, rebase_dir, value_pair.second, err);
265 if (!pair.second)
266 return std::make_pair(value, false);
267
268 rebased_scope_value.scope_value()->SetValue(value_pair.first, pair.first,
269 value.origin());
270 }
271 return std::make_pair(rebased_scope_value, true);
272 }
273