1 // Copyright 2018 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "gn/metadata.h"
6
7 #include "gn/filesystem_utils.h"
8
9 const char kMetadata_Help[] =
10 R"(Metadata Collection
11
12 Metadata is information attached to targets throughout the dependency tree. GN
13 allows for the collection of this data into files written during the generation
14 step, enabling users to expose and aggregate this data based on the dependency
15 tree.
16
17 generated_file targets
18
19 Similar to the write_file() function, the generated_file target type
20 creates a file in the specified location with the specified content. The
21 primary difference between write_file() and this target type is that the
22 write_file function does the file write at parse time, while the
23 generated_file target type writes at target resolution time. See
24 "gn help generated_file" for more detail.
25
26 When written at target resolution time, generated_file enables GN to
27 collect and write aggregated metadata from dependents.
28
29 A generated_file target can declare either 'contents' to write statically
30 known contents to a file or 'data_keys' to aggregate metadata and write the
31 result to a file. It can also specify 'walk_keys' (to restrict the metadata
32 collection), 'output_conversion', and 'rebase'.
33
34
35 Collection and Aggregation
36
37 Targets can declare a 'metadata' variable containing a scope, and this
38 metadata may be collected and written out to a file specified by
39 generated_file aggregation targets. The 'metadata' scope must contain
40 only list values since the aggregation step collects a list of these values.
41
42 During the target resolution, generated_file targets will walk their
43 dependencies recursively, collecting metadata based on the specified
44 'data_keys'. 'data_keys' is specified as a list of strings, used by the walk
45 to identify which variables in dependencies' 'metadata' scopes to collect.
46
47 The walk begins with the listed dependencies of the 'generated_file' target.
48 The 'metadata' scope for each dependency is inspected for matching elements
49 of the 'generated_file' target's 'data_keys' list. If a match is found, the
50 data from the dependent's matching key list is appended to the aggregate walk
51 list. Note that this means that if more than one walk key is specified, the
52 data in all of them will be aggregated into one list. From there, the walk
53 will then recurse into the dependencies of each target it encounters,
54 collecting the specified metadata for each.
55
56 For example:
57
58 group("a") {
59 metadata = {
60 doom_melon = [ "enable" ]
61 my_files = [ "foo.cpp" ]
62 my_extra_files = [ "bar.cpp" ]
63 }
64
65 deps = [ ":b" ]
66 }
67
68 group("b") {
69 metadata = {
70 my_files = [ "baz.cpp" ]
71 }
72 }
73
74 generated_file("metadata") {
75 outputs = [ "$root_build_dir/my_files.json" ]
76 data_keys = [ "my_files", "my_extra_files" ]
77
78 deps = [ ":a" ]
79 }
80
81 The above will produce the following file data:
82
83 foo.cpp
84 bar.cpp
85 baz.cpp
86
87 The dependency walk can be limited by using the 'walk_keys'. This is a list of
88 labels that should be included in the walk. All labels specified here should
89 also be in one of the deps lists. These keys act as barriers, where the walk
90 will only recurse into the targets listed. An empty list in all specified
91 barriers will end that portion of the walk.
92
93 group("a") {
94 metadata = {
95 my_files = [ "foo.cpp" ]
96 my_files_barrier = [ ":b" ]
97 }
98
99 deps = [ ":b", ":c" ]
100 }
101
102 group("b") {
103 metadata = {
104 my_files = [ "bar.cpp" ]
105 }
106 }
107
108 group("c") {
109 metadata = {
110 my_files = [ "doom_melon.cpp" ]
111 }
112 }
113
114 generated_file("metadata") {
115 outputs = [ "$root_build_dir/my_files.json" ]
116 data_keys = [ "my_files" ]
117 walk_keys = [ "my_files_barrier" ]
118
119 deps = [ ":a" ]
120 }
121
122 The above will produce the following file data (note that `doom_melon.cpp` is
123 not included):
124
125 foo.cpp
126 bar.cpp
127
128 A common example of this sort of barrier is in builds that have host tools
129 built as part of the tree, but do not want the metadata from those host tools
130 to be collected with the target-side code.
131
132 Common Uses
133
134 Metadata can be used to collect information about the different targets in the
135 build, and so a common use is to provide post-build tooling with a set of data
136 necessary to do aggregation tasks. For example, if each test target specifies
137 the output location of its binary to run in a metadata field, that can be
138 collected into a single file listing the locations of all tests in the
139 dependency tree. A local build tool (or continuous integration infrastructure)
140 can then use that file to know which tests exist, and where, and run them
141 accordingly.
142
143 Another use is in image creation, where a post-build image tool needs to know
144 various pieces of information about the components it should include in order
145 to put together the correct image.
146 )";
147
WalkStep(const BuildSettings * settings,const std::vector<std::string> & keys_to_extract,const std::vector<std::string> & keys_to_walk,const SourceDir & rebase_dir,std::vector<Value> * next_walk_keys,std::vector<Value> * result,Err * err) const148 bool Metadata::WalkStep(const BuildSettings* settings,
149 const std::vector<std::string>& keys_to_extract,
150 const std::vector<std::string>& keys_to_walk,
151 const SourceDir& rebase_dir,
152 std::vector<Value>* next_walk_keys,
153 std::vector<Value>* result,
154 Err* err) const {
155 // If there's no metadata, there's nothing to find, so quick exit.
156 if (contents_.empty()) {
157 next_walk_keys->emplace_back(nullptr, "");
158 return true;
159 }
160
161 // Pull the data from each specified key.
162 for (const auto& key : keys_to_extract) {
163 auto iter = contents_.find(key);
164 if (iter == contents_.end())
165 continue;
166 assert(iter->second.type() == Value::LIST);
167
168 if (!rebase_dir.is_null()) {
169 for (const auto& val : iter->second.list_value()) {
170 std::pair<Value, bool> pair =
171 this->RebaseValue(settings, rebase_dir, val, err);
172 if (!pair.second)
173 return false;
174 result->push_back(pair.first);
175 }
176 } else {
177 result->insert(result->end(), iter->second.list_value().begin(),
178 iter->second.list_value().end());
179 }
180 }
181
182 // Get the targets to look at next. If no keys_to_walk are present, we push
183 // the empty string to the list so that the target knows to include its deps
184 // and data_deps. The values used here must be lists of strings.
185 bool found_walk_key = false;
186 for (const auto& key : keys_to_walk) {
187 auto iter = contents_.find(key);
188 if (iter != contents_.end()) {
189 found_walk_key = true;
190 assert(iter->second.type() == Value::LIST);
191 for (const auto& val : iter->second.list_value()) {
192 if (!val.VerifyTypeIs(Value::STRING, err))
193 return false;
194 next_walk_keys->emplace_back(val);
195 }
196 }
197 }
198
199 if (!found_walk_key)
200 next_walk_keys->emplace_back(nullptr, "");
201
202 return true;
203 }
204
RebaseValue(const BuildSettings * settings,const SourceDir & rebase_dir,const Value & value,Err * err) const205 std::pair<Value, bool> Metadata::RebaseValue(const BuildSettings* settings,
206 const SourceDir& rebase_dir,
207 const Value& value,
208 Err* err) const {
209 switch (value.type()) {
210 case Value::STRING:
211 return this->RebaseStringValue(settings, rebase_dir, value, err);
212 case Value::LIST:
213 return this->RebaseListValue(settings, rebase_dir, value, err);
214 case Value::SCOPE:
215 return this->RebaseScopeValue(settings, rebase_dir, value, err);
216 default:
217 return std::make_pair(value, true);
218 }
219 }
220
RebaseStringValue(const BuildSettings * settings,const SourceDir & rebase_dir,const Value & value,Err * err) const221 std::pair<Value, bool> Metadata::RebaseStringValue(
222 const BuildSettings* settings,
223 const SourceDir& rebase_dir,
224 const Value& value,
225 Err* err) const {
226 if (!value.VerifyTypeIs(Value::STRING, err))
227 return std::make_pair(value, false);
228 std::string filename = source_dir_.ResolveRelativeAs(
229 /*as_file = */ true, value, err, settings->root_path_utf8());
230 if (err->has_error())
231 return std::make_pair(value, false);
232 Value rebased_value(value.origin(), RebasePath(filename, rebase_dir,
233 settings->root_path_utf8()));
234 return std::make_pair(rebased_value, true);
235 }
236
RebaseListValue(const BuildSettings * settings,const SourceDir & rebase_dir,const Value & value,Err * err) const237 std::pair<Value, bool> Metadata::RebaseListValue(const BuildSettings* settings,
238 const SourceDir& rebase_dir,
239 const Value& value,
240 Err* err) const {
241 if (!value.VerifyTypeIs(Value::LIST, err))
242 return std::make_pair(value, false);
243
244 Value rebased_list_value(value.origin(), Value::LIST);
245 for (auto& val : value.list_value()) {
246 std::pair<Value, bool> pair = RebaseValue(settings, rebase_dir, val, err);
247 if (!pair.second)
248 return std::make_pair(value, false);
249 rebased_list_value.list_value().push_back(pair.first);
250 }
251 return std::make_pair(rebased_list_value, true);
252 }
253
RebaseScopeValue(const BuildSettings * settings,const SourceDir & rebase_dir,const Value & value,Err * err) const254 std::pair<Value, bool> Metadata::RebaseScopeValue(const BuildSettings* settings,
255 const SourceDir& rebase_dir,
256 const Value& value,
257 Err* err) const {
258 if (!value.VerifyTypeIs(Value::SCOPE, err))
259 return std::make_pair(value, false);
260 Value rebased_scope_value(value);
261 Scope::KeyValueMap scope_values;
262 value.scope_value()->GetCurrentScopeValues(&scope_values);
263 for (auto& value_pair : scope_values) {
264 std::pair<Value, bool> pair =
265 RebaseValue(settings, rebase_dir, value_pair.second, err);
266 if (!pair.second)
267 return std::make_pair(value, false);
268
269 rebased_scope_value.scope_value()->SetValue(value_pair.first, pair.first,
270 value.origin());
271 }
272 return std::make_pair(rebased_scope_value, true);
273 }
274