1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/util/descriptors.h"
18 #include "perfetto/ext/base/string_view.h"
19 #include "perfetto/protozero/field.h"
20 #include "perfetto/protozero/scattered_heap_buffer.h"
21 #include "protos/perfetto/common/descriptor.pbzero.h"
22 #include "protos/perfetto/trace_processor/trace_processor.pbzero.h"
23 #include "src/trace_processor/util/status_macros.h"
24
25 namespace perfetto {
26 namespace trace_processor {
27
CreateFieldFromDecoder(const protos::pbzero::FieldDescriptorProto::Decoder & f_decoder,bool is_extension)28 FieldDescriptor CreateFieldFromDecoder(
29 const protos::pbzero::FieldDescriptorProto::Decoder& f_decoder,
30 bool is_extension) {
31 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
32 std::string type_name =
33 f_decoder.has_type_name()
34 ? base::StringView(f_decoder.type_name()).ToStdString()
35 : "";
36 // TODO(lalitm): add support for enums here.
37 uint32_t type =
38 f_decoder.has_type()
39 ? static_cast<uint32_t>(f_decoder.type())
40 : static_cast<uint32_t>(FieldDescriptorProto::TYPE_MESSAGE);
41 return FieldDescriptor(
42 base::StringView(f_decoder.name()).ToStdString(),
43 static_cast<uint32_t>(f_decoder.number()), type, std::move(type_name),
44 f_decoder.label() == FieldDescriptorProto::LABEL_REPEATED, is_extension);
45 }
46
ResolveShortType(const std::string & parent_path,const std::string & short_type)47 base::Optional<uint32_t> DescriptorPool::ResolveShortType(
48 const std::string& parent_path,
49 const std::string& short_type) {
50 PERFETTO_DCHECK(!short_type.empty());
51
52 std::string search_path = short_type[0] == '.'
53 ? parent_path + short_type
54 : parent_path + '.' + short_type;
55 auto opt_idx = FindDescriptorIdx(search_path);
56 if (opt_idx)
57 return opt_idx;
58
59 if (parent_path.empty())
60 return base::nullopt;
61
62 auto parent_dot_idx = parent_path.rfind('.');
63 auto parent_substr = parent_dot_idx == std::string::npos
64 ? ""
65 : parent_path.substr(0, parent_dot_idx);
66 return ResolveShortType(parent_substr, short_type);
67 }
68
AddExtensionField(const std::string & package_name,protozero::ConstBytes field_desc_proto)69 util::Status DescriptorPool::AddExtensionField(
70 const std::string& package_name,
71 protozero::ConstBytes field_desc_proto) {
72 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
73 FieldDescriptorProto::Decoder f_decoder(field_desc_proto);
74 auto field = CreateFieldFromDecoder(f_decoder, true);
75
76 auto extendee_name = base::StringView(f_decoder.extendee()).ToStdString();
77 if (extendee_name.empty()) {
78 return util::ErrStatus("Extendee name is empty");
79 }
80
81 if (extendee_name[0] != '.') {
82 // Only prepend if the extendee is not fully qualified
83 extendee_name = package_name + "." + extendee_name;
84 }
85 auto extendee = FindDescriptorIdx(extendee_name);
86 if (!extendee.has_value()) {
87 return util::ErrStatus("Extendee does not exist %s", extendee_name.c_str());
88 }
89 descriptors_[extendee.value()].AddField(field);
90 return util::OkStatus();
91 }
92
AddNestedProtoDescriptors(const std::string & file_name,const std::string & package_name,base::Optional<uint32_t> parent_idx,protozero::ConstBytes descriptor_proto,std::vector<ExtensionInfo> * extensions,bool merge_existing_messages)93 util::Status DescriptorPool::AddNestedProtoDescriptors(
94 const std::string& file_name,
95 const std::string& package_name,
96 base::Optional<uint32_t> parent_idx,
97 protozero::ConstBytes descriptor_proto,
98 std::vector<ExtensionInfo>* extensions,
99 bool merge_existing_messages) {
100 protos::pbzero::DescriptorProto::Decoder decoder(descriptor_proto);
101
102 auto parent_name =
103 parent_idx ? descriptors_[*parent_idx].full_name() : package_name;
104 auto full_name =
105 parent_name + "." + base::StringView(decoder.name()).ToStdString();
106
107 auto prev_idx = FindDescriptorIdx(full_name);
108 if (prev_idx.has_value() && !merge_existing_messages) {
109 const auto& existing_descriptor = descriptors_[*prev_idx];
110 return util::ErrStatus("%s: %s was already defined in file %s",
111 file_name.c_str(), full_name.c_str(),
112 existing_descriptor.file_name().c_str());
113 }
114 if (!prev_idx.has_value()) {
115 prev_idx = static_cast<unsigned int>(descriptors_.size());
116 ProtoDescriptor proto_descriptor(file_name, package_name, full_name,
117 ProtoDescriptor::Type::kMessage,
118 parent_idx);
119 descriptors_.emplace_back(std::move(proto_descriptor));
120 }
121 ProtoDescriptor& proto_descriptor = descriptors_[*prev_idx];
122 if (proto_descriptor.type() != ProtoDescriptor::Type::kMessage) {
123 return util::ErrStatus("%s was enum, redefined as message",
124 full_name.c_str());
125 }
126
127 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
128 for (auto it = decoder.field(); it; ++it) {
129 FieldDescriptorProto::Decoder f_decoder(*it);
130 auto field = CreateFieldFromDecoder(f_decoder, /*is_extension=*/false);
131 auto existing_field = proto_descriptor.FindFieldByTag(field.number());
132 if (!existing_field) {
133 proto_descriptor.AddField(std::move(field));
134 } else {
135 if (field.type() != existing_field->type()) {
136 return util::ErrStatus("Field %s is re-introduced with different type",
137 field.name().c_str());
138 }
139 if (field.type() == FieldDescriptorProto::TYPE_MESSAGE &&
140 field.resolved_type_name() != existing_field->resolved_type_name()) {
141 return util::ErrStatus(
142 "Field %s is re-introduced with different type %s (was %s)",
143 field.name().c_str(), field.resolved_type_name().c_str(),
144 existing_field->resolved_type_name().c_str());
145 }
146 }
147 }
148
149 auto idx = static_cast<uint32_t>(descriptors_.size()) - 1;
150 for (auto it = decoder.enum_type(); it; ++it) {
151 AddEnumProtoDescriptors(file_name, package_name, idx, *it,
152 merge_existing_messages);
153 }
154 for (auto it = decoder.nested_type(); it; ++it) {
155 RETURN_IF_ERROR(AddNestedProtoDescriptors(file_name, package_name, idx, *it,
156 extensions,
157 merge_existing_messages));
158 }
159 for (auto ext_it = decoder.extension(); ext_it; ++ext_it) {
160 extensions->emplace_back(package_name, *ext_it);
161 }
162 return util::OkStatus();
163 }
164
AddEnumProtoDescriptors(const std::string & file_name,const std::string & package_name,base::Optional<uint32_t> parent_idx,protozero::ConstBytes descriptor_proto,bool merge_existing_messages)165 util::Status DescriptorPool::AddEnumProtoDescriptors(
166 const std::string& file_name,
167 const std::string& package_name,
168 base::Optional<uint32_t> parent_idx,
169 protozero::ConstBytes descriptor_proto,
170 bool merge_existing_messages) {
171 protos::pbzero::EnumDescriptorProto::Decoder decoder(descriptor_proto);
172
173 auto parent_name =
174 parent_idx ? descriptors_[*parent_idx].full_name() : package_name;
175 auto full_name =
176 parent_name + "." + base::StringView(decoder.name()).ToStdString();
177
178 auto prev_idx = FindDescriptorIdx(full_name);
179 if (prev_idx.has_value() && !merge_existing_messages) {
180 const auto& existing_descriptor = descriptors_[*prev_idx];
181 return util::ErrStatus("%s: %s was already defined in file %s",
182 file_name.c_str(), full_name.c_str(),
183 existing_descriptor.file_name().c_str());
184 }
185 if (!prev_idx.has_value()) {
186 prev_idx = static_cast<unsigned int>(descriptors_.size());
187 ProtoDescriptor proto_descriptor(file_name, package_name, full_name,
188 ProtoDescriptor::Type::kEnum,
189 base::nullopt);
190 descriptors_.emplace_back(std::move(proto_descriptor));
191 }
192 ProtoDescriptor& proto_descriptor = descriptors_[*prev_idx];
193 if (proto_descriptor.type() != ProtoDescriptor::Type::kEnum) {
194 return util::ErrStatus("%s was message, redefined as enum",
195 full_name.c_str());
196 }
197
198 for (auto it = decoder.value(); it; ++it) {
199 protos::pbzero::EnumValueDescriptorProto::Decoder enum_value(it->data(),
200 it->size());
201 proto_descriptor.AddEnumValue(enum_value.number(),
202 enum_value.name().ToStdString());
203 }
204
205 return util::OkStatus();
206 }
207
AddFromFileDescriptorSet(const uint8_t * file_descriptor_set_proto,size_t size,bool merge_existing_messages)208 util::Status DescriptorPool::AddFromFileDescriptorSet(
209 const uint8_t* file_descriptor_set_proto,
210 size_t size,
211 bool merge_existing_messages) {
212 // First pass: extract all the message descriptors from the file and add them
213 // to the pool.
214 protos::pbzero::FileDescriptorSet::Decoder proto(file_descriptor_set_proto,
215 size);
216 std::vector<ExtensionInfo> extensions;
217 for (auto it = proto.file(); it; ++it) {
218 protos::pbzero::FileDescriptorProto::Decoder file(*it);
219 std::string file_name = file.name().ToStdString();
220 if (processed_files_.find(file_name) != processed_files_.end()) {
221 // This file has been loaded once already. Skip.
222 continue;
223 }
224 processed_files_.insert(file_name);
225 std::string package = "." + base::StringView(file.package()).ToStdString();
226 for (auto message_it = file.message_type(); message_it; ++message_it) {
227 RETURN_IF_ERROR(AddNestedProtoDescriptors(
228 file_name, package, base::nullopt, *message_it, &extensions,
229 merge_existing_messages));
230 }
231 for (auto enum_it = file.enum_type(); enum_it; ++enum_it) {
232 AddEnumProtoDescriptors(file_name, package, base::nullopt, *enum_it,
233 merge_existing_messages);
234 }
235 for (auto ext_it = file.extension(); ext_it; ++ext_it) {
236 extensions.emplace_back(package, *ext_it);
237 }
238 }
239
240 // Second pass: Add extension fields to the real protos.
241 for (const auto& extension : extensions) {
242 auto status = AddExtensionField(extension.first, extension.second);
243 if (!status.ok())
244 return status;
245 }
246
247 // Third pass: resolve the types of all the fields to the correct indiices.
248 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
249 for (auto& descriptor : descriptors_) {
250 for (auto& entry : *descriptor.mutable_fields()) {
251 auto& field = entry.second;
252 if (!field.resolved_type_name().empty())
253 continue;
254
255 if (field.type() == FieldDescriptorProto::TYPE_MESSAGE ||
256 field.type() == FieldDescriptorProto::TYPE_ENUM) {
257 auto opt_desc =
258 ResolveShortType(descriptor.full_name(), field.raw_type_name());
259 if (!opt_desc.has_value()) {
260 return util::ErrStatus(
261 "Unable to find short type %s in field inside message %s",
262 field.raw_type_name().c_str(), descriptor.full_name().c_str());
263 }
264 field.set_resolved_type_name(
265 descriptors_[opt_desc.value()].full_name());
266 }
267 }
268 }
269 return util::OkStatus();
270 }
271
FindDescriptorIdx(const std::string & full_name) const272 base::Optional<uint32_t> DescriptorPool::FindDescriptorIdx(
273 const std::string& full_name) const {
274 auto it = std::find_if(descriptors_.begin(), descriptors_.end(),
275 [full_name](const ProtoDescriptor& desc) {
276 return desc.full_name() == full_name;
277 });
278 auto idx = static_cast<uint32_t>(std::distance(descriptors_.begin(), it));
279 return idx < descriptors_.size() ? base::Optional<uint32_t>(idx)
280 : base::nullopt;
281 }
282
SerializeAsDescriptorSet()283 std::vector<uint8_t> DescriptorPool::SerializeAsDescriptorSet() {
284 protozero::HeapBuffered<protos::pbzero::DescriptorSet> descs;
285 for (auto& desc : descriptors()) {
286 protos::pbzero::DescriptorProto* proto_descriptor =
287 descs->add_descriptors();
288 proto_descriptor->set_name(desc.full_name());
289 for (auto& entry : desc.fields()) {
290 auto& field = entry.second;
291 protos::pbzero::FieldDescriptorProto* field_descriptor =
292 proto_descriptor->add_field();
293 field_descriptor->set_name(field.name());
294 field_descriptor->set_number(static_cast<int32_t>(field.number()));
295 // We do not support required fields. They will show up as optional
296 // after serialization.
297 field_descriptor->set_label(
298 field.is_repeated()
299 ? protos::pbzero::FieldDescriptorProto::LABEL_REPEATED
300 : protos::pbzero::FieldDescriptorProto::LABEL_OPTIONAL);
301 field_descriptor->set_type_name(field.resolved_type_name());
302 field_descriptor->set_type(
303 static_cast<protos::pbzero::FieldDescriptorProto_Type>(field.type()));
304 }
305 }
306 return descs.SerializeAsArray();
307 }
308
ProtoDescriptor(std::string file_name,std::string package_name,std::string full_name,Type type,base::Optional<uint32_t> parent_id)309 ProtoDescriptor::ProtoDescriptor(std::string file_name,
310 std::string package_name,
311 std::string full_name,
312 Type type,
313 base::Optional<uint32_t> parent_id)
314 : file_name_(std::move(file_name)),
315 package_name_(std::move(package_name)),
316 full_name_(std::move(full_name)),
317 type_(type),
318 parent_id_(parent_id) {}
319
FieldDescriptor(std::string name,uint32_t number,uint32_t type,std::string raw_type_name,bool is_repeated,bool is_extension)320 FieldDescriptor::FieldDescriptor(std::string name,
321 uint32_t number,
322 uint32_t type,
323 std::string raw_type_name,
324 bool is_repeated,
325 bool is_extension)
326 : name_(std::move(name)),
327 number_(number),
328 type_(type),
329 raw_type_name_(std::move(raw_type_name)),
330 is_repeated_(is_repeated),
331 is_extension_(is_extension) {}
332
333 } // namespace trace_processor
334 } // namespace perfetto
335