• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "src/trace_processor/util/descriptors.h"
18 #include "perfetto/ext/base/string_view.h"
19 #include "perfetto/protozero/field.h"
20 #include "perfetto/protozero/scattered_heap_buffer.h"
21 #include "protos/perfetto/common/descriptor.pbzero.h"
22 #include "protos/perfetto/trace_processor/trace_processor.pbzero.h"
23 #include "src/trace_processor/util/status_macros.h"
24 
25 namespace perfetto {
26 namespace trace_processor {
27 
CreateFieldFromDecoder(const protos::pbzero::FieldDescriptorProto::Decoder & f_decoder,bool is_extension)28 FieldDescriptor CreateFieldFromDecoder(
29     const protos::pbzero::FieldDescriptorProto::Decoder& f_decoder,
30     bool is_extension) {
31   using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
32   std::string type_name =
33       f_decoder.has_type_name()
34           ? base::StringView(f_decoder.type_name()).ToStdString()
35           : "";
36   // TODO(lalitm): add support for enums here.
37   uint32_t type =
38       f_decoder.has_type()
39           ? static_cast<uint32_t>(f_decoder.type())
40           : static_cast<uint32_t>(FieldDescriptorProto::TYPE_MESSAGE);
41   return FieldDescriptor(
42       base::StringView(f_decoder.name()).ToStdString(),
43       static_cast<uint32_t>(f_decoder.number()), type, std::move(type_name),
44       f_decoder.label() == FieldDescriptorProto::LABEL_REPEATED, is_extension);
45 }
46 
ResolveShortType(const std::string & parent_path,const std::string & short_type)47 base::Optional<uint32_t> DescriptorPool::ResolveShortType(
48     const std::string& parent_path,
49     const std::string& short_type) {
50   PERFETTO_DCHECK(!short_type.empty());
51 
52   std::string search_path = short_type[0] == '.'
53                                 ? parent_path + short_type
54                                 : parent_path + '.' + short_type;
55   auto opt_idx = FindDescriptorIdx(search_path);
56   if (opt_idx)
57     return opt_idx;
58 
59   if (parent_path.empty())
60     return base::nullopt;
61 
62   auto parent_dot_idx = parent_path.rfind('.');
63   auto parent_substr = parent_dot_idx == std::string::npos
64                            ? ""
65                            : parent_path.substr(0, parent_dot_idx);
66   return ResolveShortType(parent_substr, short_type);
67 }
68 
AddExtensionField(const std::string & package_name,protozero::ConstBytes field_desc_proto)69 util::Status DescriptorPool::AddExtensionField(
70     const std::string& package_name,
71     protozero::ConstBytes field_desc_proto) {
72   using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
73   FieldDescriptorProto::Decoder f_decoder(field_desc_proto);
74   auto field = CreateFieldFromDecoder(f_decoder, true);
75 
76   auto extendee_name = base::StringView(f_decoder.extendee()).ToStdString();
77   if (extendee_name.empty()) {
78     return util::ErrStatus("Extendee name is empty");
79   }
80 
81   if (extendee_name[0] != '.') {
82     // Only prepend if the extendee is not fully qualified
83     extendee_name = package_name + "." + extendee_name;
84   }
85   auto extendee = FindDescriptorIdx(extendee_name);
86   if (!extendee.has_value()) {
87     return util::ErrStatus("Extendee does not exist %s", extendee_name.c_str());
88   }
89   descriptors_[extendee.value()].AddField(field);
90   return util::OkStatus();
91 }
92 
AddNestedProtoDescriptors(const std::string & file_name,const std::string & package_name,base::Optional<uint32_t> parent_idx,protozero::ConstBytes descriptor_proto,std::vector<ExtensionInfo> * extensions,bool merge_existing_messages)93 util::Status DescriptorPool::AddNestedProtoDescriptors(
94     const std::string& file_name,
95     const std::string& package_name,
96     base::Optional<uint32_t> parent_idx,
97     protozero::ConstBytes descriptor_proto,
98     std::vector<ExtensionInfo>* extensions,
99     bool merge_existing_messages) {
100   protos::pbzero::DescriptorProto::Decoder decoder(descriptor_proto);
101 
102   auto parent_name =
103       parent_idx ? descriptors_[*parent_idx].full_name() : package_name;
104   auto full_name =
105       parent_name + "." + base::StringView(decoder.name()).ToStdString();
106 
107   auto prev_idx = FindDescriptorIdx(full_name);
108   if (prev_idx.has_value() && !merge_existing_messages) {
109     const auto& existing_descriptor = descriptors_[*prev_idx];
110     return util::ErrStatus("%s: %s was already defined in file %s",
111                            file_name.c_str(), full_name.c_str(),
112                            existing_descriptor.file_name().c_str());
113   }
114   if (!prev_idx.has_value()) {
115     prev_idx = static_cast<unsigned int>(descriptors_.size());
116     ProtoDescriptor proto_descriptor(file_name, package_name, full_name,
117                                      ProtoDescriptor::Type::kMessage,
118                                      parent_idx);
119     descriptors_.emplace_back(std::move(proto_descriptor));
120   }
121   ProtoDescriptor& proto_descriptor = descriptors_[*prev_idx];
122   if (proto_descriptor.type() != ProtoDescriptor::Type::kMessage) {
123     return util::ErrStatus("%s was enum, redefined as message",
124                            full_name.c_str());
125   }
126 
127   using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
128   for (auto it = decoder.field(); it; ++it) {
129     FieldDescriptorProto::Decoder f_decoder(*it);
130     auto field = CreateFieldFromDecoder(f_decoder, /*is_extension=*/false);
131     auto existing_field = proto_descriptor.FindFieldByTag(field.number());
132     if (!existing_field) {
133       proto_descriptor.AddField(std::move(field));
134     } else {
135       if (field.type() != existing_field->type()) {
136         return util::ErrStatus("Field %s is re-introduced with different type",
137                                field.name().c_str());
138       }
139       if (field.type() == FieldDescriptorProto::TYPE_MESSAGE &&
140           field.resolved_type_name() != existing_field->resolved_type_name()) {
141         return util::ErrStatus(
142             "Field %s is re-introduced with different type %s (was %s)",
143             field.name().c_str(), field.resolved_type_name().c_str(),
144             existing_field->resolved_type_name().c_str());
145       }
146     }
147   }
148 
149   auto idx = static_cast<uint32_t>(descriptors_.size()) - 1;
150   for (auto it = decoder.enum_type(); it; ++it) {
151     AddEnumProtoDescriptors(file_name, package_name, idx, *it,
152                             merge_existing_messages);
153   }
154   for (auto it = decoder.nested_type(); it; ++it) {
155     RETURN_IF_ERROR(AddNestedProtoDescriptors(file_name, package_name, idx, *it,
156                                               extensions,
157                                               merge_existing_messages));
158   }
159   for (auto ext_it = decoder.extension(); ext_it; ++ext_it) {
160     extensions->emplace_back(package_name, *ext_it);
161   }
162   return util::OkStatus();
163 }
164 
AddEnumProtoDescriptors(const std::string & file_name,const std::string & package_name,base::Optional<uint32_t> parent_idx,protozero::ConstBytes descriptor_proto,bool merge_existing_messages)165 util::Status DescriptorPool::AddEnumProtoDescriptors(
166     const std::string& file_name,
167     const std::string& package_name,
168     base::Optional<uint32_t> parent_idx,
169     protozero::ConstBytes descriptor_proto,
170     bool merge_existing_messages) {
171   protos::pbzero::EnumDescriptorProto::Decoder decoder(descriptor_proto);
172 
173   auto parent_name =
174       parent_idx ? descriptors_[*parent_idx].full_name() : package_name;
175   auto full_name =
176       parent_name + "." + base::StringView(decoder.name()).ToStdString();
177 
178   auto prev_idx = FindDescriptorIdx(full_name);
179   if (prev_idx.has_value() && !merge_existing_messages) {
180     const auto& existing_descriptor = descriptors_[*prev_idx];
181     return util::ErrStatus("%s: %s was already defined in file %s",
182                            file_name.c_str(), full_name.c_str(),
183                            existing_descriptor.file_name().c_str());
184   }
185   if (!prev_idx.has_value()) {
186     prev_idx = static_cast<unsigned int>(descriptors_.size());
187     ProtoDescriptor proto_descriptor(file_name, package_name, full_name,
188                                      ProtoDescriptor::Type::kEnum,
189                                      base::nullopt);
190     descriptors_.emplace_back(std::move(proto_descriptor));
191   }
192   ProtoDescriptor& proto_descriptor = descriptors_[*prev_idx];
193   if (proto_descriptor.type() != ProtoDescriptor::Type::kEnum) {
194     return util::ErrStatus("%s was message, redefined as enum",
195                            full_name.c_str());
196   }
197 
198   for (auto it = decoder.value(); it; ++it) {
199     protos::pbzero::EnumValueDescriptorProto::Decoder enum_value(it->data(),
200                                                                  it->size());
201     proto_descriptor.AddEnumValue(enum_value.number(),
202                                   enum_value.name().ToStdString());
203   }
204 
205   return util::OkStatus();
206 }
207 
AddFromFileDescriptorSet(const uint8_t * file_descriptor_set_proto,size_t size,bool merge_existing_messages)208 util::Status DescriptorPool::AddFromFileDescriptorSet(
209     const uint8_t* file_descriptor_set_proto,
210     size_t size,
211     bool merge_existing_messages) {
212   // First pass: extract all the message descriptors from the file and add them
213   // to the pool.
214   protos::pbzero::FileDescriptorSet::Decoder proto(file_descriptor_set_proto,
215                                                    size);
216   std::vector<ExtensionInfo> extensions;
217   for (auto it = proto.file(); it; ++it) {
218     protos::pbzero::FileDescriptorProto::Decoder file(*it);
219     std::string file_name = file.name().ToStdString();
220     if (processed_files_.find(file_name) != processed_files_.end()) {
221       // This file has been loaded once already. Skip.
222       continue;
223     }
224     processed_files_.insert(file_name);
225     std::string package = "." + base::StringView(file.package()).ToStdString();
226     for (auto message_it = file.message_type(); message_it; ++message_it) {
227       RETURN_IF_ERROR(AddNestedProtoDescriptors(
228           file_name, package, base::nullopt, *message_it, &extensions,
229           merge_existing_messages));
230     }
231     for (auto enum_it = file.enum_type(); enum_it; ++enum_it) {
232       AddEnumProtoDescriptors(file_name, package, base::nullopt, *enum_it,
233                               merge_existing_messages);
234     }
235     for (auto ext_it = file.extension(); ext_it; ++ext_it) {
236       extensions.emplace_back(package, *ext_it);
237     }
238   }
239 
240   // Second pass: Add extension fields to the real protos.
241   for (const auto& extension : extensions) {
242     auto status = AddExtensionField(extension.first, extension.second);
243     if (!status.ok())
244       return status;
245   }
246 
247   // Third pass: resolve the types of all the fields to the correct indiices.
248   using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
249   for (auto& descriptor : descriptors_) {
250     for (auto& entry : *descriptor.mutable_fields()) {
251       auto& field = entry.second;
252       if (!field.resolved_type_name().empty())
253         continue;
254 
255       if (field.type() == FieldDescriptorProto::TYPE_MESSAGE ||
256           field.type() == FieldDescriptorProto::TYPE_ENUM) {
257         auto opt_desc =
258             ResolveShortType(descriptor.full_name(), field.raw_type_name());
259         if (!opt_desc.has_value()) {
260           return util::ErrStatus(
261               "Unable to find short type %s in field inside message %s",
262               field.raw_type_name().c_str(), descriptor.full_name().c_str());
263         }
264         field.set_resolved_type_name(
265             descriptors_[opt_desc.value()].full_name());
266       }
267     }
268   }
269   return util::OkStatus();
270 }
271 
FindDescriptorIdx(const std::string & full_name) const272 base::Optional<uint32_t> DescriptorPool::FindDescriptorIdx(
273     const std::string& full_name) const {
274   auto it = std::find_if(descriptors_.begin(), descriptors_.end(),
275                          [full_name](const ProtoDescriptor& desc) {
276                            return desc.full_name() == full_name;
277                          });
278   auto idx = static_cast<uint32_t>(std::distance(descriptors_.begin(), it));
279   return idx < descriptors_.size() ? base::Optional<uint32_t>(idx)
280                                    : base::nullopt;
281 }
282 
SerializeAsDescriptorSet()283 std::vector<uint8_t> DescriptorPool::SerializeAsDescriptorSet() {
284   protozero::HeapBuffered<protos::pbzero::DescriptorSet> descs;
285   for (auto& desc : descriptors()) {
286     protos::pbzero::DescriptorProto* proto_descriptor =
287         descs->add_descriptors();
288     proto_descriptor->set_name(desc.full_name());
289     for (auto& entry : desc.fields()) {
290       auto& field = entry.second;
291       protos::pbzero::FieldDescriptorProto* field_descriptor =
292           proto_descriptor->add_field();
293       field_descriptor->set_name(field.name());
294       field_descriptor->set_number(static_cast<int32_t>(field.number()));
295       // We do not support required fields. They will show up as optional
296       // after serialization.
297       field_descriptor->set_label(
298           field.is_repeated()
299               ? protos::pbzero::FieldDescriptorProto::LABEL_REPEATED
300               : protos::pbzero::FieldDescriptorProto::LABEL_OPTIONAL);
301       field_descriptor->set_type_name(field.resolved_type_name());
302       field_descriptor->set_type(
303           static_cast<protos::pbzero::FieldDescriptorProto_Type>(field.type()));
304     }
305   }
306   return descs.SerializeAsArray();
307 }
308 
ProtoDescriptor(std::string file_name,std::string package_name,std::string full_name,Type type,base::Optional<uint32_t> parent_id)309 ProtoDescriptor::ProtoDescriptor(std::string file_name,
310                                  std::string package_name,
311                                  std::string full_name,
312                                  Type type,
313                                  base::Optional<uint32_t> parent_id)
314     : file_name_(std::move(file_name)),
315       package_name_(std::move(package_name)),
316       full_name_(std::move(full_name)),
317       type_(type),
318       parent_id_(parent_id) {}
319 
FieldDescriptor(std::string name,uint32_t number,uint32_t type,std::string raw_type_name,bool is_repeated,bool is_extension)320 FieldDescriptor::FieldDescriptor(std::string name,
321                                  uint32_t number,
322                                  uint32_t type,
323                                  std::string raw_type_name,
324                                  bool is_repeated,
325                                  bool is_extension)
326     : name_(std::move(name)),
327       number_(number),
328       type_(type),
329       raw_type_name_(std::move(raw_type_name)),
330       is_repeated_(is_repeated),
331       is_extension_(is_extension) {}
332 
333 }  // namespace trace_processor
334 }  // namespace perfetto
335