1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/util/proto_to_args_parser.h"
18
19 #include <stdint.h>
20
21 #include "perfetto/base/status.h"
22 #include "perfetto/protozero/proto_decoder.h"
23 #include "perfetto/protozero/proto_utils.h"
24 #include "protos/perfetto/common/descriptor.pbzero.h"
25 #include "src/trace_processor/util/descriptors.h"
26 #include "src/trace_processor/util/status_macros.h"
27
28 namespace perfetto {
29 namespace trace_processor {
30 namespace util {
31
32 namespace {
33
34 template <protozero::proto_utils::ProtoWireType wire_type, typename cpp_type>
35 using PRFI = protozero::PackedRepeatedFieldIterator<wire_type, cpp_type>;
36
AppendProtoType(std::string & target,const std::string & value)37 void AppendProtoType(std::string& target, const std::string& value) {
38 if (!target.empty())
39 target += '.';
40 target += value;
41 }
42
43 } // namespace
44
45 ProtoToArgsParser::Key::Key() = default;
Key(const std::string & k)46 ProtoToArgsParser::Key::Key(const std::string& k) : flat_key(k), key(k) {}
Key(const std::string & fk,const std::string & k)47 ProtoToArgsParser::Key::Key(const std::string& fk, const std::string& k)
48 : flat_key(fk), key(k) {}
49 ProtoToArgsParser::Key::~Key() = default;
50
ScopedNestedKeyContext(Key & key)51 ProtoToArgsParser::ScopedNestedKeyContext::ScopedNestedKeyContext(Key& key)
52 : key_(key),
53 old_flat_key_length_(key.flat_key.length()),
54 old_key_length_(key.key.length()) {}
55
ScopedNestedKeyContext(ProtoToArgsParser::ScopedNestedKeyContext && other)56 ProtoToArgsParser::ScopedNestedKeyContext::ScopedNestedKeyContext(
57 ProtoToArgsParser::ScopedNestedKeyContext&& other)
58 : key_(other.key_),
59 old_flat_key_length_(other.old_flat_key_length_),
60 old_key_length_(other.old_key_length_) {
61 other.old_flat_key_length_ = std::nullopt;
62 other.old_key_length_ = std::nullopt;
63 }
64
~ScopedNestedKeyContext()65 ProtoToArgsParser::ScopedNestedKeyContext::~ScopedNestedKeyContext() {
66 RemoveFieldSuffix();
67 }
68
RemoveFieldSuffix()69 void ProtoToArgsParser::ScopedNestedKeyContext::RemoveFieldSuffix() {
70 if (old_flat_key_length_)
71 key_.flat_key.resize(old_flat_key_length_.value());
72 if (old_key_length_)
73 key_.key.resize(old_key_length_.value());
74 old_flat_key_length_ = std::nullopt;
75 old_key_length_ = std::nullopt;
76 }
77
78 ProtoToArgsParser::Delegate::~Delegate() = default;
79
ProtoToArgsParser(const DescriptorPool & pool)80 ProtoToArgsParser::ProtoToArgsParser(const DescriptorPool& pool) : pool_(pool) {
81 constexpr int kDefaultSize = 64;
82 key_prefix_.key.reserve(kDefaultSize);
83 key_prefix_.flat_key.reserve(kDefaultSize);
84 }
85
ParseMessage(const protozero::ConstBytes & cb,const std::string & type,const std::vector<uint16_t> * allowed_fields,Delegate & delegate,int * unknown_extensions)86 base::Status ProtoToArgsParser::ParseMessage(
87 const protozero::ConstBytes& cb,
88 const std::string& type,
89 const std::vector<uint16_t>* allowed_fields,
90 Delegate& delegate,
91 int* unknown_extensions) {
92 ScopedNestedKeyContext key_context(key_prefix_);
93 return ParseMessageInternal(key_context, cb, type, allowed_fields, delegate,
94 unknown_extensions);
95 }
96
ParseMessageInternal(ScopedNestedKeyContext & key_context,const protozero::ConstBytes & cb,const std::string & type,const std::vector<uint16_t> * allowed_fields,Delegate & delegate,int * unknown_extensions)97 base::Status ProtoToArgsParser::ParseMessageInternal(
98 ScopedNestedKeyContext& key_context,
99 const protozero::ConstBytes& cb,
100 const std::string& type,
101 const std::vector<uint16_t>* allowed_fields,
102 Delegate& delegate,
103 int* unknown_extensions) {
104 if (auto override_result =
105 MaybeApplyOverrideForType(type, key_context, cb, delegate)) {
106 return override_result.value();
107 }
108
109 auto idx = pool_.FindDescriptorIdx(type);
110 if (!idx) {
111 return base::Status("Failed to find proto descriptor");
112 }
113
114 auto& descriptor = pool_.descriptors()[*idx];
115
116 std::unordered_map<size_t, int> repeated_field_index;
117 bool empty_message = true;
118 protozero::ProtoDecoder decoder(cb);
119 for (protozero::Field f = decoder.ReadField(); f.valid();
120 f = decoder.ReadField()) {
121 empty_message = false;
122 auto field = descriptor.FindFieldByTag(f.id());
123 if (!field) {
124 if (unknown_extensions != nullptr) {
125 (*unknown_extensions)++;
126 }
127 // Unknown field, possibly an unknown extension.
128 continue;
129 }
130
131 // If allowlist is not provided, reflect all fields. Otherwise, check if the
132 // current field either an extension or is in allowlist.
133 bool is_allowed = field->is_extension() || !allowed_fields ||
134 std::find(allowed_fields->begin(), allowed_fields->end(),
135 f.id()) != allowed_fields->end();
136
137 if (!is_allowed) {
138 // Field is neither an extension, nor is allowed to be
139 // reflected.
140 continue;
141 }
142
143 // Packed fields need to be handled specially because
144 if (field->is_packed()) {
145 RETURN_IF_ERROR(ParsePackedField(*field, repeated_field_index, f,
146 delegate, unknown_extensions));
147 continue;
148 }
149
150 RETURN_IF_ERROR(ParseField(*field, repeated_field_index[f.id()], f,
151 delegate, unknown_extensions));
152 if (field->is_repeated()) {
153 repeated_field_index[f.id()]++;
154 }
155 }
156
157 if (empty_message) {
158 delegate.AddNull(key_prefix_);
159 }
160
161 return base::OkStatus();
162 }
163
ParseField(const FieldDescriptor & field_descriptor,int repeated_field_number,protozero::Field field,Delegate & delegate,int * unknown_extensions)164 base::Status ProtoToArgsParser::ParseField(
165 const FieldDescriptor& field_descriptor,
166 int repeated_field_number,
167 protozero::Field field,
168 Delegate& delegate,
169 int* unknown_extensions) {
170 std::string prefix_part = field_descriptor.name();
171 if (field_descriptor.is_repeated()) {
172 std::string number = std::to_string(repeated_field_number);
173 prefix_part.reserve(prefix_part.length() + number.length() + 2);
174 prefix_part.append("[");
175 prefix_part.append(number);
176 prefix_part.append("]");
177 }
178
179 // In the args table we build up message1.message2.field1 as the column
180 // name. This will append the ".field1" suffix to |key_prefix| and then
181 // remove it when it goes out of scope.
182 ScopedNestedKeyContext key_context(key_prefix_);
183 AppendProtoType(key_prefix_.flat_key, field_descriptor.name());
184 AppendProtoType(key_prefix_.key, prefix_part);
185
186 // If we have an override parser then use that instead and move onto the
187 // next loop.
188 if (std::optional<base::Status> status =
189 MaybeApplyOverrideForField(field, delegate)) {
190 return *status;
191 }
192
193 // If this is not a message we can just immediately add the column name and
194 // get the value out of |field|. However if it is a message we need to
195 // recurse into it.
196 if (field_descriptor.type() ==
197 protos::pbzero::FieldDescriptorProto::TYPE_MESSAGE) {
198 return ParseMessageInternal(key_context, field.as_bytes(),
199 field_descriptor.resolved_type_name(), nullptr,
200 delegate, unknown_extensions);
201 }
202 return ParseSimpleField(field_descriptor, field, delegate);
203 }
204
ParsePackedField(const FieldDescriptor & field_descriptor,std::unordered_map<size_t,int> & repeated_field_index,protozero::Field field,Delegate & delegate,int * unknown_extensions)205 base::Status ProtoToArgsParser::ParsePackedField(
206 const FieldDescriptor& field_descriptor,
207 std::unordered_map<size_t, int>& repeated_field_index,
208 protozero::Field field,
209 Delegate& delegate,
210 int* unknown_extensions) {
211 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
212 using PWT = protozero::proto_utils::ProtoWireType;
213
214 if (!field_descriptor.is_repeated()) {
215 return base::ErrStatus("Packed field %s must be repeated",
216 field_descriptor.name().c_str());
217 }
218 if (field.type() != PWT::kLengthDelimited) {
219 return base::ErrStatus(
220 "Packed field %s must have a length delimited wire type",
221 field_descriptor.name().c_str());
222 }
223
224 auto parse = [&](uint64_t new_value, PWT wire_type) {
225 protozero::Field f;
226 f.initialize(field.id(), static_cast<uint8_t>(wire_type), new_value, 0);
227 return ParseField(field_descriptor, repeated_field_index[field.id()]++, f,
228 delegate, unknown_extensions);
229 };
230
231 const uint8_t* data = field.as_bytes().data;
232 size_t size = field.as_bytes().size;
233 bool perr = false;
234 switch (field_descriptor.type()) {
235 case FieldDescriptorProto::TYPE_INT32:
236 case FieldDescriptorProto::TYPE_INT64:
237 case FieldDescriptorProto::TYPE_UINT32:
238 case FieldDescriptorProto::TYPE_UINT64:
239 case FieldDescriptorProto::TYPE_ENUM:
240 for (PRFI<PWT::kVarInt, uint64_t> it(data, size, &perr); it; ++it) {
241 parse(*it, PWT::kVarInt);
242 }
243 break;
244 case FieldDescriptorProto::TYPE_FIXED32:
245 case FieldDescriptorProto::TYPE_SFIXED32:
246 case FieldDescriptorProto::TYPE_FLOAT:
247 for (PRFI<PWT::kFixed32, uint32_t> it(data, size, &perr); it; ++it) {
248 parse(*it, PWT::kFixed32);
249 }
250 break;
251 case FieldDescriptorProto::TYPE_FIXED64:
252 case FieldDescriptorProto::TYPE_SFIXED64:
253 case FieldDescriptorProto::TYPE_DOUBLE:
254 for (PRFI<PWT::kFixed64, uint64_t> it(data, size, &perr); it; ++it) {
255 parse(*it, PWT::kFixed64);
256 }
257 break;
258 default:
259 return base::ErrStatus("Unsupported packed repeated field");
260 }
261 return base::OkStatus();
262 }
263
AddParsingOverrideForField(const std::string & field,ParsingOverrideForField func)264 void ProtoToArgsParser::AddParsingOverrideForField(
265 const std::string& field,
266 ParsingOverrideForField func) {
267 field_overrides_[field] = std::move(func);
268 }
269
AddParsingOverrideForType(const std::string & type,ParsingOverrideForType func)270 void ProtoToArgsParser::AddParsingOverrideForType(const std::string& type,
271 ParsingOverrideForType func) {
272 type_overrides_[type] = std::move(func);
273 }
274
MaybeApplyOverrideForField(const protozero::Field & field,Delegate & delegate)275 std::optional<base::Status> ProtoToArgsParser::MaybeApplyOverrideForField(
276 const protozero::Field& field,
277 Delegate& delegate) {
278 auto it = field_overrides_.find(key_prefix_.flat_key);
279 if (it == field_overrides_.end())
280 return std::nullopt;
281 return it->second(field, delegate);
282 }
283
MaybeApplyOverrideForType(const std::string & message_type,ScopedNestedKeyContext & key,const protozero::ConstBytes & data,Delegate & delegate)284 std::optional<base::Status> ProtoToArgsParser::MaybeApplyOverrideForType(
285 const std::string& message_type,
286 ScopedNestedKeyContext& key,
287 const protozero::ConstBytes& data,
288 Delegate& delegate) {
289 auto it = type_overrides_.find(message_type);
290 if (it == type_overrides_.end())
291 return std::nullopt;
292 return it->second(key, data, delegate);
293 }
294
ParseSimpleField(const FieldDescriptor & descriptor,const protozero::Field & field,Delegate & delegate)295 base::Status ProtoToArgsParser::ParseSimpleField(
296 const FieldDescriptor& descriptor,
297 const protozero::Field& field,
298 Delegate& delegate) {
299 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
300 switch (descriptor.type()) {
301 case FieldDescriptorProto::TYPE_INT32:
302 case FieldDescriptorProto::TYPE_SFIXED32:
303 delegate.AddInteger(key_prefix_, field.as_int32());
304 return base::OkStatus();
305 case FieldDescriptorProto::TYPE_SINT32:
306 delegate.AddInteger(key_prefix_, field.as_sint32());
307 return base::OkStatus();
308 case FieldDescriptorProto::TYPE_INT64:
309 case FieldDescriptorProto::TYPE_SFIXED64:
310 delegate.AddInteger(key_prefix_, field.as_int64());
311 return base::OkStatus();
312 case FieldDescriptorProto::TYPE_SINT64:
313 delegate.AddInteger(key_prefix_, field.as_sint64());
314 return base::OkStatus();
315 case FieldDescriptorProto::TYPE_UINT32:
316 case FieldDescriptorProto::TYPE_FIXED32:
317 delegate.AddUnsignedInteger(key_prefix_, field.as_uint32());
318 return base::OkStatus();
319 case FieldDescriptorProto::TYPE_UINT64:
320 case FieldDescriptorProto::TYPE_FIXED64:
321 delegate.AddUnsignedInteger(key_prefix_, field.as_uint64());
322 return base::OkStatus();
323 case FieldDescriptorProto::TYPE_BOOL:
324 delegate.AddBoolean(key_prefix_, field.as_bool());
325 return base::OkStatus();
326 case FieldDescriptorProto::TYPE_DOUBLE:
327 delegate.AddDouble(key_prefix_, field.as_double());
328 return base::OkStatus();
329 case FieldDescriptorProto::TYPE_FLOAT:
330 delegate.AddDouble(key_prefix_, static_cast<double>(field.as_float()));
331 return base::OkStatus();
332 case FieldDescriptorProto::TYPE_STRING:
333 delegate.AddString(key_prefix_, field.as_string());
334 return base::OkStatus();
335 case FieldDescriptorProto::TYPE_ENUM: {
336 auto opt_enum_descriptor_idx =
337 pool_.FindDescriptorIdx(descriptor.resolved_type_name());
338 if (!opt_enum_descriptor_idx) {
339 delegate.AddInteger(key_prefix_, field.as_int32());
340 return base::OkStatus();
341 }
342 auto opt_enum_string =
343 pool_.descriptors()[*opt_enum_descriptor_idx].FindEnumString(
344 field.as_int32());
345 if (!opt_enum_string) {
346 // Fall back to the integer representation of the field.
347 delegate.AddInteger(key_prefix_, field.as_int32());
348 return base::OkStatus();
349 }
350 delegate.AddString(key_prefix_,
351 protozero::ConstChars{opt_enum_string->data(),
352 opt_enum_string->size()});
353 return base::OkStatus();
354 }
355 default:
356 return base::ErrStatus(
357 "Tried to write value of type field %s (in proto type "
358 "%s) which has type enum %d",
359 descriptor.name().c_str(), descriptor.resolved_type_name().c_str(),
360 descriptor.type());
361 }
362 }
363
EnterArray(size_t index)364 ProtoToArgsParser::ScopedNestedKeyContext ProtoToArgsParser::EnterArray(
365 size_t index) {
366 auto context = ScopedNestedKeyContext(key_prefix_);
367 key_prefix_.key += "[" + std::to_string(index) + "]";
368 return context;
369 }
370
EnterDictionary(const std::string & name)371 ProtoToArgsParser::ScopedNestedKeyContext ProtoToArgsParser::EnterDictionary(
372 const std::string& name) {
373 auto context = ScopedNestedKeyContext(key_prefix_);
374 AppendProtoType(key_prefix_.key, name);
375 AppendProtoType(key_prefix_.flat_key, name);
376 return context;
377 }
378
379 } // namespace util
380 } // namespace trace_processor
381 } // namespace perfetto
382