1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/util/proto_to_args_parser.h"
18
19 #include <algorithm>
20 #include <cstddef>
21 #include <cstdint>
22 #include <optional>
23 #include <string>
24 #include <string_view>
25 #include <unordered_map>
26 #include <unordered_set>
27 #include <utility>
28 #include <vector>
29
30 #include "perfetto/base/status.h"
31 #include "perfetto/ext/base/string_utils.h"
32 #include "perfetto/protozero/field.h"
33 #include "perfetto/protozero/proto_decoder.h"
34 #include "perfetto/protozero/proto_utils.h"
35 #include "src/trace_processor/util/descriptors.h"
36 #include "src/trace_processor/util/status_macros.h"
37
38 #include "protos/perfetto/common/descriptor.pbzero.h"
39
40 namespace perfetto::trace_processor::util {
41
42 namespace {
43
44 template <protozero::proto_utils::ProtoWireType wire_type, typename cpp_type>
45 using PRFI = protozero::PackedRepeatedFieldIterator<wire_type, cpp_type>;
46
AppendProtoType(std::string & target,const std::string & value)47 void AppendProtoType(std::string& target, const std::string& value) {
48 if (!target.empty())
49 target += '.';
50 target += value;
51 }
52
IsFieldAllowed(const FieldDescriptor & field,const std::vector<uint32_t> * allowed_fields)53 bool IsFieldAllowed(const FieldDescriptor& field,
54 const std::vector<uint32_t>* allowed_fields) {
55 // If allowlist is not provided, reflect all fields. Otherwise, check if the
56 // current field either an extension or is in allowlist.
57 return field.is_extension() || !allowed_fields ||
58 std::find(allowed_fields->begin(), allowed_fields->end(),
59 field.number()) != allowed_fields->end();
60 }
61
62 } // namespace
63
64 ProtoToArgsParser::Key::Key() = default;
Key(const std::string & k)65 ProtoToArgsParser::Key::Key(const std::string& k) : flat_key(k), key(k) {}
Key(const std::string & fk,const std::string & k)66 ProtoToArgsParser::Key::Key(const std::string& fk, const std::string& k)
67 : flat_key(fk), key(k) {}
68 ProtoToArgsParser::Key::~Key() = default;
69
ScopedNestedKeyContext(Key & key)70 ProtoToArgsParser::ScopedNestedKeyContext::ScopedNestedKeyContext(Key& key)
71 : key_(key),
72 old_flat_key_length_(key.flat_key.length()),
73 old_key_length_(key.key.length()) {}
74
ScopedNestedKeyContext(ProtoToArgsParser::ScopedNestedKeyContext && other)75 ProtoToArgsParser::ScopedNestedKeyContext::ScopedNestedKeyContext(
76 ProtoToArgsParser::ScopedNestedKeyContext&& other) noexcept
77 : key_(other.key_),
78 old_flat_key_length_(other.old_flat_key_length_),
79 old_key_length_(other.old_key_length_) {
80 other.old_flat_key_length_ = std::nullopt;
81 other.old_key_length_ = std::nullopt;
82 }
83
~ScopedNestedKeyContext()84 ProtoToArgsParser::ScopedNestedKeyContext::~ScopedNestedKeyContext() {
85 RemoveFieldSuffix();
86 }
87
RemoveFieldSuffix()88 void ProtoToArgsParser::ScopedNestedKeyContext::RemoveFieldSuffix() {
89 if (old_flat_key_length_)
90 key_.flat_key.resize(old_flat_key_length_.value());
91 if (old_key_length_)
92 key_.key.resize(old_key_length_.value());
93 old_flat_key_length_ = std::nullopt;
94 old_key_length_ = std::nullopt;
95 }
96
97 ProtoToArgsParser::Delegate::~Delegate() = default;
98
ProtoToArgsParser(const DescriptorPool & pool)99 ProtoToArgsParser::ProtoToArgsParser(const DescriptorPool& pool) : pool_(pool) {
100 constexpr int kDefaultSize = 64;
101 key_prefix_.key.reserve(kDefaultSize);
102 key_prefix_.flat_key.reserve(kDefaultSize);
103 }
104
ParseMessage(const protozero::ConstBytes & cb,const std::string & type,const std::vector<uint32_t> * allowed_fields,Delegate & delegate,int * unknown_extensions,bool add_defaults)105 base::Status ProtoToArgsParser::ParseMessage(
106 const protozero::ConstBytes& cb,
107 const std::string& type,
108 const std::vector<uint32_t>* allowed_fields,
109 Delegate& delegate,
110 int* unknown_extensions,
111 bool add_defaults) {
112 ScopedNestedKeyContext key_context(key_prefix_);
113 return ParseMessageInternal(key_context, cb, type, allowed_fields, delegate,
114 unknown_extensions, add_defaults);
115 }
116
ParseMessageInternal(ScopedNestedKeyContext & key_context,const protozero::ConstBytes & cb,const std::string & type,const std::vector<uint32_t> * allowed_fields,Delegate & delegate,int * unknown_extensions,bool add_defaults)117 base::Status ProtoToArgsParser::ParseMessageInternal(
118 ScopedNestedKeyContext& key_context,
119 const protozero::ConstBytes& cb,
120 const std::string& type,
121 const std::vector<uint32_t>* allowed_fields,
122 Delegate& delegate,
123 int* unknown_extensions,
124 bool add_defaults) {
125 if (auto override_result =
126 MaybeApplyOverrideForType(type, key_context, cb, delegate)) {
127 return override_result.value();
128 }
129
130 auto idx = pool_.FindDescriptorIdx(type);
131 if (!idx) {
132 return base::Status("Failed to find proto descriptor");
133 }
134
135 const auto& descriptor = pool_.descriptors()[*idx];
136
137 std::unordered_map<size_t, int> repeated_field_index;
138 bool empty_message = true;
139 protozero::ProtoDecoder decoder(cb);
140 std::unordered_set<std::string_view> existing_fields;
141 for (protozero::Field f = decoder.ReadField(); f.valid();
142 f = decoder.ReadField()) {
143 empty_message = false;
144 const auto* field = descriptor.FindFieldByTag(f.id());
145 if (!field) {
146 if (unknown_extensions != nullptr) {
147 (*unknown_extensions)++;
148 }
149 // Unknown field, possibly an unknown extension.
150 continue;
151 }
152
153 if (add_defaults) {
154 existing_fields.insert(field->name());
155 }
156
157 if (!IsFieldAllowed(*field, allowed_fields)) {
158 // Field is neither an extension, nor is allowed to be
159 // reflected.
160 continue;
161 }
162
163 // Packed fields need to be handled specially because
164 if (field->is_packed()) {
165 RETURN_IF_ERROR(ParsePackedField(*field, repeated_field_index, f,
166 delegate, unknown_extensions,
167 add_defaults));
168 continue;
169 }
170
171 RETURN_IF_ERROR(ParseField(*field, repeated_field_index[f.id()], f,
172 delegate, unknown_extensions, add_defaults));
173 if (field->is_repeated()) {
174 repeated_field_index[f.id()]++;
175 }
176 }
177
178 if (empty_message) {
179 delegate.AddNull(key_prefix_);
180 } else if (add_defaults) {
181 for (const auto& [id, field] : descriptor.fields()) {
182 if (!IsFieldAllowed(field, allowed_fields)) {
183 continue;
184 }
185 const std::string& field_name = field.name();
186 bool field_exists =
187 existing_fields.find(field_name) != existing_fields.cend();
188 if (field_exists) {
189 continue;
190 }
191 ScopedNestedKeyContext key_context_default(key_prefix_);
192 AppendProtoType(key_prefix_.flat_key, field_name);
193 AppendProtoType(key_prefix_.key, field_name);
194 RETURN_IF_ERROR(AddDefault(field, delegate));
195 }
196 }
197
198 return base::OkStatus();
199 }
200
ParseField(const FieldDescriptor & field_descriptor,int repeated_field_number,protozero::Field field,Delegate & delegate,int * unknown_extensions,bool add_defaults)201 base::Status ProtoToArgsParser::ParseField(
202 const FieldDescriptor& field_descriptor,
203 int repeated_field_number,
204 protozero::Field field,
205 Delegate& delegate,
206 int* unknown_extensions,
207 bool add_defaults) {
208 std::string prefix_part = field_descriptor.name();
209 if (field_descriptor.is_repeated()) {
210 std::string number = std::to_string(repeated_field_number);
211 prefix_part.reserve(prefix_part.length() + number.length() + 2);
212 prefix_part.append("[");
213 prefix_part.append(number);
214 prefix_part.append("]");
215 }
216
217 // In the args table we build up message1.message2.field1 as the column
218 // name. This will append the ".field1" suffix to |key_prefix| and then
219 // remove it when it goes out of scope.
220 ScopedNestedKeyContext key_context(key_prefix_);
221 AppendProtoType(key_prefix_.flat_key, field_descriptor.name());
222 AppendProtoType(key_prefix_.key, prefix_part);
223
224 // If we have an override parser then use that instead and move onto the
225 // next loop.
226 if (std::optional<base::Status> status =
227 MaybeApplyOverrideForField(field, delegate)) {
228 return *status;
229 }
230
231 // If this is not a message we can just immediately add the column name and
232 // get the value out of |field|. However if it is a message we need to
233 // recurse into it.
234 if (field_descriptor.type() ==
235 protos::pbzero::FieldDescriptorProto::TYPE_MESSAGE) {
236 return ParseMessageInternal(key_context, field.as_bytes(),
237 field_descriptor.resolved_type_name(), nullptr,
238 delegate, unknown_extensions, add_defaults);
239 }
240 return ParseSimpleField(field_descriptor, field, delegate);
241 }
242
ParsePackedField(const FieldDescriptor & field_descriptor,std::unordered_map<size_t,int> & repeated_field_index,protozero::Field field,Delegate & delegate,int * unknown_extensions,bool add_defaults)243 base::Status ProtoToArgsParser::ParsePackedField(
244 const FieldDescriptor& field_descriptor,
245 std::unordered_map<size_t, int>& repeated_field_index,
246 protozero::Field field,
247 Delegate& delegate,
248 int* unknown_extensions,
249 bool add_defaults) {
250 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
251 using PWT = protozero::proto_utils::ProtoWireType;
252
253 if (!field_descriptor.is_repeated()) {
254 return base::ErrStatus("Packed field %s must be repeated",
255 field_descriptor.name().c_str());
256 }
257 if (field.type() != PWT::kLengthDelimited) {
258 return base::ErrStatus(
259 "Packed field %s must have a length delimited wire type",
260 field_descriptor.name().c_str());
261 }
262
263 auto parse = [&](uint64_t new_value, PWT wire_type) {
264 protozero::Field f;
265 f.initialize(field.id(), static_cast<uint8_t>(wire_type), new_value, 0);
266 return ParseField(field_descriptor, repeated_field_index[field.id()]++, f,
267 delegate, unknown_extensions, add_defaults);
268 };
269
270 const uint8_t* data = field.as_bytes().data;
271 size_t size = field.as_bytes().size;
272 bool perr = false;
273 switch (field_descriptor.type()) {
274 case FieldDescriptorProto::TYPE_INT32:
275 case FieldDescriptorProto::TYPE_INT64:
276 case FieldDescriptorProto::TYPE_UINT32:
277 case FieldDescriptorProto::TYPE_UINT64:
278 case FieldDescriptorProto::TYPE_ENUM:
279 for (PRFI<PWT::kVarInt, uint64_t> it(data, size, &perr); it; ++it) {
280 parse(*it, PWT::kVarInt);
281 }
282 break;
283 case FieldDescriptorProto::TYPE_FIXED32:
284 case FieldDescriptorProto::TYPE_SFIXED32:
285 case FieldDescriptorProto::TYPE_FLOAT:
286 for (PRFI<PWT::kFixed32, uint32_t> it(data, size, &perr); it; ++it) {
287 parse(*it, PWT::kFixed32);
288 }
289 break;
290 case FieldDescriptorProto::TYPE_FIXED64:
291 case FieldDescriptorProto::TYPE_SFIXED64:
292 case FieldDescriptorProto::TYPE_DOUBLE:
293 for (PRFI<PWT::kFixed64, uint64_t> it(data, size, &perr); it; ++it) {
294 parse(*it, PWT::kFixed64);
295 }
296 break;
297 default:
298 return base::ErrStatus("Unsupported packed repeated field");
299 }
300 return base::OkStatus();
301 }
302
AddParsingOverrideForField(const std::string & field,ParsingOverrideForField func)303 void ProtoToArgsParser::AddParsingOverrideForField(
304 const std::string& field,
305 ParsingOverrideForField func) {
306 field_overrides_[field] = std::move(func);
307 }
308
AddParsingOverrideForType(const std::string & type,ParsingOverrideForType func)309 void ProtoToArgsParser::AddParsingOverrideForType(const std::string& type,
310 ParsingOverrideForType func) {
311 type_overrides_[type] = std::move(func);
312 }
313
MaybeApplyOverrideForField(const protozero::Field & field,Delegate & delegate)314 std::optional<base::Status> ProtoToArgsParser::MaybeApplyOverrideForField(
315 const protozero::Field& field,
316 Delegate& delegate) {
317 auto it = field_overrides_.find(key_prefix_.flat_key);
318 if (it == field_overrides_.end())
319 return std::nullopt;
320 return it->second(field, delegate);
321 }
322
MaybeApplyOverrideForType(const std::string & message_type,ScopedNestedKeyContext & key,const protozero::ConstBytes & data,Delegate & delegate)323 std::optional<base::Status> ProtoToArgsParser::MaybeApplyOverrideForType(
324 const std::string& message_type,
325 ScopedNestedKeyContext& key,
326 const protozero::ConstBytes& data,
327 Delegate& delegate) {
328 auto it = type_overrides_.find(message_type);
329 if (it == type_overrides_.end())
330 return std::nullopt;
331 return it->second(key, data, delegate);
332 }
333
ParseSimpleField(const FieldDescriptor & descriptor,const protozero::Field & field,Delegate & delegate)334 base::Status ProtoToArgsParser::ParseSimpleField(
335 const FieldDescriptor& descriptor,
336 const protozero::Field& field,
337 Delegate& delegate) {
338 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
339 switch (descriptor.type()) {
340 case FieldDescriptorProto::TYPE_INT32:
341 case FieldDescriptorProto::TYPE_SFIXED32:
342 delegate.AddInteger(key_prefix_, field.as_int32());
343 return base::OkStatus();
344 case FieldDescriptorProto::TYPE_SINT32:
345 delegate.AddInteger(key_prefix_, field.as_sint32());
346 return base::OkStatus();
347 case FieldDescriptorProto::TYPE_INT64:
348 case FieldDescriptorProto::TYPE_SFIXED64:
349 delegate.AddInteger(key_prefix_, field.as_int64());
350 return base::OkStatus();
351 case FieldDescriptorProto::TYPE_SINT64:
352 delegate.AddInteger(key_prefix_, field.as_sint64());
353 return base::OkStatus();
354 case FieldDescriptorProto::TYPE_UINT32:
355 case FieldDescriptorProto::TYPE_FIXED32:
356 delegate.AddUnsignedInteger(key_prefix_, field.as_uint32());
357 return base::OkStatus();
358 case FieldDescriptorProto::TYPE_UINT64:
359 case FieldDescriptorProto::TYPE_FIXED64:
360 delegate.AddUnsignedInteger(key_prefix_, field.as_uint64());
361 return base::OkStatus();
362 case FieldDescriptorProto::TYPE_BOOL:
363 delegate.AddBoolean(key_prefix_, field.as_bool());
364 return base::OkStatus();
365 case FieldDescriptorProto::TYPE_DOUBLE:
366 delegate.AddDouble(key_prefix_, field.as_double());
367 return base::OkStatus();
368 case FieldDescriptorProto::TYPE_FLOAT:
369 delegate.AddDouble(key_prefix_, static_cast<double>(field.as_float()));
370 return base::OkStatus();
371 case FieldDescriptorProto::TYPE_BYTES:
372 delegate.AddBytes(key_prefix_, field.as_bytes());
373 return base::OkStatus();
374 case FieldDescriptorProto::TYPE_STRING:
375 delegate.AddString(key_prefix_, field.as_string());
376 return base::OkStatus();
377 case FieldDescriptorProto::TYPE_ENUM:
378 return AddEnum(descriptor, field.as_int32(), delegate);
379 default:
380 return base::ErrStatus(
381 "Tried to write value of type field %s (in proto type "
382 "%s) which has type enum %u",
383 descriptor.name().c_str(), descriptor.resolved_type_name().c_str(),
384 descriptor.type());
385 }
386 }
387
EnterArray(size_t index)388 ProtoToArgsParser::ScopedNestedKeyContext ProtoToArgsParser::EnterArray(
389 size_t index) {
390 ScopedNestedKeyContext context(key_prefix_);
391 key_prefix_.key += "[" + std::to_string(index) + "]";
392 return context;
393 }
394
EnterDictionary(const std::string & name)395 ProtoToArgsParser::ScopedNestedKeyContext ProtoToArgsParser::EnterDictionary(
396 const std::string& name) {
397 ScopedNestedKeyContext context(key_prefix_);
398 AppendProtoType(key_prefix_.key, name);
399 AppendProtoType(key_prefix_.flat_key, name);
400 return context;
401 }
402
AddDefault(const FieldDescriptor & descriptor,Delegate & delegate)403 base::Status ProtoToArgsParser::AddDefault(const FieldDescriptor& descriptor,
404 Delegate& delegate) {
405 using FieldDescriptorProto = protos::pbzero::FieldDescriptorProto;
406 if (!delegate.ShouldAddDefaultArg(key_prefix_)) {
407 return base::OkStatus();
408 }
409 if (descriptor.is_repeated()) {
410 delegate.AddNull(key_prefix_);
411 return base::OkStatus();
412 }
413 const auto& default_value = descriptor.default_value();
414 const auto& default_value_if_number =
415 default_value ? default_value.value() : "0";
416 switch (descriptor.type()) {
417 case FieldDescriptorProto::TYPE_INT32:
418 case FieldDescriptorProto::TYPE_SFIXED32:
419 delegate.AddInteger(key_prefix_,
420 base::StringToInt32(default_value_if_number).value());
421 return base::OkStatus();
422 case FieldDescriptorProto::TYPE_SINT32:
423 delegate.AddInteger(
424 key_prefix_,
425 protozero::proto_utils::ZigZagDecode(
426 base::StringToInt64(default_value_if_number).value()));
427 return base::OkStatus();
428 case FieldDescriptorProto::TYPE_INT64:
429 case FieldDescriptorProto::TYPE_SFIXED64:
430 delegate.AddInteger(key_prefix_,
431 base::StringToInt64(default_value_if_number).value());
432 return base::OkStatus();
433 case FieldDescriptorProto::TYPE_SINT64:
434 delegate.AddInteger(
435 key_prefix_,
436 protozero::proto_utils::ZigZagDecode(
437 base::StringToInt64(default_value_if_number).value()));
438 return base::OkStatus();
439 case FieldDescriptorProto::TYPE_UINT32:
440 case FieldDescriptorProto::TYPE_FIXED32:
441 delegate.AddUnsignedInteger(
442 key_prefix_, base::StringToUInt32(default_value_if_number).value());
443 return base::OkStatus();
444 case FieldDescriptorProto::TYPE_UINT64:
445 case FieldDescriptorProto::TYPE_FIXED64:
446 delegate.AddUnsignedInteger(
447 key_prefix_, base::StringToUInt64(default_value_if_number).value());
448 return base::OkStatus();
449 case FieldDescriptorProto::TYPE_BOOL:
450 delegate.AddBoolean(key_prefix_, default_value == "true");
451 return base::OkStatus();
452 case FieldDescriptorProto::TYPE_DOUBLE:
453 case FieldDescriptorProto::TYPE_FLOAT:
454 delegate.AddDouble(key_prefix_,
455 base::StringToDouble(default_value_if_number).value());
456 return base::OkStatus();
457 case FieldDescriptorProto::TYPE_BYTES:
458 delegate.AddBytes(key_prefix_, protozero::ConstBytes{});
459 return base::OkStatus();
460 case FieldDescriptorProto::TYPE_STRING:
461 if (default_value) {
462 delegate.AddString(key_prefix_, default_value.value());
463 } else {
464 delegate.AddNull(key_prefix_);
465 }
466 return base::OkStatus();
467 case FieldDescriptorProto::TYPE_MESSAGE:
468 delegate.AddNull(key_prefix_);
469 return base::OkStatus();
470 case FieldDescriptorProto::TYPE_ENUM:
471 return AddEnum(descriptor,
472 base::StringToInt32(default_value_if_number).value(),
473 delegate);
474 default:
475 return base::ErrStatus(
476 "Tried to write default value of type field %s (in proto type "
477 "%s) which has type enum %u",
478 descriptor.name().c_str(), descriptor.resolved_type_name().c_str(),
479 descriptor.type());
480 }
481 }
482
AddEnum(const FieldDescriptor & descriptor,int32_t value,Delegate & delegate)483 base::Status ProtoToArgsParser::AddEnum(const FieldDescriptor& descriptor,
484 int32_t value,
485 Delegate& delegate) {
486 auto opt_enum_descriptor_idx =
487 pool_.FindDescriptorIdx(descriptor.resolved_type_name());
488 if (!opt_enum_descriptor_idx) {
489 // Fall back to the integer representation of the field.
490 // We add the string representation of the int value here in order that
491 // EXTRACT_ARG() should return consistent types under error conditions and
492 // that CREATE PERFETTO TABLE AS EXTRACT_ARG(...) should be generally safe
493 // to use.
494 delegate.AddString(key_prefix_, std::to_string(value));
495 return base::OkStatus();
496 }
497 auto opt_enum_string =
498 pool_.descriptors()[*opt_enum_descriptor_idx].FindEnumString(value);
499 if (!opt_enum_string) {
500 // Fall back to the integer representation of the field. See above for
501 // motivation.
502 delegate.AddString(key_prefix_, std::to_string(value));
503 return base::OkStatus();
504 }
505 delegate.AddString(
506 key_prefix_,
507 protozero::ConstChars{opt_enum_string->data(), opt_enum_string->size()});
508 return base::OkStatus();
509 }
510 } // namespace perfetto::trace_processor::util
511