1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/util/protozero_to_json.h"
18
19 #include <algorithm>
20 #include <cstddef>
21 #include <cstdint>
22 #include <optional>
23 #include <string>
24 #include <unordered_set>
25 #include <utility>
26 #include <vector>
27
28 #include "perfetto/base/logging.h"
29 #include "perfetto/ext/base/string_utils.h"
30 #include "perfetto/ext/base/string_view.h"
31 #include "perfetto/protozero/field.h"
32 #include "perfetto/protozero/proto_decoder.h"
33 #include "perfetto/protozero/proto_utils.h"
34 #include "src/trace_processor/util/descriptors.h"
35
36 #include "protos/perfetto/common/descriptor.pbzero.h"
37
38 namespace perfetto::trace_processor::protozero_to_json {
39
40 namespace {
41
42 using protos::pbzero::FieldDescriptorProto;
43 using protozero::PackedRepeatedFieldIterator;
44 using protozero::proto_utils::ProtoWireType;
45
46 class JsonBuilder {
47 public:
JsonBuilder(int flags)48 explicit JsonBuilder(int flags) : flags_(flags) {}
49
OpenObject()50 void OpenObject() {
51 if (is_array_scope()) {
52 if (!is_empty_scope()) {
53 Append(",");
54 }
55 MaybeAppendNewline();
56 MaybeAppendIndent();
57 }
58 Append("{");
59 stack_.push_back(Scope{ScopeContext::kObject});
60 }
61
CloseObject()62 void CloseObject() {
63 bool needs_newline = !is_empty_scope();
64 stack_.pop_back();
65 if (needs_newline) {
66 MaybeAppendNewline();
67 MaybeAppendIndent();
68 }
69
70 MarkScopeAsNonEmpty();
71 Append("}");
72 }
73
OpenArray()74 void OpenArray() {
75 Append("[");
76 stack_.push_back(Scope{ScopeContext::kArray});
77 }
78
CloseArray()79 void CloseArray() {
80 bool needs_newline = !is_empty_scope();
81 stack_.pop_back();
82 if (needs_newline) {
83 MaybeAppendNewline();
84 MaybeAppendIndent();
85 }
86 Append("]");
87 if (is_array_scope() && !is_empty_scope()) {
88 Append(",");
89 }
90 }
91
Key(const std::string & key)92 void Key(const std::string& key) {
93 if (is_object_scope() && !is_empty_scope()) {
94 Append(",");
95 }
96 MaybeAppendNewline();
97 MaybeAppendIndent();
98 Append(EscapeString(base::StringView(key)));
99 Append(":");
100 MaybeAppendSpace();
101 MarkScopeAsNonEmpty();
102 }
103
104 template <typename T>
NumberValue(T v)105 void NumberValue(T v) {
106 AppendValue(std::to_string(v));
107 }
108
BoolValue(bool v)109 void BoolValue(bool v) { AppendValue(v ? "true" : "false"); }
110
FloatValue(float v)111 void FloatValue(float v) { NumberValue(v); }
112
DoubleValue(double v)113 void DoubleValue(double v) { NumberValue(v); }
114
StringValue(base::StringView v)115 void StringValue(base::StringView v) { AppendValue(EscapeString(v)); }
116
AddError(const std::string & s)117 void AddError(const std::string& s) { errors_.push_back(s); }
118
ToString()119 std::string ToString() { return base::Join(parts_, ""); }
120
is_empty_scope()121 bool is_empty_scope() { return !stack_.empty() && stack_.back().is_empty; }
122
is_pretty() const123 bool is_pretty() const { return flags_ & Flags::kPretty; }
124
is_inline_errors() const125 bool is_inline_errors() const { return flags_ & Flags::kInlineErrors; }
126
errors() const127 const std::vector<std::string>& errors() const { return errors_; }
128
129 private:
130 enum class ScopeContext {
131 kObject,
132 kArray,
133 };
134
135 struct Scope {
136 ScopeContext ctx;
137 bool is_empty = true;
138 };
139
140 int flags_;
141 std::vector<std::string> parts_;
142 std::vector<Scope> stack_;
143 std::vector<std::string> errors_;
144
is_object_scope()145 bool is_object_scope() {
146 return !stack_.empty() && stack_.back().ctx == ScopeContext::kObject;
147 }
148
is_array_scope()149 bool is_array_scope() {
150 return !stack_.empty() && stack_.back().ctx == ScopeContext::kArray;
151 }
152
MarkScopeAsNonEmpty()153 void MarkScopeAsNonEmpty() {
154 if (!stack_.empty()) {
155 stack_.back().is_empty = false;
156 }
157 }
158
MaybeAppendSpace()159 void MaybeAppendSpace() {
160 if (is_pretty()) {
161 Append(" ");
162 }
163 }
164
MaybeAppendIndent()165 void MaybeAppendIndent() {
166 if (is_pretty()) {
167 Append(std::string(stack_.size() * 2, ' '));
168 }
169 }
170
MaybeAppendNewline()171 void MaybeAppendNewline() {
172 if (is_pretty()) {
173 Append("\n");
174 }
175 }
176
AppendValue(const std::string & s)177 void AppendValue(const std::string& s) {
178 if (is_array_scope() && !is_empty_scope()) {
179 Append(",");
180 }
181 if (is_array_scope()) {
182 MaybeAppendNewline();
183 MaybeAppendIndent();
184 }
185 Append(s);
186 MarkScopeAsNonEmpty();
187 }
188
Append(const std::string & s)189 void Append(const std::string& s) { parts_.push_back(s); }
190
EscapeString(base::StringView raw)191 std::string EscapeString(base::StringView raw) {
192 std::string result;
193 result.reserve(raw.size() + 2);
194 result += "\"";
195 for (size_t i = 0; i < raw.size(); ++i) {
196 char c = *(raw.begin() + i);
197 switch (c) {
198 case '"':
199 case '\\':
200 result += '\\';
201 result += c;
202 break;
203 case '\n':
204 result += R"(\n)";
205 break;
206 case '\b':
207 result += R"(\b)";
208 break;
209 case '\f':
210 result += R"(\f)";
211 break;
212 case '\r':
213 result += R"(\r)";
214 break;
215 case '\t':
216 result += R"(\t)";
217 break;
218 default:
219 // ASCII characters between 0x20 (space) and 0x7e (tilde) are
220 // inserted directly. All others are escaped.
221 if (c >= 0x20 && c <= 0x7e) {
222 result += c;
223 } else {
224 unsigned char uc = static_cast<unsigned char>(c);
225 uint32_t codepoint = 0;
226
227 // Compute the number of bytes:
228 size_t extra = 1 + (uc >= 0xc0u) + (uc >= 0xe0u) + (uc >= 0xf0u);
229
230 // We want to consume |extra| bytes but also need to not
231 // read out of bounds:
232 size_t stop = std::min(raw.size(), i + extra);
233
234 // Manually insert the bits from first byte:
235 codepoint |= uc & (0xff >> (extra + 1));
236
237 // Insert remaining bits:
238 for (size_t j = i + 1; j < stop; ++j) {
239 uc = static_cast<unsigned char>(*(raw.begin() + j));
240 codepoint = (codepoint << 6) | (uc & 0x3f);
241 }
242
243 // Update i to show the consumed chars:
244 i = stop - 1;
245
246 static const char hex_chars[] = "0123456789abcdef";
247 // JSON does not have proper utf-8 escapes. Instead you
248 // have to use utf-16 codes. For the low codepoints
249 // \uXXXX and for the high codepoints a surrogate pair:
250 // \uXXXX\uYYYY
251 if (codepoint <= 0xffff) {
252 result += R"(\u)";
253 result += hex_chars[(codepoint >> 12) & 0xf];
254 result += hex_chars[(codepoint >> 8) & 0xf];
255 result += hex_chars[(codepoint >> 4) & 0xf];
256 result += hex_chars[(codepoint >> 0) & 0xf];
257 } else {
258 uint32_t high = ((codepoint - 0x10000) >> 10) + 0xD800;
259 uint32_t low = (codepoint & 0x4fff) + 0xDC00;
260 result += R"(\u)";
261 result += hex_chars[(high >> 12) & 0xf];
262 result += hex_chars[(high >> 8) & 0xf];
263 result += hex_chars[(high >> 4) & 0xf];
264 result += hex_chars[(high >> 0) & 0xf];
265 result += R"(\u)";
266 result += hex_chars[(low >> 12) & 0xf];
267 result += hex_chars[(low >> 8) & 0xf];
268 result += hex_chars[(low >> 4) & 0xf];
269 result += hex_chars[(low >> 0) & 0xf];
270 }
271 }
272 break;
273 }
274 }
275 result += "\"";
276 return result;
277 }
278 };
279
HasFieldOptions(const FieldDescriptor & field_desc)280 bool HasFieldOptions(const FieldDescriptor& field_desc) {
281 return !field_desc.options().empty();
282 }
283
FulllyQualifiedFieldName(const ProtoDescriptor & desc,const FieldDescriptor & field_desc)284 std::string FulllyQualifiedFieldName(const ProtoDescriptor& desc,
285 const FieldDescriptor& field_desc) {
286 return desc.package_name().substr(1) + "." + field_desc.name();
287 }
288
IsTypeMatch(ProtoWireType wire,uint32_t type)289 bool IsTypeMatch(ProtoWireType wire, uint32_t type) {
290 switch (wire) {
291 case ProtoWireType::kVarInt:
292 switch (type) {
293 case FieldDescriptorProto::TYPE_INT32:
294 case FieldDescriptorProto::TYPE_SINT32:
295 case FieldDescriptorProto::TYPE_UINT32:
296 case FieldDescriptorProto::TYPE_INT64:
297 case FieldDescriptorProto::TYPE_SINT64:
298 case FieldDescriptorProto::TYPE_UINT64:
299 case FieldDescriptorProto::TYPE_BOOL:
300 case FieldDescriptorProto::TYPE_ENUM:
301 return true;
302 default:
303 return false;
304 }
305 case ProtoWireType::kLengthDelimited:
306 switch (type) {
307 case FieldDescriptorProto::TYPE_BYTES:
308 case FieldDescriptorProto::TYPE_MESSAGE:
309 case FieldDescriptorProto::TYPE_STRING:
310 // The normal case.
311 return true;
312 case FieldDescriptorProto::TYPE_INT32:
313 case FieldDescriptorProto::TYPE_SINT32:
314 case FieldDescriptorProto::TYPE_UINT32:
315 case FieldDescriptorProto::TYPE_INT64:
316 case FieldDescriptorProto::TYPE_SINT64:
317 case FieldDescriptorProto::TYPE_UINT64:
318 case FieldDescriptorProto::TYPE_BOOL:
319 case FieldDescriptorProto::TYPE_ENUM:
320 case FieldDescriptorProto::TYPE_FIXED32:
321 case FieldDescriptorProto::TYPE_SFIXED32:
322 case FieldDescriptorProto::TYPE_FLOAT:
323 case FieldDescriptorProto::TYPE_FIXED64:
324 case FieldDescriptorProto::TYPE_SFIXED64:
325 case FieldDescriptorProto::TYPE_DOUBLE:
326 // Packed repeated fields.
327 return true;
328 default:
329 return false;
330 }
331 case ProtoWireType::kFixed32:
332 switch (type) {
333 case FieldDescriptorProto::TYPE_FIXED32:
334 case FieldDescriptorProto::TYPE_SFIXED32:
335 case FieldDescriptorProto::TYPE_FLOAT:
336 return true;
337 default:
338 return false;
339 }
340 case ProtoWireType::kFixed64:
341 switch (type) {
342 case FieldDescriptorProto::TYPE_FIXED64:
343 case FieldDescriptorProto::TYPE_SFIXED64:
344 case FieldDescriptorProto::TYPE_DOUBLE:
345 return true;
346 default:
347 return false;
348 }
349 }
350 PERFETTO_FATAL("For GCC");
351 }
352
IsNumericFieldType(uint32_t type)353 bool IsNumericFieldType(uint32_t type) {
354 switch (type) {
355 case FieldDescriptorProto::TYPE_BYTES:
356 case FieldDescriptorProto::TYPE_MESSAGE:
357 case FieldDescriptorProto::TYPE_STRING:
358 return false;
359 case FieldDescriptorProto::TYPE_INT32:
360 case FieldDescriptorProto::TYPE_SINT32:
361 case FieldDescriptorProto::TYPE_UINT32:
362 case FieldDescriptorProto::TYPE_INT64:
363 case FieldDescriptorProto::TYPE_SINT64:
364 case FieldDescriptorProto::TYPE_UINT64:
365 case FieldDescriptorProto::TYPE_BOOL:
366 case FieldDescriptorProto::TYPE_ENUM:
367 case FieldDescriptorProto::TYPE_FIXED32:
368 case FieldDescriptorProto::TYPE_SFIXED32:
369 case FieldDescriptorProto::TYPE_FLOAT:
370 case FieldDescriptorProto::TYPE_FIXED64:
371 case FieldDescriptorProto::TYPE_SFIXED64:
372 case FieldDescriptorProto::TYPE_DOUBLE:
373 default:
374 return true;
375 }
376 }
377
378 void MessageField(const DescriptorPool& pool,
379 const std::string& type,
380 protozero::ConstBytes protobytes,
381 bool fully_qualify_extensions,
382 JsonBuilder* out);
383 void EnumField(const DescriptorPool& pool,
384 const FieldDescriptor& fd,
385 int32_t value,
386 JsonBuilder* out);
387
388 template <ProtoWireType W, typename T>
PackedField(const DescriptorPool & pool,const FieldDescriptor & fd,const protozero::Field & field,JsonBuilder * out)389 void PackedField(const DescriptorPool& pool,
390 const FieldDescriptor& fd,
391 const protozero::Field& field,
392 JsonBuilder* out) {
393 out->OpenArray();
394 bool e = false;
395 for (PackedRepeatedFieldIterator<W, T> it(field.data(), field.size(), &e); it;
396 it++) {
397 T value = *it;
398 if (fd.type() == FieldDescriptorProto::TYPE_ENUM) {
399 EnumField(pool, fd, static_cast<int32_t>(value), out);
400 } else {
401 out->NumberValue<T>(value);
402 }
403 }
404 out->CloseArray();
405 if (e) {
406 out->AddError(
407 std::string("Decoding failure for field '" + fd.name() + "'"));
408 }
409 }
410
411 template <ProtoWireType W>
PackedBoolField(const DescriptorPool &,const FieldDescriptor & fd,const protozero::Field & field,JsonBuilder * out)412 void PackedBoolField(const DescriptorPool&,
413 const FieldDescriptor& fd,
414 const protozero::Field& field,
415 JsonBuilder* out) {
416 out->OpenArray();
417 bool e = false;
418 for (PackedRepeatedFieldIterator<W, int32_t> it(field.data(), field.size(),
419 &e);
420 it; it++) {
421 bool value = *it;
422 out->BoolValue(value);
423 }
424 out->CloseArray();
425 if (e) {
426 out->AddError(
427 std::string("Decoding failure for field '" + fd.name() + "'"));
428 }
429 }
430
LengthField(const DescriptorPool & pool,const FieldDescriptor * fd,const protozero::Field & field,bool fully_qualify_extensions,JsonBuilder * out)431 void LengthField(const DescriptorPool& pool,
432 const FieldDescriptor* fd,
433 const protozero::Field& field,
434 bool fully_qualify_extensions,
435 JsonBuilder* out) {
436 uint32_t type = fd ? fd->type() : 0;
437 switch (type) {
438 case FieldDescriptorProto::TYPE_BYTES:
439 out->StringValue(field.as_string());
440 return;
441 case FieldDescriptorProto::TYPE_STRING:
442 out->StringValue(field.as_string());
443 return;
444 case FieldDescriptorProto::TYPE_MESSAGE:
445 MessageField(pool, fd->resolved_type_name(), field.as_bytes(),
446 fully_qualify_extensions, out);
447 return;
448 case FieldDescriptorProto::TYPE_DOUBLE:
449 PackedField<ProtoWireType::kFixed64, double>(pool, *fd, field, out);
450 return;
451 case FieldDescriptorProto::TYPE_FLOAT:
452 PackedField<ProtoWireType::kFixed32, float>(pool, *fd, field, out);
453 return;
454 case FieldDescriptorProto::TYPE_FIXED32:
455 PackedField<ProtoWireType::kFixed32, uint32_t>(pool, *fd, field, out);
456 return;
457 case FieldDescriptorProto::TYPE_SFIXED32:
458 PackedField<ProtoWireType::kFixed32, int32_t>(pool, *fd, field, out);
459 return;
460 case FieldDescriptorProto::TYPE_INT32:
461 PackedField<ProtoWireType::kVarInt, int32_t>(pool, *fd, field, out);
462 return;
463 case FieldDescriptorProto::TYPE_SINT32:
464 PackedField<ProtoWireType::kVarInt, int32_t>(pool, *fd, field, out);
465 return;
466 case FieldDescriptorProto::TYPE_UINT32:
467 PackedField<ProtoWireType::kVarInt, uint32_t>(pool, *fd, field, out);
468 return;
469 case FieldDescriptorProto::TYPE_FIXED64:
470 PackedField<ProtoWireType::kFixed64, uint64_t>(pool, *fd, field, out);
471 return;
472 case FieldDescriptorProto::TYPE_SFIXED64:
473 PackedField<ProtoWireType::kFixed64, int64_t>(pool, *fd, field, out);
474 return;
475 case FieldDescriptorProto::TYPE_INT64:
476 PackedField<ProtoWireType::kVarInt, int64_t>(pool, *fd, field, out);
477 return;
478 case FieldDescriptorProto::TYPE_SINT64:
479 PackedField<ProtoWireType::kVarInt, int64_t>(pool, *fd, field, out);
480 return;
481 case FieldDescriptorProto::TYPE_UINT64:
482 PackedField<ProtoWireType::kVarInt, uint64_t>(pool, *fd, field, out);
483 return;
484 case FieldDescriptorProto::TYPE_ENUM:
485 PackedField<ProtoWireType::kVarInt, int32_t>(pool, *fd, field, out);
486 return;
487 case FieldDescriptorProto::TYPE_BOOL:
488 PackedBoolField<ProtoWireType::kVarInt>(pool, *fd, field, out);
489 return;
490 case 0:
491 default:
492 // In the absence of specific information display bytes.
493 out->StringValue(field.as_string());
494 return;
495 }
496 }
497
EnumField(const DescriptorPool & pool,const FieldDescriptor & fd,int32_t value,JsonBuilder * out)498 void EnumField(const DescriptorPool& pool,
499 const FieldDescriptor& fd,
500 int32_t value,
501 JsonBuilder* out) {
502 auto opt_enum_descriptor_idx =
503 pool.FindDescriptorIdx(fd.resolved_type_name());
504 if (!opt_enum_descriptor_idx) {
505 out->NumberValue(value);
506 return;
507 }
508 auto opt_enum_string =
509 pool.descriptors()[*opt_enum_descriptor_idx].FindEnumString(value);
510 // If the enum value is unknown, treat it like a completely unknown field.
511 if (!opt_enum_string) {
512 out->NumberValue(value);
513 return;
514 }
515
516 out->StringValue(base::StringView(*opt_enum_string));
517 }
518
VarIntField(const DescriptorPool & pool,const FieldDescriptor * fd,const protozero::Field & field,JsonBuilder * out)519 void VarIntField(const DescriptorPool& pool,
520 const FieldDescriptor* fd,
521 const protozero::Field& field,
522 JsonBuilder* out) {
523 uint32_t type = fd ? fd->type() : 0;
524 switch (type) {
525 case FieldDescriptorProto::TYPE_INT32:
526 out->NumberValue(field.as_int32());
527 return;
528 case FieldDescriptorProto::TYPE_SINT32:
529 out->NumberValue(field.as_sint32());
530 return;
531 case FieldDescriptorProto::TYPE_UINT32:
532 out->NumberValue(field.as_uint32());
533 return;
534 case FieldDescriptorProto::TYPE_INT64:
535 out->NumberValue(field.as_int64());
536 return;
537 case FieldDescriptorProto::TYPE_SINT64:
538 out->NumberValue(field.as_sint64());
539 return;
540 case FieldDescriptorProto::TYPE_UINT64:
541 out->NumberValue(field.as_uint64());
542 return;
543 case FieldDescriptorProto::TYPE_BOOL:
544 out->BoolValue(field.as_bool());
545 return;
546 case FieldDescriptorProto::TYPE_ENUM:
547 EnumField(pool, *fd, field.as_int32(), out);
548 return;
549 case 0:
550 default:
551 out->NumberValue(field.as_int64());
552 return;
553 }
554 }
555
Fixed32Field(const FieldDescriptor * fd,const protozero::Field & field,JsonBuilder * out)556 void Fixed32Field(const FieldDescriptor* fd,
557 const protozero::Field& field,
558 JsonBuilder* out) {
559 uint32_t type = fd ? fd->type() : 0;
560 switch (type) {
561 case FieldDescriptorProto::TYPE_SFIXED32:
562 out->NumberValue(field.as_int32());
563 break;
564 case FieldDescriptorProto::TYPE_FIXED32:
565 out->NumberValue(field.as_uint32());
566 break;
567 case FieldDescriptorProto::TYPE_FLOAT:
568 out->FloatValue(field.as_float());
569 break;
570 case 0:
571 default:
572 out->NumberValue(field.as_uint32());
573 break;
574 }
575 }
576
Fixed64Field(const FieldDescriptor * fd,const protozero::Field & field,JsonBuilder * out)577 void Fixed64Field(const FieldDescriptor* fd,
578 const protozero::Field& field,
579 JsonBuilder* out) {
580 uint64_t type = fd ? fd->type() : 0;
581 switch (type) {
582 case FieldDescriptorProto::TYPE_SFIXED64:
583 out->NumberValue(field.as_int64());
584 break;
585 case FieldDescriptorProto::TYPE_FIXED64:
586 out->NumberValue(field.as_uint64());
587 break;
588 case FieldDescriptorProto::TYPE_DOUBLE:
589 out->DoubleValue(field.as_double());
590 break;
591 case 0:
592 default:
593 out->NumberValue(field.as_uint64());
594 break;
595 }
596 }
597
RepeatedVarInt(const DescriptorPool & pool,protozero::ConstBytes protobytes,const FieldDescriptor * fd,uint32_t id,JsonBuilder * out)598 void RepeatedVarInt(const DescriptorPool& pool,
599 protozero::ConstBytes protobytes,
600 const FieldDescriptor* fd,
601 uint32_t id,
602 JsonBuilder* out) {
603 out->OpenArray();
604 protozero::ProtoDecoder decoder(protobytes.data, protobytes.size);
605 for (auto field = decoder.ReadField(); field.valid();
606 field = decoder.ReadField()) {
607 if (field.id() == id) {
608 VarIntField(pool, fd, field, out);
609 }
610 }
611 out->CloseArray();
612 }
613
RepeatedLengthField(const DescriptorPool & pool,protozero::ConstBytes protobytes,const FieldDescriptor * fd,uint32_t id,bool fully_qualify_extensions,JsonBuilder * out)614 void RepeatedLengthField(const DescriptorPool& pool,
615 protozero::ConstBytes protobytes,
616 const FieldDescriptor* fd,
617 uint32_t id,
618 bool fully_qualify_extensions,
619 JsonBuilder* out) {
620 out->OpenArray();
621 protozero::ProtoDecoder decoder(protobytes.data, protobytes.size);
622 for (auto field = decoder.ReadField(); field.valid();
623 field = decoder.ReadField()) {
624 if (field.id() == id) {
625 LengthField(pool, fd, field, fully_qualify_extensions, out);
626 }
627 }
628 out->CloseArray();
629 }
630
RepeatedFixed64(protozero::ConstBytes protobytes,const FieldDescriptor * fd,uint32_t id,JsonBuilder * out)631 void RepeatedFixed64(protozero::ConstBytes protobytes,
632 const FieldDescriptor* fd,
633 uint32_t id,
634 JsonBuilder* out) {
635 out->OpenArray();
636 protozero::ProtoDecoder decoder(protobytes.data, protobytes.size);
637 for (auto field = decoder.ReadField(); field.valid();
638 field = decoder.ReadField()) {
639 if (field.id() == id) {
640 Fixed64Field(fd, field, out);
641 }
642 }
643 out->CloseArray();
644 }
645
RepeatedFixed32(protozero::ConstBytes protobytes,const FieldDescriptor * fd,uint32_t id,JsonBuilder * out)646 void RepeatedFixed32(protozero::ConstBytes protobytes,
647 const FieldDescriptor* fd,
648 uint32_t id,
649 JsonBuilder* out) {
650 out->OpenArray();
651 protozero::ProtoDecoder decoder(protobytes.data, protobytes.size);
652 for (auto field = decoder.ReadField(); field.valid();
653 field = decoder.ReadField()) {
654 if (field.id() == id) {
655 Fixed32Field(fd, field, out);
656 }
657 }
658 out->CloseArray();
659 }
660
InnerMessageField(const DescriptorPool & pool,const std::string & type,protozero::ConstBytes protobytes,bool fully_qualify_extensions,JsonBuilder * out)661 void InnerMessageField(const DescriptorPool& pool,
662 const std::string& type,
663 protozero::ConstBytes protobytes,
664 bool fully_qualify_extensions,
665 JsonBuilder* out) {
666 std::optional<uint32_t> opt_proto_desc_idx = pool.FindDescriptorIdx(type);
667 const ProtoDescriptor* opt_proto_descriptor =
668 opt_proto_desc_idx ? &pool.descriptors()[*opt_proto_desc_idx] : nullptr;
669
670 protozero::ProtoDecoder decoder(protobytes.data, protobytes.size);
671 std::unordered_set<uint32_t> fields_seen;
672
673 for (auto field = decoder.ReadField(); field.valid();
674 field = decoder.ReadField()) {
675 auto* opt_field_descriptor =
676 opt_proto_descriptor ? opt_proto_descriptor->FindFieldByTag(field.id())
677 : nullptr;
678 bool is_repeated = false;
679 if (opt_field_descriptor &&
680 IsTypeMatch(field.type(), opt_field_descriptor->type())) {
681 is_repeated = opt_field_descriptor->is_repeated();
682 // The first time we see a repeated field we consume them all:
683 if (fields_seen.count(field.id())) {
684 continue;
685 }
686 if (opt_field_descriptor->is_extension() && fully_qualify_extensions) {
687 out->Key(FulllyQualifiedFieldName(*opt_proto_descriptor,
688 *opt_field_descriptor));
689 } else {
690 out->Key(opt_field_descriptor->name());
691 }
692 } else {
693 out->Key(std::to_string(field.id()));
694 }
695 if (is_repeated) {
696 fields_seen.insert(field.id());
697
698 switch (field.type()) {
699 case ProtoWireType::kVarInt:
700 RepeatedVarInt(pool, protobytes, opt_field_descriptor, field.id(),
701 out);
702 break;
703 case ProtoWireType::kLengthDelimited:
704 if (opt_field_descriptor &&
705 IsNumericFieldType(opt_field_descriptor->type())) {
706 // wire_type = length + field_type in
707 // {u,s,}int{32,64}, float, double etc means this is the
708 // packed case:
709 LengthField(pool, opt_field_descriptor, field,
710 fully_qualify_extensions, out);
711 } else {
712 RepeatedLengthField(pool, protobytes, opt_field_descriptor,
713 field.id(), fully_qualify_extensions, out);
714 }
715 break;
716 case ProtoWireType::kFixed32:
717 RepeatedFixed32(protobytes, opt_field_descriptor, field.id(), out);
718 break;
719 case ProtoWireType::kFixed64:
720 RepeatedFixed64(protobytes, opt_field_descriptor, field.id(), out);
721 break;
722 }
723 } else {
724 switch (field.type()) {
725 case ProtoWireType::kVarInt:
726 VarIntField(pool, opt_field_descriptor, field, out);
727 break;
728 case ProtoWireType::kLengthDelimited:
729 LengthField(pool, opt_field_descriptor, field,
730 fully_qualify_extensions, out);
731 break;
732 case ProtoWireType::kFixed32:
733 Fixed32Field(opt_field_descriptor, field, out);
734 break;
735 case ProtoWireType::kFixed64:
736 Fixed64Field(opt_field_descriptor, field, out);
737 break;
738 }
739 }
740 }
741
742 if (decoder.bytes_left() != 0) {
743 out->AddError(std::to_string(decoder.bytes_left()) + " extra bytes");
744 }
745 }
746
MessageField(const DescriptorPool & pool,const std::string & type,protozero::ConstBytes protobytes,bool fully_qualify_extensions,JsonBuilder * out)747 void MessageField(const DescriptorPool& pool,
748 const std::string& type,
749 protozero::ConstBytes protobytes,
750 bool fully_qualify_extensions,
751 JsonBuilder* out) {
752 out->OpenObject();
753 InnerMessageField(pool, type, protobytes, fully_qualify_extensions, out);
754 out->CloseObject();
755 }
756
757 // Prints all field options for non-empty fields of a message. Example:
758 // --- Message definitions ---
759 // FooMessage {
760 // repeated int64 foo = 1 [op1 = val1, op2 = val2];
761 // optional BarMessage bar = 2 [op3 = val3];
762 // }
763 //
764 // BarMessage {
765 // optional int64 baz = 1 [op4 = val4];
766 // }
767 // --- MessageInstance ---
768 // foo_msg = { // (As JSON)
769 // foo: [23, 24, 25],
770 // bar: {
771 // baz: 42
772 // }
773 // }
774 // --- Output of MessageFieldOptionsToJson(foo_msg) ---
775 // foo: {
776 // __field_options: {
777 // op1: val1,
778 // op2: val2,
779 // },
780 // __repeated: true
781 // }
782 // bar: {
783 // __field_options: {
784 // op3 = val3,
785 // },
786 // baz: {
787 // __field_options: {
788 // op4 = val4
789 // },
790 // }
791 // }
MessageFieldOptionsToJson(const DescriptorPool & pool,const std::string & type,const std::string & field_prefix,const std::unordered_set<std::string> & allowed_fields,JsonBuilder * out)792 void MessageFieldOptionsToJson(
793 const DescriptorPool& pool,
794 const std::string& type,
795 const std::string& field_prefix,
796 const std::unordered_set<std::string>& allowed_fields,
797 JsonBuilder* out) {
798 std::optional<uint32_t> opt_proto_desc_idx = pool.FindDescriptorIdx(type);
799 if (!opt_proto_desc_idx) {
800 return;
801 }
802 const ProtoDescriptor& desc = pool.descriptors()[*opt_proto_desc_idx];
803 for (const auto& id_and_field : desc.fields()) {
804 const FieldDescriptor& field_desc = id_and_field.second;
805 std::string full_field_name = field_prefix + field_desc.name();
806 if (allowed_fields.find(full_field_name) == allowed_fields.end()) {
807 continue;
808 }
809 if (field_desc.is_extension()) {
810 out->Key(FulllyQualifiedFieldName(desc, field_desc));
811 } else {
812 out->Key(field_desc.name());
813 }
814 out->OpenObject();
815 if (HasFieldOptions(field_desc)) {
816 out->Key("__field_options");
817 MessageField(pool, ".google.protobuf.FieldOptions",
818 protozero::ConstBytes{field_desc.options().data(),
819 field_desc.options().size()},
820 false, out);
821 }
822 if (field_desc.type() == FieldDescriptorProto::Type::TYPE_MESSAGE) {
823 MessageFieldOptionsToJson(pool, field_desc.resolved_type_name(),
824 full_field_name + ".", allowed_fields, out);
825 }
826 if (field_desc.is_repeated()) {
827 out->Key("__repeated");
828 out->BoolValue(true);
829 }
830 out->CloseObject();
831 }
832 }
833
PopulateAllowedFieldOptionsSet(const DescriptorPool & pool,const std::string & type,const std::string & field_prefix,protozero::ConstBytes protobytes,std::unordered_set<std::string> & allowed_fields)834 bool PopulateAllowedFieldOptionsSet(
835 const DescriptorPool& pool,
836 const std::string& type,
837 const std::string& field_prefix,
838 protozero::ConstBytes protobytes,
839 std::unordered_set<std::string>& allowed_fields) {
840 std::optional<uint32_t> opt_proto_desc_idx = pool.FindDescriptorIdx(type);
841 if (!opt_proto_desc_idx) {
842 return false;
843 }
844 const ProtoDescriptor& desc = pool.descriptors()[*opt_proto_desc_idx];
845 protozero::ProtoDecoder decoder(protobytes);
846 bool allowed = false;
847 for (auto field = decoder.ReadField(); field.valid();
848 field = decoder.ReadField()) {
849 auto* opt_field_descriptor = desc.FindFieldByTag(field.id());
850 if (!opt_field_descriptor) {
851 continue;
852 }
853 std::string full_field_name = field_prefix + opt_field_descriptor->name();
854 bool nested = false;
855 if (opt_field_descriptor->type() ==
856 protos::pbzero::FieldDescriptorProto::TYPE_MESSAGE) {
857 nested = PopulateAllowedFieldOptionsSet(
858 pool, opt_field_descriptor->resolved_type_name(),
859 full_field_name + ".", field.as_bytes(), allowed_fields);
860 }
861 if (nested || HasFieldOptions(*opt_field_descriptor)) {
862 allowed_fields.emplace(full_field_name);
863 allowed = true;
864 }
865 }
866 return allowed;
867 }
868
869 } // namespace
870
ProtozeroToJson(const DescriptorPool & pool,const std::string & type,protozero::ConstBytes protobytes,int flags)871 std::string ProtozeroToJson(const DescriptorPool& pool,
872 const std::string& type,
873 protozero::ConstBytes protobytes,
874 int flags) {
875 JsonBuilder builder(flags);
876 builder.OpenObject();
877 InnerMessageField(pool, type, protobytes, true, &builder);
878 if (builder.is_inline_errors() && !builder.errors().empty()) {
879 builder.Key("__error");
880 builder.StringValue(base::StringView(base::Join(builder.errors(), "\n")));
881 }
882 if (flags & kInlineAnnotations) {
883 std::unordered_set<std::string> allowed_fields;
884 PopulateAllowedFieldOptionsSet(pool, type, "", protobytes, allowed_fields);
885 if (!allowed_fields.empty()) {
886 builder.Key("__annotations");
887 builder.OpenObject();
888 MessageFieldOptionsToJson(pool, type, "", allowed_fields, &builder);
889 builder.CloseObject();
890 }
891 }
892 builder.CloseObject();
893 return builder.ToString();
894 }
895
ProtozeroToJson(const DescriptorPool & pool,const std::string & type,const std::vector<uint8_t> & protobytes,int flags)896 std::string ProtozeroToJson(const DescriptorPool& pool,
897 const std::string& type,
898 const std::vector<uint8_t>& protobytes,
899 int flags) {
900 return ProtozeroToJson(
901 pool, type, protozero::ConstBytes{protobytes.data(), protobytes.size()},
902 flags);
903 }
904
905 } // namespace perfetto::trace_processor::protozero_to_json
906