1 /*
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "src/trace_processor/importers/proto/statsd_module.h"
17
18 #include <cstddef>
19 #include <cstdint>
20 #include <optional>
21 #include <string>
22 #include <utility>
23 #include <vector>
24
25 #include "perfetto/base/logging.h"
26 #include "perfetto/base/status.h"
27 #include "perfetto/ext/base/string_utils.h"
28 #include "perfetto/ext/base/string_view.h"
29 #include "perfetto/protozero/field.h"
30 #include "perfetto/protozero/proto_decoder.h"
31 #include "perfetto/protozero/proto_utils.h"
32 #include "perfetto/protozero/scattered_heap_buffer.h"
33 #include "perfetto/trace_processor/ref_counted.h"
34 #include "perfetto/trace_processor/trace_blob.h"
35 #include "protos/perfetto/trace/statsd/statsd_atom.pbzero.h"
36 #include "protos/perfetto/trace/trace_packet.pbzero.h"
37 #include "src/trace_processor/importers/common/args_tracker.h"
38 #include "src/trace_processor/importers/common/parser_types.h"
39 #include "src/trace_processor/importers/common/slice_tracker.h"
40 #include "src/trace_processor/importers/common/track_tracker.h"
41 #include "src/trace_processor/importers/common/tracks.h"
42 #include "src/trace_processor/importers/proto/args_parser.h"
43 #include "src/trace_processor/importers/proto/atoms.descriptor.h"
44 #include "src/trace_processor/importers/proto/packet_sequence_state_generation.h"
45 #include "src/trace_processor/importers/proto/proto_importer_module.h"
46 #include "src/trace_processor/sorter/trace_sorter.h"
47 #include "src/trace_processor/storage/stats.h"
48 #include "src/trace_processor/storage/trace_storage.h"
49 #include "src/trace_processor/util/descriptors.h"
50 #include "src/trace_processor/util/proto_to_args_parser.h"
51
52 namespace perfetto::trace_processor {
53 namespace {
54
55 constexpr const char* kAtomProtoName = ".android.os.statsd.Atom";
56
57 // If we don't know about the atom format put whatever details we
58 // can. This has the following restrictions:
59 // - We can't tell the difference between double, fixed64, sfixed64
60 // so those all show up as double
61 // - We can't tell the difference between float, fixed32, sfixed32
62 // so those all show up as float
63 // - We can't tell the difference between int32, int64 and sint32
64 // and sint64. We assume int32/int64.
65 // - We only show the length of strings, nested messages, packed ints
66 // and any other length delimited fields.
ParseGenericEvent(const protozero::ConstBytes & cb,util::ProtoToArgsParser::Delegate & delegate)67 base::Status ParseGenericEvent(const protozero::ConstBytes& cb,
68 util::ProtoToArgsParser::Delegate& delegate) {
69 protozero::ProtoDecoder decoder(cb);
70 for (auto f = decoder.ReadField(); f.valid(); f = decoder.ReadField()) {
71 switch (f.type()) {
72 case protozero::proto_utils::ProtoWireType::kLengthDelimited: {
73 base::StackString<64> name("field_%u", f.id());
74 std::string name_str = name.ToStdString();
75 util::ProtoToArgsParser::Key key{name_str, name_str};
76 delegate.AddBytes(key, f.as_bytes());
77 break;
78 }
79 case protozero::proto_utils::ProtoWireType::kVarInt: {
80 base::StackString<64> name("field_%u", f.id());
81 std::string name_str = name.ToStdString();
82 util::ProtoToArgsParser::Key key{name_str, name_str};
83 delegate.AddInteger(key, f.as_int64());
84 break;
85 }
86 case protozero::proto_utils::ProtoWireType::kFixed32: {
87 base::StackString<64> name("field_%u_assuming_float", f.id());
88 std::string name_str = name.ToStdString();
89 util::ProtoToArgsParser::Key key{name_str, name_str};
90 delegate.AddDouble(key, static_cast<double>(f.as_float()));
91 break;
92 }
93 case protozero::proto_utils::ProtoWireType::kFixed64: {
94 base::StackString<64> name("field_%u_assuming_double", f.id());
95 std::string name_str = name.ToStdString();
96 util::ProtoToArgsParser::Key key{name_str, name_str};
97 delegate.AddDouble(key, f.as_double());
98 break;
99 }
100 }
101 }
102 return base::OkStatus();
103 }
104
105 } // namespace
106
107 using perfetto::protos::pbzero::StatsdAtom;
108 using perfetto::protos::pbzero::TracePacket;
109
StatsdModule(TraceProcessorContext * context)110 StatsdModule::StatsdModule(TraceProcessorContext* context)
111 : context_(context), args_parser_(*context_->descriptor_pool_) {
112 RegisterForField(TracePacket::kStatsdAtomFieldNumber, context);
113 context_->descriptor_pool_->AddFromFileDescriptorSet(kAtomsDescriptor.data(),
114 kAtomsDescriptor.size());
115
116 std::optional<uint32_t> opt_idx =
117 context_->descriptor_pool_->FindDescriptorIdx(kAtomProtoName);
118 if (opt_idx.has_value()) {
119 descriptor_ = &context_->descriptor_pool_->descriptors()[opt_idx.value()];
120 }
121 }
122
123 StatsdModule::~StatsdModule() = default;
124
TokenizePacket(const TracePacket::Decoder & decoder,TraceBlobView *,int64_t packet_timestamp,RefPtr<PacketSequenceStateGeneration> state,uint32_t field_id)125 ModuleResult StatsdModule::TokenizePacket(
126 const TracePacket::Decoder& decoder,
127 TraceBlobView* /*packet*/,
128 int64_t packet_timestamp,
129 RefPtr<PacketSequenceStateGeneration> state,
130 uint32_t field_id) {
131 if (field_id != TracePacket::kStatsdAtomFieldNumber) {
132 return ModuleResult::Ignored();
133 }
134 const auto& atoms_wrapper = StatsdAtom::Decoder(decoder.statsd_atom());
135 auto it_timestamps = atoms_wrapper.timestamp_nanos();
136 for (auto it = atoms_wrapper.atom(); it; ++it) {
137 int64_t atom_timestamp;
138
139 if (it_timestamps) {
140 atom_timestamp = *it_timestamps++;
141 } else {
142 context_->storage->IncrementStats(stats::atom_timestamp_missing);
143 atom_timestamp = packet_timestamp;
144 }
145
146 protozero::HeapBuffered<TracePacket> forged;
147
148 forged->set_timestamp(static_cast<uint64_t>(atom_timestamp));
149
150 auto* statsd = forged->set_statsd_atom();
151 statsd->AppendBytes(StatsdAtom::kAtomFieldNumber, (*it).data, (*it).size);
152
153 std::vector<uint8_t> vec = forged.SerializeAsArray();
154 TraceBlob blob = TraceBlob::CopyFrom(vec.data(), vec.size());
155
156 context_->sorter->PushTracePacket(atom_timestamp, state,
157 TraceBlobView(std::move(blob)),
158 context_->machine_id());
159 }
160
161 return ModuleResult::Handled();
162 }
163
ParseTracePacketData(const TracePacket::Decoder & decoder,int64_t ts,const TracePacketData &,uint32_t field_id)164 void StatsdModule::ParseTracePacketData(const TracePacket::Decoder& decoder,
165 int64_t ts,
166 const TracePacketData&,
167 uint32_t field_id) {
168 if (field_id != TracePacket::kStatsdAtomFieldNumber) {
169 return;
170 }
171 const auto& atoms_wrapper = StatsdAtom::Decoder(decoder.statsd_atom());
172 auto it = atoms_wrapper.atom();
173 // There should be exactly one atom per trace packet at this point.
174 // If not something has gone wrong in tokenization above.
175 PERFETTO_CHECK(it);
176 ParseAtom(ts, *it++);
177 PERFETTO_CHECK(!it);
178 }
179
ParseAtom(int64_t ts,protozero::ConstBytes nested_bytes)180 void StatsdModule::ParseAtom(int64_t ts, protozero::ConstBytes nested_bytes) {
181 // nested_bytes is an Atom proto. We (deliberately) don't generate
182 // decoding code for every kind of atom (or the parent Atom proto)
183 // and instead use the descriptor to parse the args/name.
184
185 // Atom is a giant oneof of all the possible 'kinds' of atom so here
186 // we use the protozero decoder implementation to grab the first
187 // field id which we we use to look up the field name:
188 protozero::ProtoDecoder nested_decoder(nested_bytes);
189 protozero::Field field = nested_decoder.ReadField();
190 uint32_t nested_field_id = 0;
191 if (field.valid()) {
192 nested_field_id = field.id();
193 }
194 StringId atom_name = GetAtomName(nested_field_id);
195 context_->slice_tracker->Scoped(
196 ts, InternTrackId(), kNullStringId, atom_name, 0,
197 [&](ArgsTracker::BoundInserter* inserter) {
198 ArgsParser delegate(ts, *inserter, *context_->storage);
199
200 const auto& fields = descriptor_->fields();
201 const auto& field_it = fields.find(nested_field_id);
202 base::Status status;
203
204 if (field_it == fields.end()) {
205 /// Field ids 100000 and over are OEM atoms - we can't have the
206 // descriptor for them so don't report errors. See:
207 // https://cs.android.com/android/platform/superproject/main/+/main:frameworks/proto_logging/stats/atoms.proto;l=1290;drc=a34b11bfebe897259a0340a59f1793ae2dffd762
208 if (nested_field_id < 100000) {
209 context_->storage->IncrementStats(stats::atom_unknown);
210 }
211
212 status = ParseGenericEvent(field.as_bytes(), delegate);
213 } else {
214 status = args_parser_.ParseMessage(nested_bytes, kAtomProtoName,
215 nullptr /* parse all fields */,
216 delegate);
217 }
218
219 if (!status.ok()) {
220 context_->storage->IncrementStats(stats::atom_unknown);
221 }
222 });
223 }
224
GetAtomName(uint32_t atom_field_id)225 StringId StatsdModule::GetAtomName(uint32_t atom_field_id) {
226 StringId* cached_name = atom_names_.Find(atom_field_id);
227 if (cached_name == nullptr) {
228 if (descriptor_ == nullptr) {
229 context_->storage->IncrementStats(stats::atom_unknown);
230 return context_->storage->InternString("Could not load atom descriptor");
231 }
232
233 StringId name_id;
234 const auto& fields = descriptor_->fields();
235 const auto& field_it = fields.find(atom_field_id);
236 if (field_it == fields.end()) {
237 base::StackString<255> name("atom_%u", atom_field_id);
238 name_id = context_->storage->InternString(name.string_view());
239 } else {
240 const FieldDescriptor& field = field_it->second;
241 name_id = context_->storage->InternString(base::StringView(field.name()));
242 }
243 atom_names_[atom_field_id] = name_id;
244 return name_id;
245 }
246 return *cached_name;
247 }
248
InternTrackId()249 TrackId StatsdModule::InternTrackId() {
250 if (!track_id_) {
251 static constexpr auto kBlueprint =
252 tracks::SliceBlueprint("statsd_atoms", tracks::DimensionBlueprints(),
253 tracks::StaticNameBlueprint("Statsd Atoms"));
254 track_id_ = context_->track_tracker->InternTrack(kBlueprint);
255 }
256 return *track_id_;
257 }
258
259 } // namespace perfetto::trace_processor
260