1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/importers/proto/proto_trace_parser_impl.h"
18
19 #include <string.h>
20
21 #include <cinttypes>
22 #include <string>
23
24 #include "perfetto/base/logging.h"
25 #include "perfetto/ext/base/metatrace_events.h"
26 #include "perfetto/ext/base/string_utils.h"
27 #include "perfetto/ext/base/string_view.h"
28 #include "perfetto/ext/base/string_writer.h"
29 #include "perfetto/ext/base/uuid.h"
30
31 #include "src/trace_processor/importers/common/args_tracker.h"
32 #include "src/trace_processor/importers/common/cpu_tracker.h"
33 #include "src/trace_processor/importers/common/event_tracker.h"
34 #include "src/trace_processor/importers/common/metadata_tracker.h"
35 #include "src/trace_processor/importers/common/parser_types.h"
36 #include "src/trace_processor/importers/common/process_tracker.h"
37 #include "src/trace_processor/importers/common/slice_tracker.h"
38 #include "src/trace_processor/importers/common/track_tracker.h"
39 #include "src/trace_processor/importers/etw/etw_module.h"
40 #include "src/trace_processor/importers/ftrace/ftrace_module.h"
41 #include "src/trace_processor/importers/proto/track_event_module.h"
42 #include "src/trace_processor/storage/metadata.h"
43 #include "src/trace_processor/storage/stats.h"
44 #include "src/trace_processor/types/trace_processor_context.h"
45 #include "src/trace_processor/types/variadic.h"
46
47 #include "protos/perfetto/config/trace_config.pbzero.h"
48 #include "protos/perfetto/trace/chrome/chrome_trace_event.pbzero.h"
49 #include "protos/perfetto/trace/perfetto/perfetto_metatrace.pbzero.h"
50 #include "protos/perfetto/trace/trace_packet.pbzero.h"
51
52 namespace perfetto {
53 namespace trace_processor {
54
ProtoTraceParserImpl(TraceProcessorContext * context)55 ProtoTraceParserImpl::ProtoTraceParserImpl(TraceProcessorContext* context)
56 : context_(context),
57 metatrace_id_(context->storage->InternString("metatrace")),
58 data_name_id_(context->storage->InternString("data")),
59 raw_chrome_metadata_event_id_(
60 context->storage->InternString("chrome_event.metadata")),
61 raw_chrome_legacy_system_trace_event_id_(
62 context->storage->InternString("chrome_event.legacy_system_trace")),
63 raw_chrome_legacy_user_trace_event_id_(
64 context->storage->InternString("chrome_event.legacy_user_trace")),
65 missing_metatrace_interned_string_id_(
66 context->storage->InternString("MISSING STRING")) {}
67
68 ProtoTraceParserImpl::~ProtoTraceParserImpl() = default;
69
ParseTracePacket(int64_t ts,TracePacketData data)70 void ProtoTraceParserImpl::ParseTracePacket(int64_t ts, TracePacketData data) {
71 const TraceBlobView& blob = data.packet;
72 protos::pbzero::TracePacket::Decoder packet(blob.data(), blob.length());
73 // TODO(eseckler): Propagate statuses from modules.
74 auto& modules = context_->modules_by_field;
75 for (uint32_t field_id = 1; field_id < modules.size(); ++field_id) {
76 if (!modules[field_id].empty() && packet.Get(field_id).valid()) {
77 for (ProtoImporterModule* global_module :
78 context_->modules_for_all_fields) {
79 global_module->ParseTracePacketData(packet, ts, data, field_id);
80 }
81 for (ProtoImporterModule* module : modules[field_id])
82 module->ParseTracePacketData(packet, ts, data, field_id);
83 return;
84 }
85 }
86
87 if (packet.has_chrome_events()) {
88 ParseChromeEvents(ts, packet.chrome_events());
89 }
90
91 if (packet.has_perfetto_metatrace()) {
92 ParseMetatraceEvent(ts, packet.perfetto_metatrace());
93 }
94
95 if (packet.has_trace_config()) {
96 // TODO(eseckler): Propagate statuses from modules.
97 protos::pbzero::TraceConfig::Decoder config(packet.trace_config());
98 for (auto& module : context_->modules) {
99 module->ParseTraceConfig(config);
100 }
101 }
102 }
103
ParseTrackEvent(int64_t ts,TrackEventData data)104 void ProtoTraceParserImpl::ParseTrackEvent(int64_t ts, TrackEventData data) {
105 const TraceBlobView& blob = data.trace_packet_data.packet;
106 protos::pbzero::TracePacket::Decoder packet(blob.data(), blob.length());
107 context_->track_module->ParseTrackEventData(packet, ts, data);
108 context_->args_tracker->Flush();
109 }
110
ParseEtwEvent(uint32_t cpu,int64_t ts,TracePacketData data)111 void ProtoTraceParserImpl::ParseEtwEvent(uint32_t cpu,
112 int64_t ts,
113 TracePacketData data) {
114 PERFETTO_DCHECK(context_->etw_module);
115 context_->etw_module->ParseEtwEventData(cpu, ts, data);
116
117 // TODO(lalitm): maybe move this to the flush method in the trace processor
118 // once we have it. This may reduce performance in the ArgsTracker though so
119 // needs to be handled carefully.
120 context_->args_tracker->Flush();
121 }
122
ParseFtraceEvent(uint32_t cpu,int64_t ts,TracePacketData data)123 void ProtoTraceParserImpl::ParseFtraceEvent(uint32_t cpu,
124 int64_t ts,
125 TracePacketData data) {
126 PERFETTO_DCHECK(context_->ftrace_module);
127 context_->ftrace_module->ParseFtraceEventData(cpu, ts, data);
128
129 // TODO(lalitm): maybe move this to the flush method in the trace processor
130 // once we have it. This may reduce performance in the ArgsTracker though so
131 // needs to be handled carefully.
132 context_->args_tracker->Flush();
133 }
134
ParseInlineSchedSwitch(uint32_t cpu,int64_t ts,InlineSchedSwitch data)135 void ProtoTraceParserImpl::ParseInlineSchedSwitch(uint32_t cpu,
136 int64_t ts,
137 InlineSchedSwitch data) {
138 PERFETTO_DCHECK(context_->ftrace_module);
139 context_->ftrace_module->ParseInlineSchedSwitch(cpu, ts, data);
140
141 // TODO(lalitm): maybe move this to the flush method in the trace processor
142 // once we have it. This may reduce performance in the ArgsTracker though so
143 // needs to be handled carefully.
144 context_->args_tracker->Flush();
145 }
146
ParseInlineSchedWaking(uint32_t cpu,int64_t ts,InlineSchedWaking data)147 void ProtoTraceParserImpl::ParseInlineSchedWaking(uint32_t cpu,
148 int64_t ts,
149 InlineSchedWaking data) {
150 PERFETTO_DCHECK(context_->ftrace_module);
151 context_->ftrace_module->ParseInlineSchedWaking(cpu, ts, data);
152
153 // TODO(lalitm): maybe move this to the flush method in the trace processor
154 // once we have it. This may reduce performance in the ArgsTracker though so
155 // needs to be handled carefully.
156 context_->args_tracker->Flush();
157 }
158
ParseChromeEvents(int64_t ts,ConstBytes blob)159 void ProtoTraceParserImpl::ParseChromeEvents(int64_t ts, ConstBytes blob) {
160 TraceStorage* storage = context_->storage.get();
161 protos::pbzero::ChromeEventBundle::Decoder bundle(blob.data, blob.size);
162 ArgsTracker args(context_);
163 if (bundle.has_metadata()) {
164 auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0);
165 RawId id = storage->mutable_raw_table()
166 ->Insert({ts, raw_chrome_metadata_event_id_, 0, 0, 0, ucpu})
167 .id;
168 auto inserter = args.AddArgsTo(id);
169
170 uint32_t bundle_index =
171 context_->metadata_tracker->IncrementChromeMetadataBundleCount();
172
173 // The legacy untyped metadata is proxied via a special event in the raw
174 // table to JSON export.
175 for (auto it = bundle.metadata(); it; ++it) {
176 protos::pbzero::ChromeMetadata::Decoder metadata(*it);
177 Variadic value;
178 if (metadata.has_string_value()) {
179 value =
180 Variadic::String(storage->InternString(metadata.string_value()));
181 } else if (metadata.has_int_value()) {
182 value = Variadic::Integer(metadata.int_value());
183 } else if (metadata.has_bool_value()) {
184 value = Variadic::Integer(metadata.bool_value());
185 } else if (metadata.has_json_value()) {
186 value = Variadic::Json(storage->InternString(metadata.json_value()));
187 } else {
188 context_->storage->IncrementStats(stats::empty_chrome_metadata);
189 continue;
190 }
191
192 StringId name_id = storage->InternString(metadata.name());
193 args.AddArgsTo(id).AddArg(name_id, value);
194
195 char buffer[2048];
196 base::StringWriter writer(buffer, sizeof(buffer));
197 writer.AppendString("cr-");
198 // If we have data from multiple Chrome instances, append a suffix
199 // to differentiate them.
200 if (bundle_index > 1) {
201 writer.AppendUnsignedInt(bundle_index);
202 writer.AppendChar('-');
203 }
204 writer.AppendString(metadata.name());
205
206 auto metadata_id = storage->InternString(writer.GetStringView());
207 context_->metadata_tracker->SetDynamicMetadata(metadata_id, value);
208 }
209 }
210
211 if (bundle.has_legacy_ftrace_output()) {
212 auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0);
213 RawId id = storage->mutable_raw_table()
214 ->Insert({ts, raw_chrome_legacy_system_trace_event_id_, 0, 0,
215 0, ucpu})
216 .id;
217
218 std::string data;
219 for (auto it = bundle.legacy_ftrace_output(); it; ++it) {
220 data += (*it).ToStdString();
221 }
222 Variadic value =
223 Variadic::String(storage->InternString(base::StringView(data)));
224 args.AddArgsTo(id).AddArg(data_name_id_, value);
225 }
226
227 if (bundle.has_legacy_json_trace()) {
228 for (auto it = bundle.legacy_json_trace(); it; ++it) {
229 protos::pbzero::ChromeLegacyJsonTrace::Decoder legacy_trace(*it);
230 if (legacy_trace.type() !=
231 protos::pbzero::ChromeLegacyJsonTrace::USER_TRACE) {
232 continue;
233 }
234 auto ucpu = context_->cpu_tracker->GetOrCreateCpu(0);
235 RawId id = storage->mutable_raw_table()
236 ->Insert({ts, raw_chrome_legacy_user_trace_event_id_, 0, 0,
237 0, ucpu})
238 .id;
239 Variadic value =
240 Variadic::String(storage->InternString(legacy_trace.data()));
241 args.AddArgsTo(id).AddArg(data_name_id_, value);
242 }
243 }
244 }
245
ParseMetatraceEvent(int64_t ts,ConstBytes blob)246 void ProtoTraceParserImpl::ParseMetatraceEvent(int64_t ts, ConstBytes blob) {
247 protos::pbzero::PerfettoMetatrace::Decoder event(blob.data, blob.size);
248 auto utid = context_->process_tracker->GetOrCreateThread(event.thread_id());
249
250 StringId cat_id = metatrace_id_;
251 StringId name_id = kNullStringId;
252
253 for (auto it = event.interned_strings(); it; ++it) {
254 protos::pbzero::PerfettoMetatrace::InternedString::Decoder interned_string(
255 it->data(), it->size());
256 metatrace_interned_strings_.Insert(
257 interned_string.iid(),
258 context_->storage->InternString(interned_string.value()));
259 }
260
261 // This function inserts the args from the proto into the args table.
262 // Args inserted with the same key multiple times are treated as an array:
263 // this function correctly creates the key and flat key for each arg array.
264 auto args_fn = [this, &event](ArgsTracker::BoundInserter* inserter) {
265 using Arg = std::pair<StringId, StringId>;
266
267 // First, get a list of all the args so we can group them by key.
268 std::vector<Arg> interned;
269 for (auto it = event.args(); it; ++it) {
270 protos::pbzero::PerfettoMetatrace::Arg::Decoder arg_proto(*it);
271 StringId key;
272 if (arg_proto.has_key_iid()) {
273 key = GetMetatraceInternedString(arg_proto.key_iid());
274 } else {
275 key = context_->storage->InternString(arg_proto.key());
276 }
277 StringId value;
278 if (arg_proto.has_value_iid()) {
279 value = GetMetatraceInternedString(arg_proto.value_iid());
280 } else {
281 value = context_->storage->InternString(arg_proto.value());
282 }
283 interned.emplace_back(key, value);
284 }
285
286 // We stable sort insted of sorting here to avoid changing the order of the
287 // args in arrays.
288 std::stable_sort(interned.begin(), interned.end(),
289 [](const Arg& a, const Arg& b) {
290 return a.first.raw_id() < b.first.raw_id();
291 });
292
293 // Compute the correct key for each arg, possibly adding an index to
294 // the end of the key if needed.
295 char buffer[2048];
296 uint32_t current_idx = 0;
297 for (auto it = interned.begin(); it != interned.end(); ++it) {
298 auto next = it + 1;
299 StringId key = it->first;
300 StringId next_key = next == interned.end() ? kNullStringId : next->first;
301
302 if (key != next_key && current_idx == 0) {
303 inserter->AddArg(key, Variadic::String(it->second));
304 } else {
305 constexpr size_t kMaxIndexSize = 20;
306 NullTermStringView key_str = context_->storage->GetString(key);
307 if (key_str.size() >= sizeof(buffer) - kMaxIndexSize) {
308 PERFETTO_DLOG("Ignoring arg with unreasonbly large size");
309 continue;
310 }
311
312 base::StackString<2048> array_key("%s[%u]", key_str.c_str(),
313 current_idx);
314 StringId new_key =
315 context_->storage->InternString(array_key.string_view());
316 inserter->AddArg(key, new_key, Variadic::String(it->second));
317
318 current_idx = key == next_key ? current_idx + 1 : 0;
319 }
320 }
321 };
322
323 if (event.has_event_id() || event.has_event_name() ||
324 event.has_event_name_iid()) {
325 if (event.has_event_id()) {
326 auto eid = event.event_id();
327 if (eid < metatrace::EVENTS_MAX) {
328 name_id = context_->storage->InternString(metatrace::kEventNames[eid]);
329 } else {
330 base::StackString<64> fallback("Event %d", eid);
331 name_id = context_->storage->InternString(fallback.string_view());
332 }
333 } else if (event.has_event_name_iid()) {
334 name_id = GetMetatraceInternedString(event.event_name_iid());
335 } else {
336 name_id = context_->storage->InternString(event.event_name());
337 }
338 TrackId track_id = context_->track_tracker->InternThreadTrack(utid);
339 context_->slice_tracker->Scoped(
340 ts, track_id, cat_id, name_id,
341 static_cast<int64_t>(event.event_duration_ns()), args_fn);
342 } else if (event.has_counter_id() || event.has_counter_name()) {
343 if (event.has_counter_id()) {
344 auto cid = event.counter_id();
345 if (cid < metatrace::COUNTERS_MAX) {
346 name_id =
347 context_->storage->InternString(metatrace::kCounterNames[cid]);
348 } else {
349 base::StackString<64> fallback("Counter %d", cid);
350 name_id = context_->storage->InternString(fallback.string_view());
351 }
352 } else {
353 name_id = context_->storage->InternString(event.counter_name());
354 }
355 TrackId track =
356 context_->track_tracker->InternThreadCounterTrack(name_id, utid);
357 auto opt_id =
358 context_->event_tracker->PushCounter(ts, event.counter_value(), track);
359 if (opt_id) {
360 auto inserter = context_->args_tracker->AddArgsTo(*opt_id);
361 args_fn(&inserter);
362 }
363 }
364
365 if (event.has_overruns())
366 context_->storage->IncrementStats(stats::metatrace_overruns);
367 }
368
GetMetatraceInternedString(uint64_t iid)369 StringId ProtoTraceParserImpl::GetMetatraceInternedString(uint64_t iid) {
370 StringId* maybe_id = metatrace_interned_strings_.Find(iid);
371 if (!maybe_id)
372 return missing_metatrace_interned_string_id_;
373 return *maybe_id;
374 }
375
376 } // namespace trace_processor
377 } // namespace perfetto
378