1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/importers/proto/proto_trace_parser_impl.h"
18
19 #include <algorithm>
20 #include <cstdint>
21 #include <cstring>
22 #include <string>
23 #include <utility>
24 #include <vector>
25
26 #include "perfetto/base/logging.h"
27 #include "perfetto/ext/base/metatrace_events.h"
28 #include "perfetto/ext/base/string_utils.h"
29 #include "perfetto/ext/base/string_view.h"
30 #include "perfetto/ext/base/string_writer.h"
31 #include "perfetto/trace_processor/trace_blob_view.h"
32 #include "src/trace_processor/containers/null_term_string_view.h"
33 #include "src/trace_processor/importers/common/args_tracker.h"
34 #include "src/trace_processor/importers/common/cpu_tracker.h"
35 #include "src/trace_processor/importers/common/event_tracker.h"
36 #include "src/trace_processor/importers/common/metadata_tracker.h"
37 #include "src/trace_processor/importers/common/parser_types.h"
38 #include "src/trace_processor/importers/common/process_tracker.h"
39 #include "src/trace_processor/importers/common/slice_tracker.h"
40 #include "src/trace_processor/importers/common/track_tracker.h"
41 #include "src/trace_processor/importers/common/tracks.h"
42 #include "src/trace_processor/importers/common/tracks_common.h"
43 #include "src/trace_processor/importers/etw/etw_module.h"
44 #include "src/trace_processor/importers/ftrace/ftrace_module.h"
45 #include "src/trace_processor/importers/proto/track_event_module.h"
46 #include "src/trace_processor/storage/stats.h"
47 #include "src/trace_processor/storage/trace_storage.h"
48 #include "src/trace_processor/tables/metadata_tables_py.h"
49 #include "src/trace_processor/types/trace_processor_context.h"
50 #include "src/trace_processor/types/variadic.h"
51
52 #include "protos/perfetto/config/trace_config.pbzero.h"
53 #include "protos/perfetto/trace/chrome/chrome_trace_event.pbzero.h"
54 #include "protos/perfetto/trace/perfetto/perfetto_metatrace.pbzero.h"
55 #include "protos/perfetto/trace/trace_packet.pbzero.h"
56
57 namespace perfetto::trace_processor {
58
ProtoTraceParserImpl(TraceProcessorContext * context)59 ProtoTraceParserImpl::ProtoTraceParserImpl(TraceProcessorContext* context)
60 : context_(context),
61 metatrace_id_(context->storage->InternString("metatrace")),
62 data_name_id_(context->storage->InternString("data")),
63 raw_chrome_metadata_event_id_(
64 context->storage->InternString("chrome_event.metadata")),
65 raw_chrome_legacy_system_trace_event_id_(
66 context->storage->InternString("chrome_event.legacy_system_trace")),
67 raw_chrome_legacy_user_trace_event_id_(
68 context->storage->InternString("chrome_event.legacy_user_trace")),
69 missing_metatrace_interned_string_id_(
70 context->storage->InternString("MISSING STRING")) {}
71
72 ProtoTraceParserImpl::~ProtoTraceParserImpl() = default;
73
ParseTracePacket(int64_t ts,TracePacketData data)74 void ProtoTraceParserImpl::ParseTracePacket(int64_t ts, TracePacketData data) {
75 const TraceBlobView& blob = data.packet;
76 protos::pbzero::TracePacket::Decoder packet(blob.data(), blob.length());
77 // TODO(eseckler): Propagate statuses from modules.
78 auto& modules = context_->modules_by_field;
79 for (uint32_t field_id = 1; field_id < modules.size(); ++field_id) {
80 if (!modules[field_id].empty() && packet.Get(field_id).valid()) {
81 for (ProtoImporterModule* global_module :
82 context_->modules_for_all_fields) {
83 global_module->ParseTracePacketData(packet, ts, data, field_id);
84 }
85 for (ProtoImporterModule* module : modules[field_id])
86 module->ParseTracePacketData(packet, ts, data, field_id);
87 return;
88 }
89 }
90
91 if (packet.has_chrome_events()) {
92 ParseChromeEvents(ts, packet.chrome_events());
93 }
94
95 if (packet.has_perfetto_metatrace()) {
96 ParseMetatraceEvent(ts, packet.perfetto_metatrace());
97 }
98
99 if (packet.has_trace_config()) {
100 // TODO(eseckler): Propagate statuses from modules.
101 protos::pbzero::TraceConfig::Decoder config(packet.trace_config());
102 for (auto& module : context_->modules) {
103 module->ParseTraceConfig(config);
104 }
105 }
106 }
107
ParseTrackEvent(int64_t ts,TrackEventData data)108 void ProtoTraceParserImpl::ParseTrackEvent(int64_t ts, TrackEventData data) {
109 const TraceBlobView& blob = data.trace_packet_data.packet;
110 protos::pbzero::TracePacket::Decoder packet(blob.data(), blob.length());
111 context_->track_module->ParseTrackEventData(packet, ts, data);
112 context_->args_tracker->Flush();
113 }
114
ParseEtwEvent(uint32_t cpu,int64_t ts,TracePacketData data)115 void ProtoTraceParserImpl::ParseEtwEvent(uint32_t cpu,
116 int64_t ts,
117 TracePacketData data) {
118 PERFETTO_DCHECK(context_->etw_module);
119 context_->etw_module->ParseEtwEventData(cpu, ts, data);
120
121 // TODO(lalitm): maybe move this to the flush method in the trace processor
122 // once we have it. This may reduce performance in the ArgsTracker though so
123 // needs to be handled carefully.
124 context_->args_tracker->Flush();
125 }
126
ParseFtraceEvent(uint32_t cpu,int64_t ts,TracePacketData data)127 void ProtoTraceParserImpl::ParseFtraceEvent(uint32_t cpu,
128 int64_t ts,
129 TracePacketData data) {
130 PERFETTO_DCHECK(context_->ftrace_module);
131 context_->ftrace_module->ParseFtraceEventData(cpu, ts, data);
132
133 // TODO(lalitm): maybe move this to the flush method in the trace processor
134 // once we have it. This may reduce performance in the ArgsTracker though so
135 // needs to be handled carefully.
136 context_->args_tracker->Flush();
137 }
138
ParseInlineSchedSwitch(uint32_t cpu,int64_t ts,InlineSchedSwitch data)139 void ProtoTraceParserImpl::ParseInlineSchedSwitch(uint32_t cpu,
140 int64_t ts,
141 InlineSchedSwitch data) {
142 PERFETTO_DCHECK(context_->ftrace_module);
143 context_->ftrace_module->ParseInlineSchedSwitch(cpu, ts, data);
144
145 // TODO(lalitm): maybe move this to the flush method in the trace processor
146 // once we have it. This may reduce performance in the ArgsTracker though so
147 // needs to be handled carefully.
148 context_->args_tracker->Flush();
149 }
150
ParseInlineSchedWaking(uint32_t cpu,int64_t ts,InlineSchedWaking data)151 void ProtoTraceParserImpl::ParseInlineSchedWaking(uint32_t cpu,
152 int64_t ts,
153 InlineSchedWaking data) {
154 PERFETTO_DCHECK(context_->ftrace_module);
155 context_->ftrace_module->ParseInlineSchedWaking(cpu, ts, data);
156
157 // TODO(lalitm): maybe move this to the flush method in the trace processor
158 // once we have it. This may reduce performance in the ArgsTracker though so
159 // needs to be handled carefully.
160 context_->args_tracker->Flush();
161 }
162
ParseChromeEvents(int64_t ts,ConstBytes blob)163 void ProtoTraceParserImpl::ParseChromeEvents(int64_t ts, ConstBytes blob) {
164 TraceStorage* storage = context_->storage.get();
165 protos::pbzero::ChromeEventBundle::Decoder bundle(blob);
166 ArgsTracker args(context_);
167 if (bundle.has_metadata()) {
168 tables::ChromeRawTable::Id id =
169 storage->mutable_chrome_raw_table()
170 ->Insert({ts, raw_chrome_metadata_event_id_, 0, 0})
171 .id;
172 auto inserter = args.AddArgsTo(id);
173
174 uint32_t bundle_index =
175 context_->metadata_tracker->IncrementChromeMetadataBundleCount();
176
177 // The legacy untyped metadata is proxied via a special event in the raw
178 // table to JSON export.
179 for (auto it = bundle.metadata(); it; ++it) {
180 protos::pbzero::ChromeMetadata::Decoder metadata(*it);
181 Variadic value = Variadic::Null();
182 if (metadata.has_string_value()) {
183 value =
184 Variadic::String(storage->InternString(metadata.string_value()));
185 } else if (metadata.has_int_value()) {
186 value = Variadic::Integer(metadata.int_value());
187 } else if (metadata.has_bool_value()) {
188 value = Variadic::Integer(metadata.bool_value());
189 } else if (metadata.has_json_value()) {
190 value = Variadic::Json(storage->InternString(metadata.json_value()));
191 } else {
192 context_->storage->IncrementStats(stats::empty_chrome_metadata);
193 continue;
194 }
195
196 StringId name_id = storage->InternString(metadata.name());
197 args.AddArgsTo(id).AddArg(name_id, value);
198
199 char buffer[2048];
200 base::StringWriter writer(buffer, sizeof(buffer));
201 writer.AppendString("cr-");
202 // If we have data from multiple Chrome instances, append a suffix
203 // to differentiate them.
204 if (bundle_index > 1) {
205 writer.AppendUnsignedInt(bundle_index);
206 writer.AppendChar('-');
207 }
208 writer.AppendString(metadata.name());
209
210 auto metadata_id = storage->InternString(writer.GetStringView());
211 context_->metadata_tracker->SetDynamicMetadata(metadata_id, value);
212 }
213 }
214
215 if (bundle.has_legacy_ftrace_output()) {
216 tables::ChromeRawTable::Id id =
217 storage->mutable_chrome_raw_table()
218 ->Insert({ts, raw_chrome_legacy_system_trace_event_id_, 0, 0})
219 .id;
220
221 std::string data;
222 for (auto it = bundle.legacy_ftrace_output(); it; ++it) {
223 data += (*it).ToStdString();
224 }
225 Variadic value =
226 Variadic::String(storage->InternString(base::StringView(data)));
227 args.AddArgsTo(id).AddArg(data_name_id_, value);
228 }
229
230 if (bundle.has_legacy_json_trace()) {
231 for (auto it = bundle.legacy_json_trace(); it; ++it) {
232 protos::pbzero::ChromeLegacyJsonTrace::Decoder legacy_trace(*it);
233 if (legacy_trace.type() !=
234 protos::pbzero::ChromeLegacyJsonTrace::USER_TRACE) {
235 continue;
236 }
237 tables::ChromeRawTable::Id id =
238 storage->mutable_chrome_raw_table()
239 ->Insert({ts, raw_chrome_legacy_user_trace_event_id_, 0, 0})
240 .id;
241 Variadic value =
242 Variadic::String(storage->InternString(legacy_trace.data()));
243 args.AddArgsTo(id).AddArg(data_name_id_, value);
244 }
245 }
246 }
247
ParseMetatraceEvent(int64_t ts,ConstBytes blob)248 void ProtoTraceParserImpl::ParseMetatraceEvent(int64_t ts, ConstBytes blob) {
249 protos::pbzero::PerfettoMetatrace::Decoder event(blob);
250 auto utid = context_->process_tracker->GetOrCreateThread(event.thread_id());
251
252 StringId cat_id = metatrace_id_;
253 for (auto it = event.interned_strings(); it; ++it) {
254 protos::pbzero::PerfettoMetatrace::InternedString::Decoder interned_string(
255 it->data(), it->size());
256 metatrace_interned_strings_.Insert(
257 interned_string.iid(),
258 context_->storage->InternString(interned_string.value()));
259 }
260
261 // This function inserts the args from the proto into the args table.
262 // Args inserted with the same key multiple times are treated as an array:
263 // this function correctly creates the key and flat key for each arg array.
264 auto args_fn = [this, &event](ArgsTracker::BoundInserter* inserter) {
265 using Arg = std::pair<StringId, StringId>;
266
267 // First, get a list of all the args so we can group them by key.
268 std::vector<Arg> interned;
269 for (auto it = event.args(); it; ++it) {
270 protos::pbzero::PerfettoMetatrace::Arg::Decoder arg_proto(*it);
271 StringId key;
272 if (arg_proto.has_key_iid()) {
273 key = GetMetatraceInternedString(arg_proto.key_iid());
274 } else {
275 key = context_->storage->InternString(arg_proto.key());
276 }
277 StringId value;
278 if (arg_proto.has_value_iid()) {
279 value = GetMetatraceInternedString(arg_proto.value_iid());
280 } else {
281 value = context_->storage->InternString(arg_proto.value());
282 }
283 interned.emplace_back(key, value);
284 }
285
286 // We stable sort insted of sorting here to avoid changing the order of the
287 // args in arrays.
288 std::stable_sort(interned.begin(), interned.end(),
289 [](const Arg& a, const Arg& b) {
290 return a.first.raw_id() < b.first.raw_id();
291 });
292
293 // Compute the correct key for each arg, possibly adding an index to
294 // the end of the key if needed.
295 char buffer[2048];
296 uint32_t current_idx = 0;
297 for (auto it = interned.begin(); it != interned.end(); ++it) {
298 auto next = it + 1;
299 StringId key = it->first;
300 StringId next_key = next == interned.end() ? kNullStringId : next->first;
301
302 if (key != next_key && current_idx == 0) {
303 inserter->AddArg(key, Variadic::String(it->second));
304 } else {
305 constexpr size_t kMaxIndexSize = 20;
306 NullTermStringView key_str = context_->storage->GetString(key);
307 if (key_str.size() >= sizeof(buffer) - kMaxIndexSize) {
308 PERFETTO_DLOG("Ignoring arg with unreasonbly large size");
309 continue;
310 }
311
312 base::StackString<2048> array_key("%s[%u]", key_str.c_str(),
313 current_idx);
314 StringId new_key =
315 context_->storage->InternString(array_key.string_view());
316 inserter->AddArg(key, new_key, Variadic::String(it->second));
317
318 current_idx = key == next_key ? current_idx + 1 : 0;
319 }
320 }
321 };
322
323 if (event.has_event_id() || event.has_event_name() ||
324 event.has_event_name_iid()) {
325 StringId name_id;
326 if (event.has_event_id()) {
327 auto eid = event.event_id();
328 if (eid < metatrace::EVENTS_MAX) {
329 name_id = context_->storage->InternString(metatrace::kEventNames[eid]);
330 } else {
331 base::StackString<64> fallback("Event %u", eid);
332 name_id = context_->storage->InternString(fallback.string_view());
333 }
334 } else if (event.has_event_name_iid()) {
335 name_id = GetMetatraceInternedString(event.event_name_iid());
336 } else {
337 name_id = context_->storage->InternString(event.event_name());
338 }
339 TrackId track_id = context_->track_tracker->InternThreadTrack(utid);
340 context_->slice_tracker->Scoped(
341 ts, track_id, cat_id, name_id,
342 static_cast<int64_t>(event.event_duration_ns()), args_fn);
343 } else if (event.has_counter_id() || event.has_counter_name()) {
344 static constexpr auto kBlueprint = tracks::CounterBlueprint(
345 "metatrace_counter", tracks::UnknownUnitBlueprint(),
346 tracks::DimensionBlueprints(
347 tracks::kThreadDimensionBlueprint,
348 tracks::StringDimensionBlueprint("counter_name")),
349 tracks::DynamicNameBlueprint());
350 TrackId track;
351 if (event.has_counter_id()) {
352 auto cid = event.counter_id();
353 StringId name_id;
354 if (cid < metatrace::COUNTERS_MAX) {
355 name_id =
356 context_->storage->InternString(metatrace::kCounterNames[cid]);
357 } else {
358 base::StackString<64> fallback("Counter %u", cid);
359 name_id = context_->storage->InternString(fallback.string_view());
360 }
361 track = context_->track_tracker->InternTrack(
362 kBlueprint,
363 tracks::Dimensions(utid, context_->storage->GetString(name_id)),
364 tracks::DynamicName(name_id));
365 } else {
366 track = context_->track_tracker->InternTrack(
367 kBlueprint, tracks::Dimensions(utid, event.counter_name()),
368 tracks::DynamicName(
369 context_->storage->InternString(event.counter_name())));
370 }
371 auto opt_id =
372 context_->event_tracker->PushCounter(ts, event.counter_value(), track);
373 if (opt_id) {
374 auto inserter = context_->args_tracker->AddArgsTo(*opt_id);
375 args_fn(&inserter);
376 }
377 }
378
379 if (event.has_overruns())
380 context_->storage->IncrementStats(stats::metatrace_overruns);
381 }
382
GetMetatraceInternedString(uint64_t iid)383 StringId ProtoTraceParserImpl::GetMetatraceInternedString(uint64_t iid) {
384 StringId* maybe_id = metatrace_interned_strings_.Find(iid);
385 if (!maybe_id)
386 return missing_metatrace_interned_string_id_;
387 return *maybe_id;
388 }
389
390 } // namespace perfetto::trace_processor
391