1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/trace_processor_impl.h"
18
19 #include <inttypes.h>
20 #include <algorithm>
21
22 #include "perfetto/base/logging.h"
23 #include "perfetto/base/time.h"
24 #include "perfetto/ext/base/string_splitter.h"
25 #include "perfetto/ext/base/string_utils.h"
26 #include "src/trace_processor/dynamic/ancestor_generator.h"
27 #include "src/trace_processor/dynamic/connected_flow_generator.h"
28 #include "src/trace_processor/dynamic/descendant_slice_generator.h"
29 #include "src/trace_processor/dynamic/describe_slice_generator.h"
30 #include "src/trace_processor/dynamic/experimental_annotated_stack_generator.h"
31 #include "src/trace_processor/dynamic/experimental_counter_dur_generator.h"
32 #include "src/trace_processor/dynamic/experimental_flamegraph_generator.h"
33 #include "src/trace_processor/dynamic/experimental_sched_upid_generator.h"
34 #include "src/trace_processor/dynamic/experimental_slice_layout_generator.h"
35 #include "src/trace_processor/dynamic/thread_state_generator.h"
36 #include "src/trace_processor/export_json.h"
37 #include "src/trace_processor/importers/additional_modules.h"
38 #include "src/trace_processor/importers/ftrace/sched_event_tracker.h"
39 #include "src/trace_processor/importers/fuchsia/fuchsia_trace_parser.h"
40 #include "src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.h"
41 #include "src/trace_processor/importers/gzip/gzip_trace_parser.h"
42 #include "src/trace_processor/importers/json/json_trace_parser.h"
43 #include "src/trace_processor/importers/json/json_trace_tokenizer.h"
44 #include "src/trace_processor/importers/proto/metadata_tracker.h"
45 #include "src/trace_processor/importers/systrace/systrace_trace_parser.h"
46 #include "src/trace_processor/iterator_impl.h"
47 #include "src/trace_processor/sqlite/span_join_operator_table.h"
48 #include "src/trace_processor/sqlite/sql_stats_table.h"
49 #include "src/trace_processor/sqlite/sqlite3_str_split.h"
50 #include "src/trace_processor/sqlite/sqlite_raw_table.h"
51 #include "src/trace_processor/sqlite/sqlite_table.h"
52 #include "src/trace_processor/sqlite/sqlite_utils.h"
53 #include "src/trace_processor/sqlite/stats_table.h"
54 #include "src/trace_processor/sqlite/window_operator_table.h"
55 #include "src/trace_processor/tp_metatrace.h"
56 #include "src/trace_processor/types/variadic.h"
57 #include "src/trace_processor/util/protozero_to_text.h"
58
59 #include "protos/perfetto/trace/perfetto/perfetto_metatrace.pbzero.h"
60 #include "protos/perfetto/trace/trace.pbzero.h"
61 #include "protos/perfetto/trace/trace_packet.pbzero.h"
62
63 #include "src/trace_processor/metrics/chrome/all_chrome_metrics.descriptor.h"
64 #include "src/trace_processor/metrics/metrics.descriptor.h"
65 #include "src/trace_processor/metrics/metrics.h"
66 #include "src/trace_processor/metrics/sql_metrics.h"
67
68 #if !PERFETTO_BUILDFLAG(PERFETTO_OS_WIN)
69 #include <cxxabi.h>
70 #endif
71
72 // In Android and Chromium tree builds, we don't have the percentile module.
73 // Just don't include it.
74 #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
75 // defined in sqlite_src/ext/misc/percentile.c
76 extern "C" int sqlite3_percentile_init(sqlite3* db,
77 char** error,
78 const sqlite3_api_routines* api);
79 #endif // PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
80
81 namespace perfetto {
82 namespace trace_processor {
83 namespace {
84
85 const char kAllTablesQuery[] =
86 "SELECT tbl_name, type FROM (SELECT * FROM sqlite_master UNION ALL SELECT "
87 "* FROM sqlite_temp_master)";
88
InitializeSqlite(sqlite3 * db)89 void InitializeSqlite(sqlite3* db) {
90 char* error = nullptr;
91 sqlite3_exec(db, "PRAGMA temp_store=2", 0, 0, &error);
92 if (error) {
93 PERFETTO_FATAL("Error setting pragma temp_store: %s", error);
94 }
95 sqlite3_str_split_init(db);
96 // In Android tree builds, we don't have the percentile module.
97 // Just don't include it.
98 #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
99 sqlite3_percentile_init(db, &error, nullptr);
100 if (error) {
101 PERFETTO_ELOG("Error initializing: %s", error);
102 sqlite3_free(error);
103 }
104 #endif
105 }
106
BuildBoundsTable(sqlite3 * db,std::pair<int64_t,int64_t> bounds)107 void BuildBoundsTable(sqlite3* db, std::pair<int64_t, int64_t> bounds) {
108 char* error = nullptr;
109 sqlite3_exec(db, "DELETE FROM trace_bounds", nullptr, nullptr, &error);
110 if (error) {
111 PERFETTO_ELOG("Error deleting from bounds table: %s", error);
112 sqlite3_free(error);
113 return;
114 }
115
116 char* insert_sql = sqlite3_mprintf("INSERT INTO trace_bounds VALUES(%" PRId64
117 ", %" PRId64 ")",
118 bounds.first, bounds.second);
119
120 sqlite3_exec(db, insert_sql, 0, 0, &error);
121 sqlite3_free(insert_sql);
122 if (error) {
123 PERFETTO_ELOG("Error inserting bounds table: %s", error);
124 sqlite3_free(error);
125 }
126 }
127
CreateBuiltinTables(sqlite3 * db)128 void CreateBuiltinTables(sqlite3* db) {
129 char* error = nullptr;
130 sqlite3_exec(db, "CREATE TABLE perfetto_tables(name STRING)", 0, 0, &error);
131 if (error) {
132 PERFETTO_ELOG("Error initializing: %s", error);
133 sqlite3_free(error);
134 }
135 sqlite3_exec(db,
136 "CREATE TABLE trace_bounds(start_ts BIG INT, end_ts BIG INT)", 0,
137 0, &error);
138 if (error) {
139 PERFETTO_ELOG("Error initializing: %s", error);
140 sqlite3_free(error);
141 }
142 // Ensure that the entries in power_profile are unique to prevent duplicates
143 // when the power_profile is augmented with additional profiles.
144 sqlite3_exec(db,
145 "CREATE TABLE power_profile("
146 "device STRING, cpu INT, cluster INT, freq INT, power DOUBLE,"
147 "UNIQUE(device, cpu, cluster, freq));",
148 0, 0, &error);
149 if (error) {
150 PERFETTO_ELOG("Error initializing: %s", error);
151 sqlite3_free(error);
152 }
153 sqlite3_exec(db, "CREATE TABLE trace_metrics(name STRING)", 0, 0, &error);
154 if (error) {
155 PERFETTO_ELOG("Error initializing: %s", error);
156 sqlite3_free(error);
157 }
158 // This is a table intended to be used for metric debugging/developing. Data
159 // in the table is shown specially in the UI, and users can insert rows into
160 // this table to draw more things.
161 sqlite3_exec(db,
162 "CREATE TABLE debug_slices (id BIG INT, name STRING, ts BIG INT,"
163 "dur BIG INT, depth BIG INT)",
164 0, 0, &error);
165 if (error) {
166 PERFETTO_ELOG("Error initializing: %s", error);
167 sqlite3_free(error);
168 }
169
170 // Initialize the bounds table with some data so even before parsing any data,
171 // we still have a valid table.
172 BuildBoundsTable(db, std::make_pair(0, 0));
173 }
174
CreateBuiltinViews(sqlite3 * db)175 void CreateBuiltinViews(sqlite3* db) {
176 char* error = nullptr;
177 sqlite3_exec(db,
178 "CREATE VIEW counter_definitions AS "
179 "SELECT "
180 " *, "
181 " id AS counter_id "
182 "FROM counter_track",
183 0, 0, &error);
184 if (error) {
185 PERFETTO_ELOG("Error initializing: %s", error);
186 sqlite3_free(error);
187 }
188
189 sqlite3_exec(db,
190 "CREATE VIEW counter_values AS "
191 "SELECT "
192 " *, "
193 " track_id as counter_id "
194 "FROM counter",
195 0, 0, &error);
196 if (error) {
197 PERFETTO_ELOG("Error initializing: %s", error);
198 sqlite3_free(error);
199 }
200
201 sqlite3_exec(db,
202 "CREATE VIEW counters AS "
203 "SELECT * "
204 "FROM counter_values v "
205 "INNER JOIN counter_track t "
206 "ON v.track_id = t.id "
207 "ORDER BY ts;",
208 0, 0, &error);
209 if (error) {
210 PERFETTO_ELOG("Error initializing: %s", error);
211 sqlite3_free(error);
212 }
213
214 sqlite3_exec(db,
215 "CREATE VIEW slice AS "
216 "SELECT "
217 " *, "
218 " category AS cat, "
219 " id AS slice_id "
220 "FROM internal_slice;",
221 0, 0, &error);
222 if (error) {
223 PERFETTO_ELOG("Error initializing: %s", error);
224 sqlite3_free(error);
225 }
226
227 sqlite3_exec(db,
228 "CREATE VIEW instants AS "
229 "SELECT "
230 "*, "
231 "0.0 as value "
232 "FROM instant;",
233 0, 0, &error);
234
235 if (error) {
236 PERFETTO_ELOG("Error initializing: %s", error);
237 sqlite3_free(error);
238 }
239
240 sqlite3_exec(db,
241 "CREATE VIEW sched AS "
242 "SELECT "
243 "*, "
244 "ts + dur as ts_end "
245 "FROM sched_slice;",
246 0, 0, &error);
247
248 if (error) {
249 PERFETTO_ELOG("Error initializing: %s", error);
250 sqlite3_free(error);
251 }
252
253 // Legacy view for "slice" table with a deprecated table name.
254 // TODO(eseckler): Remove this view when all users have switched to "slice".
255 sqlite3_exec(db,
256 "CREATE VIEW slices AS "
257 "SELECT * FROM slice;",
258 0, 0, &error);
259 if (error) {
260 PERFETTO_ELOG("Error initializing: %s", error);
261 sqlite3_free(error);
262 }
263
264 sqlite3_exec(db,
265 "CREATE VIEW thread AS "
266 "SELECT "
267 "id as utid, "
268 "* "
269 "FROM internal_thread;",
270 0, 0, &error);
271 if (error) {
272 PERFETTO_ELOG("Error initializing: %s", error);
273 sqlite3_free(error);
274 }
275
276 sqlite3_exec(db,
277 "CREATE VIEW process AS "
278 "SELECT "
279 "id as upid, "
280 "* "
281 "FROM internal_process;",
282 0, 0, &error);
283 if (error) {
284 PERFETTO_ELOG("Error initializing: %s", error);
285 sqlite3_free(error);
286 }
287 }
288
ExportJson(sqlite3_context * ctx,int,sqlite3_value ** argv)289 void ExportJson(sqlite3_context* ctx, int /*argc*/, sqlite3_value** argv) {
290 TraceStorage* storage = static_cast<TraceStorage*>(sqlite3_user_data(ctx));
291 FILE* output;
292 if (sqlite3_value_type(argv[0]) == SQLITE_INTEGER) {
293 // Assume input is an FD.
294 output = fdopen(sqlite3_value_int(argv[0]), "w");
295 if (!output) {
296 sqlite3_result_error(ctx, "Couldn't open output file from given FD", -1);
297 return;
298 }
299 } else {
300 const char* filename =
301 reinterpret_cast<const char*>(sqlite3_value_text(argv[0]));
302 output = fopen(filename, "w");
303 if (!output) {
304 sqlite3_result_error(ctx, "Couldn't open output file", -1);
305 return;
306 }
307 }
308
309 util::Status result = json::ExportJson(storage, output);
310 if (!result.ok()) {
311 sqlite3_result_error(ctx, result.message().c_str(), -1);
312 return;
313 }
314 }
315
CreateJsonExportFunction(TraceStorage * ts,sqlite3 * db)316 void CreateJsonExportFunction(TraceStorage* ts, sqlite3* db) {
317 auto ret = sqlite3_create_function_v2(db, "EXPORT_JSON", 1, SQLITE_UTF8, ts,
318 ExportJson, nullptr, nullptr,
319 sqlite_utils::kSqliteStatic);
320 if (ret) {
321 PERFETTO_ELOG("Error initializing EXPORT_JSON");
322 }
323 }
324
Hash(sqlite3_context * ctx,int argc,sqlite3_value ** argv)325 void Hash(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
326 base::Hash hash;
327 for (int i = 0; i < argc; ++i) {
328 sqlite3_value* value = argv[i];
329 switch (sqlite3_value_type(value)) {
330 case SQLITE_INTEGER:
331 hash.Update(sqlite3_value_int64(value));
332 break;
333 case SQLITE_TEXT: {
334 const char* ptr =
335 reinterpret_cast<const char*>(sqlite3_value_text(value));
336 hash.Update(ptr, strlen(ptr));
337 break;
338 }
339 default:
340 sqlite3_result_error(ctx, "Unsupported type of arg passed to HASH", -1);
341 return;
342 }
343 }
344 sqlite3_result_int64(ctx, static_cast<int64_t>(hash.digest()));
345 }
346
Demangle(sqlite3_context * ctx,int argc,sqlite3_value ** argv)347 void Demangle(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
348 if (argc != 1) {
349 sqlite3_result_error(ctx, "Unsupported number of arg passed to DEMANGLE",
350 -1);
351 return;
352 }
353 sqlite3_value* value = argv[0];
354 if (sqlite3_value_type(value) == SQLITE_NULL) {
355 sqlite3_result_null(ctx);
356 return;
357 }
358 if (sqlite3_value_type(value) != SQLITE_TEXT) {
359 sqlite3_result_error(ctx, "Unsupported type of arg passed to DEMANGLE", -1);
360 return;
361 }
362 const char* ptr = reinterpret_cast<const char*>(sqlite3_value_text(value));
363 #if !PERFETTO_BUILDFLAG(PERFETTO_OS_WIN)
364 int ignored = 0;
365 // This memory was allocated by malloc and will be passed to SQLite to free.
366 char* demangled_name = abi::__cxa_demangle(ptr, nullptr, nullptr, &ignored);
367 if (!demangled_name) {
368 sqlite3_result_null(ctx);
369 return;
370 }
371 sqlite3_result_text(ctx, demangled_name, -1, free);
372 #else
373 sqlite3_result_text(ctx, ptr, -1, sqlite_utils::kSqliteTransient);
374 #endif
375 }
376
LastNonNullStep(sqlite3_context * ctx,int argc,sqlite3_value ** argv)377 void LastNonNullStep(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
378 if (argc != 1) {
379 sqlite3_result_error(
380 ctx, "Unsupported number of args passed to LAST_NON_NULL", -1);
381 return;
382 }
383 sqlite3_value* value = argv[0];
384 if (sqlite3_value_type(value) == SQLITE_NULL) {
385 return;
386 }
387 sqlite3_value** ptr = reinterpret_cast<sqlite3_value**>(
388 sqlite3_aggregate_context(ctx, sizeof(sqlite3_value*)));
389 if (ptr) {
390 if (*ptr != nullptr) {
391 sqlite3_value_free(*ptr);
392 }
393 *ptr = sqlite3_value_dup(value);
394 }
395 }
396
LastNonNullInverse(sqlite3_context * ctx,int argc,sqlite3_value ** argv)397 void LastNonNullInverse(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
398 // Do nothing.
399 base::ignore_result(ctx);
400 base::ignore_result(argc);
401 base::ignore_result(argv);
402 }
403
LastNonNullValue(sqlite3_context * ctx)404 void LastNonNullValue(sqlite3_context* ctx) {
405 sqlite3_value** ptr =
406 reinterpret_cast<sqlite3_value**>(sqlite3_aggregate_context(ctx, 0));
407 if (!ptr || !*ptr) {
408 sqlite3_result_null(ctx);
409 } else {
410 sqlite3_result_value(ctx, *ptr);
411 }
412 }
413
LastNonNullFinal(sqlite3_context * ctx)414 void LastNonNullFinal(sqlite3_context* ctx) {
415 sqlite3_value** ptr =
416 reinterpret_cast<sqlite3_value**>(sqlite3_aggregate_context(ctx, 0));
417 if (!ptr || !*ptr) {
418 sqlite3_result_null(ctx);
419 } else {
420 sqlite3_result_value(ctx, *ptr);
421 sqlite3_value_free(*ptr);
422 }
423 }
424
CreateHashFunction(sqlite3 * db)425 void CreateHashFunction(sqlite3* db) {
426 auto ret = sqlite3_create_function_v2(
427 db, "HASH", -1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, &Hash,
428 nullptr, nullptr, nullptr);
429 if (ret) {
430 PERFETTO_ELOG("Error initializing HASH");
431 }
432 }
433
CreateDemangledNameFunction(sqlite3 * db)434 void CreateDemangledNameFunction(sqlite3* db) {
435 auto ret = sqlite3_create_function_v2(
436 db, "DEMANGLE", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, &Demangle,
437 nullptr, nullptr, nullptr);
438 if (ret != SQLITE_OK) {
439 PERFETTO_ELOG("Error initializing DEMANGLE: %s", sqlite3_errmsg(db));
440 }
441 }
442
CreateLastNonNullFunction(sqlite3 * db)443 void CreateLastNonNullFunction(sqlite3* db) {
444 auto ret = sqlite3_create_window_function(
445 db, "LAST_NON_NULL", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr,
446 &LastNonNullStep, &LastNonNullFinal, &LastNonNullValue,
447 &LastNonNullInverse, nullptr);
448 if (ret) {
449 PERFETTO_ELOG("Error initializing LAST_NON_NULL");
450 }
451 }
452
453 struct ValueAtMaxTsContext {
454 bool initialized;
455 int value_type;
456
457 int64_t max_ts;
458 int64_t int_value_at_max_ts;
459 double double_value_at_max_ts;
460 };
461
ValueAtMaxTsStep(sqlite3_context * ctx,int,sqlite3_value ** argv)462 void ValueAtMaxTsStep(sqlite3_context* ctx, int, sqlite3_value** argv) {
463 sqlite3_value* ts = argv[0];
464 sqlite3_value* value = argv[1];
465
466 // Note that sqlite3_aggregate_context zeros the memory for us so all the
467 // variables of the struct should be zero.
468 ValueAtMaxTsContext* fn_ctx = reinterpret_cast<ValueAtMaxTsContext*>(
469 sqlite3_aggregate_context(ctx, sizeof(ValueAtMaxTsContext)));
470
471 // For performance reasons, we only do the check for the type of ts and value
472 // on the first call of the function.
473 if (PERFETTO_UNLIKELY(!fn_ctx->initialized)) {
474 if (sqlite3_value_type(ts) != SQLITE_INTEGER) {
475 sqlite3_result_error(ctx, "VALUE_AT_MAX_TS: ts passed was not an integer",
476 -1);
477 return;
478 }
479
480 fn_ctx->value_type = sqlite3_value_type(value);
481 if (fn_ctx->value_type != SQLITE_INTEGER &&
482 fn_ctx->value_type != SQLITE_FLOAT) {
483 sqlite3_result_error(
484 ctx, "VALUE_AT_MAX_TS: value passed was not an integer or float", -1);
485 return;
486 }
487
488 fn_ctx->initialized = true;
489 }
490
491 // On dcheck builds however, we check every passed ts and value.
492 #if PERFETTO_DCHECK_IS_ON()
493 if (sqlite3_value_type(ts) != SQLITE_INTEGER) {
494 sqlite3_result_error(ctx, "VALUE_AT_MAX_TS: ts passed was not an integer",
495 -1);
496 return;
497 }
498 if (sqlite3_value_type(value) != fn_ctx->value_type) {
499 sqlite3_result_error(ctx, "VALUE_AT_MAX_TS: value type is inconsistent",
500 -1);
501 return;
502 }
503 #endif
504
505 int64_t ts_int = sqlite3_value_int64(ts);
506 if (PERFETTO_LIKELY(fn_ctx->max_ts < ts_int)) {
507 fn_ctx->max_ts = ts_int;
508
509 if (fn_ctx->value_type == SQLITE_INTEGER) {
510 fn_ctx->int_value_at_max_ts = sqlite3_value_int64(value);
511 } else {
512 fn_ctx->double_value_at_max_ts = sqlite3_value_double(value);
513 }
514 }
515 }
516
ValueAtMaxTsFinal(sqlite3_context * ctx)517 void ValueAtMaxTsFinal(sqlite3_context* ctx) {
518 ValueAtMaxTsContext* fn_ctx =
519 reinterpret_cast<ValueAtMaxTsContext*>(sqlite3_aggregate_context(ctx, 0));
520 if (!fn_ctx) {
521 sqlite3_result_null(ctx);
522 return;
523 }
524 if (fn_ctx->value_type == SQLITE_INTEGER) {
525 sqlite3_result_int64(ctx, fn_ctx->int_value_at_max_ts);
526 } else {
527 sqlite3_result_double(ctx, fn_ctx->double_value_at_max_ts);
528 }
529 }
530
CreateValueAtMaxTsFunction(sqlite3 * db)531 void CreateValueAtMaxTsFunction(sqlite3* db) {
532 auto ret = sqlite3_create_function_v2(
533 db, "VALUE_AT_MAX_TS", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr,
534 nullptr, &ValueAtMaxTsStep, &ValueAtMaxTsFinal, nullptr);
535 if (ret) {
536 PERFETTO_ELOG("Error initializing VALUE_AT_MAX_TS");
537 }
538 }
539
ExtractArg(sqlite3_context * ctx,int argc,sqlite3_value ** argv)540 void ExtractArg(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
541 if (argc != 2) {
542 sqlite3_result_error(ctx, "EXTRACT_ARG: 2 args required", -1);
543 return;
544 }
545
546 // If the arg set id is null, just return null as the result.
547 if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
548 sqlite3_result_null(ctx);
549 return;
550 }
551 if (sqlite3_value_type(argv[0]) != SQLITE_INTEGER) {
552 sqlite3_result_error(ctx, "EXTRACT_ARG: 1st argument should be arg set id",
553 -1);
554 return;
555 }
556 if (sqlite3_value_type(argv[1]) != SQLITE_TEXT) {
557 sqlite3_result_error(ctx, "EXTRACT_ARG: 2nd argument should be key", -1);
558 return;
559 }
560
561 TraceStorage* storage = static_cast<TraceStorage*>(sqlite3_user_data(ctx));
562 uint32_t arg_set_id = static_cast<uint32_t>(sqlite3_value_int(argv[0]));
563 const char* key = reinterpret_cast<const char*>(sqlite3_value_text(argv[1]));
564
565 base::Optional<Variadic> opt_value;
566 util::Status status = storage->ExtractArg(arg_set_id, key, &opt_value);
567 if (!status.ok()) {
568 sqlite3_result_error(ctx, status.c_message(), -1);
569 return;
570 }
571
572 if (!opt_value) {
573 sqlite3_result_null(ctx);
574 return;
575 }
576
577 switch (opt_value->type) {
578 case Variadic::kInt:
579 sqlite3_result_int64(ctx, opt_value->int_value);
580 break;
581 case Variadic::kBool:
582 sqlite3_result_int64(ctx, opt_value->bool_value);
583 break;
584 case Variadic::kUint:
585 sqlite3_result_int64(ctx, static_cast<int64_t>(opt_value->uint_value));
586 break;
587 case Variadic::kPointer:
588 sqlite3_result_int64(ctx, static_cast<int64_t>(opt_value->pointer_value));
589 break;
590 case Variadic::kJson:
591 sqlite3_result_text(ctx, storage->GetString(opt_value->json_value).data(),
592 -1, nullptr);
593 break;
594 case Variadic::kString:
595 sqlite3_result_text(
596 ctx, storage->GetString(opt_value->string_value).data(), -1, nullptr);
597 break;
598 case Variadic::kReal:
599 sqlite3_result_double(ctx, opt_value->real_value);
600 break;
601 }
602 }
603
CreateExtractArgFunction(TraceStorage * ts,sqlite3 * db)604 void CreateExtractArgFunction(TraceStorage* ts, sqlite3* db) {
605 auto ret = sqlite3_create_function_v2(db, "EXTRACT_ARG", 2,
606 SQLITE_UTF8 | SQLITE_DETERMINISTIC, ts,
607 &ExtractArg, nullptr, nullptr, nullptr);
608 if (ret != SQLITE_OK) {
609 PERFETTO_FATAL("Error initializing EXTRACT_ARG: %s", sqlite3_errmsg(db));
610 }
611 }
612
CreateSourceGeqFunction(sqlite3 * db)613 void CreateSourceGeqFunction(sqlite3* db) {
614 auto fn = [](sqlite3_context* ctx, int, sqlite3_value**) {
615 sqlite3_result_error(
616 ctx, "SOURCE_GEQ should not be called from the global scope", -1);
617 };
618 auto ret = sqlite3_create_function_v2(db, "SOURCE_GEQ", -1,
619 SQLITE_UTF8 | SQLITE_DETERMINISTIC,
620 nullptr, fn, nullptr, nullptr, nullptr);
621 if (ret != SQLITE_OK) {
622 PERFETTO_FATAL("Error initializing SOURCE_GEQ: %s", sqlite3_errmsg(db));
623 }
624 }
625
SetupMetrics(TraceProcessor * tp,sqlite3 * db,std::vector<metrics::SqlMetricFile> * sql_metrics)626 void SetupMetrics(TraceProcessor* tp,
627 sqlite3* db,
628 std::vector<metrics::SqlMetricFile>* sql_metrics) {
629 tp->ExtendMetricsProto(kMetricsDescriptor.data(), kMetricsDescriptor.size());
630 tp->ExtendMetricsProto(kAllChromeMetricsDescriptor.data(),
631 kAllChromeMetricsDescriptor.size());
632
633 for (const auto& file_to_sql : metrics::sql_metrics::kFileToSql) {
634 tp->RegisterMetric(file_to_sql.path, file_to_sql.sql);
635 }
636
637 {
638 std::unique_ptr<metrics::RunMetricContext> ctx(
639 new metrics::RunMetricContext());
640 ctx->tp = tp;
641 ctx->metrics = sql_metrics;
642 auto ret = sqlite3_create_function_v2(
643 db, "RUN_METRIC", -1, SQLITE_UTF8, ctx.release(), metrics::RunMetric,
644 nullptr, nullptr,
645 [](void* ptr) { delete static_cast<metrics::RunMetricContext*>(ptr); });
646 if (ret)
647 PERFETTO_FATAL("Error initializing RUN_METRIC");
648 }
649
650 {
651 auto ret = sqlite3_create_function_v2(
652 db, "RepeatedField", 1, SQLITE_UTF8, nullptr, nullptr,
653 metrics::RepeatedFieldStep, metrics::RepeatedFieldFinal, nullptr);
654 if (ret)
655 PERFETTO_FATAL("Error initializing RepeatedField");
656 }
657
658 {
659 auto ret = sqlite3_create_function_v2(db, "NULL_IF_EMPTY", 1, SQLITE_UTF8,
660 nullptr, metrics::NullIfEmpty,
661 nullptr, nullptr, nullptr);
662 if (ret)
663 PERFETTO_FATAL("Error initializing NULL_IF_EMPTY");
664 }
665 }
666
EnsureSqliteInitialized()667 void EnsureSqliteInitialized() {
668 // sqlite3_initialize isn't actually thread-safe despite being documented
669 // as such; we need to make sure multiple TraceProcessorImpl instances don't
670 // call it concurrently and only gets called once per process, instead.
671 static bool init_once = [] { return sqlite3_initialize() == SQLITE_OK; }();
672 PERFETTO_CHECK(init_once);
673 }
674
InsertIntoTraceMetricsTable(sqlite3 * db,const std::string & metric_name)675 void InsertIntoTraceMetricsTable(sqlite3* db, const std::string& metric_name) {
676 char* insert_sql = sqlite3_mprintf(
677 "INSERT INTO trace_metrics(name) VALUES('%q')", metric_name.c_str());
678 char* insert_error = nullptr;
679 sqlite3_exec(db, insert_sql, nullptr, nullptr, &insert_error);
680 sqlite3_free(insert_sql);
681 if (insert_error) {
682 PERFETTO_ELOG("Error registering table: %s", insert_error);
683 sqlite3_free(insert_error);
684 }
685 }
686
687 } // namespace
688
TraceProcessorImpl(const Config & cfg)689 TraceProcessorImpl::TraceProcessorImpl(const Config& cfg)
690 : TraceProcessorStorageImpl(cfg) {
691 context_.fuchsia_trace_tokenizer.reset(new FuchsiaTraceTokenizer(&context_));
692 context_.fuchsia_trace_parser.reset(new FuchsiaTraceParser(&context_));
693
694 context_.systrace_trace_parser.reset(new SystraceTraceParser(&context_));
695
696 if (gzip::IsGzipSupported())
697 context_.gzip_trace_parser.reset(new GzipTraceParser(&context_));
698
699 if (json::IsJsonSupported()) {
700 context_.json_trace_tokenizer.reset(new JsonTraceTokenizer(&context_));
701 context_.json_trace_parser.reset(new JsonTraceParser(&context_));
702 }
703
704 RegisterAdditionalModules(&context_);
705
706 sqlite3* db = nullptr;
707 EnsureSqliteInitialized();
708 PERFETTO_CHECK(sqlite3_open(":memory:", &db) == SQLITE_OK);
709 InitializeSqlite(db);
710 CreateBuiltinTables(db);
711 CreateBuiltinViews(db);
712 db_.reset(std::move(db));
713
714 CreateJsonExportFunction(context_.storage.get(), db);
715 CreateHashFunction(db);
716 CreateDemangledNameFunction(db);
717 CreateLastNonNullFunction(db);
718 CreateExtractArgFunction(context_.storage.get(), db);
719 CreateSourceGeqFunction(db);
720 CreateValueAtMaxTsFunction(db);
721
722 SetupMetrics(this, *db_, &sql_metrics_);
723
724 // Setup the query cache.
725 query_cache_.reset(new QueryCache());
726
727 const TraceStorage* storage = context_.storage.get();
728
729 SqlStatsTable::RegisterTable(*db_, storage);
730 StatsTable::RegisterTable(*db_, storage);
731
732 // Operator tables.
733 SpanJoinOperatorTable::RegisterTable(*db_, storage);
734 WindowOperatorTable::RegisterTable(*db_, storage);
735
736 // New style tables but with some custom logic.
737 SqliteRawTable::RegisterTable(*db_, query_cache_.get(), &context_);
738
739 // Tables dynamically generated at query time.
740 RegisterDynamicTable(std::unique_ptr<ExperimentalFlamegraphGenerator>(
741 new ExperimentalFlamegraphGenerator(&context_)));
742 RegisterDynamicTable(std::unique_ptr<ExperimentalCounterDurGenerator>(
743 new ExperimentalCounterDurGenerator(storage->counter_table())));
744 RegisterDynamicTable(std::unique_ptr<DescribeSliceGenerator>(
745 new DescribeSliceGenerator(&context_)));
746 RegisterDynamicTable(std::unique_ptr<ExperimentalSliceLayoutGenerator>(
747 new ExperimentalSliceLayoutGenerator(
748 context_.storage.get()->mutable_string_pool(),
749 &storage->slice_table())));
750 RegisterDynamicTable(std::unique_ptr<AncestorGenerator>(
751 new AncestorGenerator(AncestorGenerator::Ancestor::kSlice, &context_)));
752 RegisterDynamicTable(std::unique_ptr<AncestorGenerator>(new AncestorGenerator(
753 AncestorGenerator::Ancestor::kStackProfileCallsite, &context_)));
754 RegisterDynamicTable(std::unique_ptr<DescendantSliceGenerator>(
755 new DescendantSliceGenerator(&context_)));
756 RegisterDynamicTable(
757 std::unique_ptr<ConnectedFlowGenerator>(new ConnectedFlowGenerator(
758 ConnectedFlowGenerator::Mode::kDirectlyConnectedFlow, &context_)));
759 RegisterDynamicTable(
760 std::unique_ptr<ConnectedFlowGenerator>(new ConnectedFlowGenerator(
761 ConnectedFlowGenerator::Mode::kPrecedingFlow, &context_)));
762 RegisterDynamicTable(
763 std::unique_ptr<ConnectedFlowGenerator>(new ConnectedFlowGenerator(
764 ConnectedFlowGenerator::Mode::kFollowingFlow, &context_)));
765 RegisterDynamicTable(std::unique_ptr<ExperimentalSchedUpidGenerator>(
766 new ExperimentalSchedUpidGenerator(storage->sched_slice_table(),
767 storage->thread_table())));
768 RegisterDynamicTable(std::unique_ptr<ThreadStateGenerator>(
769 new ThreadStateGenerator(&context_)));
770 RegisterDynamicTable(std::unique_ptr<ExperimentalAnnotatedStackGenerator>(
771 new ExperimentalAnnotatedStackGenerator(&context_)));
772
773 // New style db-backed tables.
774 RegisterDbTable(storage->arg_table());
775 RegisterDbTable(storage->thread_table());
776 RegisterDbTable(storage->process_table());
777
778 RegisterDbTable(storage->slice_table());
779 RegisterDbTable(storage->flow_table());
780 RegisterDbTable(storage->thread_slice_table());
781 RegisterDbTable(storage->sched_slice_table());
782 RegisterDbTable(storage->instant_table());
783 RegisterDbTable(storage->gpu_slice_table());
784
785 RegisterDbTable(storage->track_table());
786 RegisterDbTable(storage->thread_track_table());
787 RegisterDbTable(storage->process_track_table());
788 RegisterDbTable(storage->gpu_track_table());
789
790 RegisterDbTable(storage->counter_table());
791
792 RegisterDbTable(storage->counter_track_table());
793 RegisterDbTable(storage->process_counter_track_table());
794 RegisterDbTable(storage->thread_counter_track_table());
795 RegisterDbTable(storage->cpu_counter_track_table());
796 RegisterDbTable(storage->irq_counter_track_table());
797 RegisterDbTable(storage->softirq_counter_track_table());
798 RegisterDbTable(storage->gpu_counter_track_table());
799 RegisterDbTable(storage->gpu_counter_group_table());
800 RegisterDbTable(storage->perf_counter_track_table());
801
802 RegisterDbTable(storage->heap_graph_object_table());
803 RegisterDbTable(storage->heap_graph_reference_table());
804 RegisterDbTable(storage->heap_graph_class_table());
805
806 RegisterDbTable(storage->symbol_table());
807 RegisterDbTable(storage->heap_profile_allocation_table());
808 RegisterDbTable(storage->cpu_profile_stack_sample_table());
809 RegisterDbTable(storage->perf_sample_table());
810 RegisterDbTable(storage->stack_profile_callsite_table());
811 RegisterDbTable(storage->stack_profile_mapping_table());
812 RegisterDbTable(storage->stack_profile_frame_table());
813 RegisterDbTable(storage->package_list_table());
814 RegisterDbTable(storage->profiler_smaps_table());
815
816 RegisterDbTable(storage->android_log_table());
817
818 RegisterDbTable(storage->vulkan_memory_allocations_table());
819
820 RegisterDbTable(storage->graphics_frame_slice_table());
821
822 RegisterDbTable(storage->expected_frame_timeline_slice_table());
823 RegisterDbTable(storage->actual_frame_timeline_slice_table());
824
825 RegisterDbTable(storage->metadata_table());
826 RegisterDbTable(storage->cpu_table());
827 RegisterDbTable(storage->cpu_freq_table());
828 RegisterDbTable(storage->clock_snapshot_table());
829
830 RegisterDbTable(storage->memory_snapshot_table());
831 RegisterDbTable(storage->process_memory_snapshot_table());
832 RegisterDbTable(storage->memory_snapshot_node_table());
833 RegisterDbTable(storage->memory_snapshot_edge_table());
834 }
835
836 TraceProcessorImpl::~TraceProcessorImpl() = default;
837
Parse(std::unique_ptr<uint8_t[]> data,size_t size)838 util::Status TraceProcessorImpl::Parse(std::unique_ptr<uint8_t[]> data,
839 size_t size) {
840 bytes_parsed_ += size;
841 return TraceProcessorStorageImpl::Parse(std::move(data), size);
842 }
843
GetCurrentTraceName()844 std::string TraceProcessorImpl::GetCurrentTraceName() {
845 if (current_trace_name_.empty())
846 return "";
847 auto size = " (" + std::to_string(bytes_parsed_ / 1024 / 1024) + " MB)";
848 return current_trace_name_ + size;
849 }
850
SetCurrentTraceName(const std::string & name)851 void TraceProcessorImpl::SetCurrentTraceName(const std::string& name) {
852 current_trace_name_ = name;
853 }
854
NotifyEndOfFile()855 void TraceProcessorImpl::NotifyEndOfFile() {
856 if (current_trace_name_.empty())
857 current_trace_name_ = "Unnamed trace";
858
859 TraceProcessorStorageImpl::NotifyEndOfFile();
860
861 SchedEventTracker::GetOrCreate(&context_)->FlushPendingEvents();
862 context_.metadata_tracker->SetMetadata(
863 metadata::trace_size_bytes,
864 Variadic::Integer(static_cast<int64_t>(bytes_parsed_)));
865 BuildBoundsTable(*db_, context_.storage->GetTraceTimestampBoundsNs());
866
867 // Create a snapshot of all tables and views created so far. This is so later
868 // we can drop all extra tables created by the UI and reset to the original
869 // state (see RestoreInitialTables).
870 initial_tables_.clear();
871 auto it = ExecuteQuery(kAllTablesQuery);
872 while (it.Next()) {
873 auto value = it.Get(0);
874 PERFETTO_CHECK(value.type == SqlValue::Type::kString);
875 initial_tables_.push_back(value.string_value);
876 }
877 }
878
RestoreInitialTables()879 size_t TraceProcessorImpl::RestoreInitialTables() {
880 std::vector<std::pair<std::string, std::string>> deletion_list;
881 std::string msg = "Resetting DB to initial state, deleting table/views:";
882 for (auto it = ExecuteQuery(kAllTablesQuery); it.Next();) {
883 std::string name(it.Get(0).string_value);
884 std::string type(it.Get(1).string_value);
885 if (std::find(initial_tables_.begin(), initial_tables_.end(), name) ==
886 initial_tables_.end()) {
887 msg += " " + name;
888 deletion_list.push_back(std::make_pair(type, name));
889 }
890 }
891
892 PERFETTO_LOG("%s", msg.c_str());
893 for (const auto& tn : deletion_list) {
894 std::string query = "DROP " + tn.first + " " + tn.second;
895 auto it = ExecuteQuery(query);
896 while (it.Next()) {
897 }
898 // Index deletion can legitimately fail. If one creates an index "i" on a
899 // table "t" but issues the deletion in the order (t, i), the DROP index i
900 // will fail with "no such index" because deleting the table "t"
901 // automatically deletes all associated indexes.
902 if (!it.Status().ok() && tn.first != "index")
903 PERFETTO_FATAL("%s -> %s", query.c_str(), it.Status().c_message());
904 }
905 return deletion_list.size();
906 }
907
ExecuteQuery(const std::string & sql,int64_t time_queued)908 Iterator TraceProcessorImpl::ExecuteQuery(const std::string& sql,
909 int64_t time_queued) {
910 sqlite3_stmt* raw_stmt;
911 int err;
912 {
913 PERFETTO_TP_TRACE("QUERY_PREPARE");
914 err = sqlite3_prepare_v2(*db_, sql.c_str(), static_cast<int>(sql.size()),
915 &raw_stmt, nullptr);
916 }
917
918 util::Status status;
919 uint32_t col_count = 0;
920 if (err != SQLITE_OK) {
921 status = util::ErrStatus("%s", sqlite3_errmsg(*db_));
922 } else {
923 col_count = static_cast<uint32_t>(sqlite3_column_count(raw_stmt));
924 }
925
926 base::TimeNanos t_start = base::GetWallTimeNs();
927 uint32_t sql_stats_row =
928 context_.storage->mutable_sql_stats()->RecordQueryBegin(sql, time_queued,
929 t_start.count());
930
931 std::unique_ptr<IteratorImpl> impl(new IteratorImpl(
932 this, *db_, ScopedStmt(raw_stmt), col_count, status, sql_stats_row));
933 return Iterator(std::move(impl));
934 }
935
InterruptQuery()936 void TraceProcessorImpl::InterruptQuery() {
937 if (!db_)
938 return;
939 query_interrupted_.store(true);
940 sqlite3_interrupt(db_.get());
941 }
942
IsRootMetricField(const std::string & metric_name)943 bool TraceProcessorImpl::IsRootMetricField(const std::string& metric_name) {
944 base::Optional<uint32_t> desc_idx =
945 pool_.FindDescriptorIdx(".perfetto.protos.TraceMetrics");
946 if (!desc_idx.has_value())
947 return false;
948 auto field_idx = pool_.descriptors()[*desc_idx].FindFieldByName(metric_name);
949 return field_idx != nullptr;
950 }
951
RegisterMetric(const std::string & path,const std::string & sql)952 util::Status TraceProcessorImpl::RegisterMetric(const std::string& path,
953 const std::string& sql) {
954 std::string stripped_sql;
955 for (base::StringSplitter sp(sql, '\n'); sp.Next();) {
956 if (strncmp(sp.cur_token(), "--", 2) != 0) {
957 stripped_sql.append(sp.cur_token());
958 stripped_sql.push_back('\n');
959 }
960 }
961
962 // Check if the metric with the given path already exists and if it does, just
963 // update the SQL associated with it.
964 auto it = std::find_if(
965 sql_metrics_.begin(), sql_metrics_.end(),
966 [&path](const metrics::SqlMetricFile& m) { return m.path == path; });
967 if (it != sql_metrics_.end()) {
968 it->sql = stripped_sql;
969 return util::OkStatus();
970 }
971
972 auto sep_idx = path.rfind('/');
973 std::string basename =
974 sep_idx == std::string::npos ? path : path.substr(sep_idx + 1);
975
976 auto sql_idx = basename.rfind(".sql");
977 if (sql_idx == std::string::npos) {
978 return util::ErrStatus("Unable to find .sql extension for metric");
979 }
980 auto no_ext_name = basename.substr(0, sql_idx);
981
982 metrics::SqlMetricFile metric;
983 metric.path = path;
984 metric.sql = stripped_sql;
985
986 if (IsRootMetricField(no_ext_name)) {
987 metric.proto_field_name = no_ext_name;
988 metric.output_table_name = no_ext_name + "_output";
989 InsertIntoTraceMetricsTable(*db_, no_ext_name);
990 }
991
992 sql_metrics_.emplace_back(metric);
993 return util::OkStatus();
994 }
995
ExtendMetricsProto(const uint8_t * data,size_t size)996 util::Status TraceProcessorImpl::ExtendMetricsProto(const uint8_t* data,
997 size_t size) {
998 util::Status status = pool_.AddFromFileDescriptorSet(data, size);
999 if (!status.ok())
1000 return status;
1001
1002 for (const auto& desc : pool_.descriptors()) {
1003 // Convert the full name (e.g. .perfetto.protos.TraceMetrics.SubMetric)
1004 // into a function name of the form (TraceMetrics_SubMetric).
1005 auto fn_name = desc.full_name().substr(desc.package_name().size() + 1);
1006 std::replace(fn_name.begin(), fn_name.end(), '.', '_');
1007
1008 std::unique_ptr<metrics::BuildProtoContext> ctx(
1009 new metrics::BuildProtoContext());
1010 ctx->tp = this;
1011 ctx->pool = &pool_;
1012 ctx->desc = &desc;
1013
1014 auto ret = sqlite3_create_function_v2(
1015 *db_, fn_name.c_str(), -1, SQLITE_UTF8, ctx.release(),
1016 metrics::BuildProto, nullptr, nullptr, [](void* ptr) {
1017 delete static_cast<metrics::BuildProtoContext*>(ptr);
1018 });
1019 if (ret != SQLITE_OK)
1020 return util::ErrStatus("%s", sqlite3_errmsg(*db_));
1021 }
1022 return util::OkStatus();
1023 }
1024
ComputeMetric(const std::vector<std::string> & metric_names,std::vector<uint8_t> * metrics_proto)1025 util::Status TraceProcessorImpl::ComputeMetric(
1026 const std::vector<std::string>& metric_names,
1027 std::vector<uint8_t>* metrics_proto) {
1028 auto opt_idx = pool_.FindDescriptorIdx(".perfetto.protos.TraceMetrics");
1029 if (!opt_idx.has_value())
1030 return util::Status("Root metrics proto descriptor not found");
1031
1032 const auto& root_descriptor = pool_.descriptors()[opt_idx.value()];
1033 return metrics::ComputeMetrics(this, metric_names, sql_metrics_,
1034 root_descriptor, metrics_proto);
1035 }
1036
ComputeMetricText(const std::vector<std::string> & metric_names,TraceProcessor::MetricResultFormat format,std::string * metrics_string)1037 util::Status TraceProcessorImpl::ComputeMetricText(
1038 const std::vector<std::string>& metric_names,
1039 TraceProcessor::MetricResultFormat format,
1040 std::string* metrics_string) {
1041 std::vector<uint8_t> metrics_proto;
1042 util::Status status = ComputeMetric(metric_names, &metrics_proto);
1043 if (!status.ok())
1044 return status;
1045 switch (format) {
1046 case TraceProcessor::MetricResultFormat::kProtoText:
1047 *metrics_string = protozero_to_text::ProtozeroToText(
1048 pool_, ".perfetto.protos.TraceMetrics",
1049 protozero::ConstBytes{metrics_proto.data(), metrics_proto.size()},
1050 protozero_to_text::kIncludeNewLines);
1051 break;
1052 case TraceProcessor::MetricResultFormat::kJson:
1053 // TODO(dproy): Implement this.
1054 PERFETTO_FATAL("Json formatted metrics not supported yet.");
1055 break;
1056 }
1057 return status;
1058 }
1059
GetMetricDescriptors()1060 std::vector<uint8_t> TraceProcessorImpl::GetMetricDescriptors() {
1061 return pool_.SerializeAsDescriptorSet();
1062 }
1063
EnableMetatrace()1064 void TraceProcessorImpl::EnableMetatrace() {
1065 metatrace::Enable();
1066 }
1067
DisableAndReadMetatrace(std::vector<uint8_t> * trace_proto)1068 util::Status TraceProcessorImpl::DisableAndReadMetatrace(
1069 std::vector<uint8_t>* trace_proto) {
1070 protozero::HeapBuffered<protos::pbzero::Trace> trace;
1071 metatrace::DisableAndReadBuffer([&trace](metatrace::Record* record) {
1072 auto packet = trace->add_packet();
1073 packet->set_timestamp(record->timestamp_ns);
1074 auto* evt = packet->set_perfetto_metatrace();
1075 evt->set_event_name(record->event_name);
1076 evt->set_event_duration_ns(record->duration_ns);
1077 evt->set_thread_id(1); // Not really important, just required for the ui.
1078
1079 if (record->args_buffer_size == 0)
1080 return;
1081
1082 base::StringSplitter s(record->args_buffer, record->args_buffer_size, '\0');
1083 for (; s.Next();) {
1084 auto* arg_proto = evt->add_args();
1085 arg_proto->set_key(s.cur_token());
1086
1087 bool has_next = s.Next();
1088 PERFETTO_CHECK(has_next);
1089 arg_proto->set_value(s.cur_token());
1090 }
1091 });
1092 *trace_proto = trace.SerializeAsArray();
1093 return util::OkStatus();
1094 }
1095
1096 } // namespace trace_processor
1097 } // namespace perfetto
1098