1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/trace_processor_impl.h"
18
19 #include <inttypes.h>
20 #include <algorithm>
21 #include <functional>
22
23 #include "perfetto/base/logging.h"
24 #include "perfetto/base/string_utils.h"
25 #include "perfetto/base/time.h"
26 #include "perfetto/protozero/scattered_heap_buffer.h"
27 #include "src/trace_processor/android_logs_table.h"
28 #include "src/trace_processor/args_table.h"
29 #include "src/trace_processor/args_tracker.h"
30 #include "src/trace_processor/clock_tracker.h"
31 #include "src/trace_processor/counter_definitions_table.h"
32 #include "src/trace_processor/counter_values_table.h"
33 #include "src/trace_processor/event_tracker.h"
34 #include "src/trace_processor/fuchsia_trace_parser.h"
35 #include "src/trace_processor/fuchsia_trace_tokenizer.h"
36 #include "src/trace_processor/heap_profile_tracker.h"
37 #include "src/trace_processor/instants_table.h"
38 #include "src/trace_processor/metrics/metrics.h"
39 #include "src/trace_processor/process_table.h"
40 #include "src/trace_processor/process_tracker.h"
41 #include "src/trace_processor/proto_trace_parser.h"
42 #include "src/trace_processor/proto_trace_tokenizer.h"
43 #include "src/trace_processor/raw_table.h"
44 #include "src/trace_processor/sched_slice_table.h"
45 #include "src/trace_processor/slice_table.h"
46 #include "src/trace_processor/slice_tracker.h"
47 #include "src/trace_processor/span_join_operator_table.h"
48 #include "src/trace_processor/sql_stats_table.h"
49 #include "src/trace_processor/sqlite3_str_split.h"
50 #include "src/trace_processor/stats_table.h"
51 #include "src/trace_processor/string_table.h"
52 #include "src/trace_processor/syscall_tracker.h"
53 #include "src/trace_processor/table.h"
54 #include "src/trace_processor/thread_table.h"
55 #include "src/trace_processor/trace_blob_view.h"
56 #include "src/trace_processor/trace_sorter.h"
57 #include "src/trace_processor/window_operator_table.h"
58
59 #include "perfetto/metrics/android/mem_metric.pbzero.h"
60 #include "perfetto/metrics/metrics.pbzero.h"
61
62 // JSON parsing is only supported in the standalone build.
63 #if PERFETTO_BUILDFLAG(PERFETTO_STANDALONE_BUILD)
64 #include "src/trace_processor/json_trace_parser.h"
65 #include "src/trace_processor/json_trace_tokenizer.h"
66 #endif
67
68 // In Android tree builds, we don't have the percentile module.
69 // Just don't include it.
70 #if !PERFETTO_BUILDFLAG(PERFETTO_ANDROID_BUILD)
71 // defined in sqlite_src/ext/misc/percentile.c
72 extern "C" int sqlite3_percentile_init(sqlite3* db,
73 char** error,
74 const sqlite3_api_routines* api);
75 #endif
76
77 namespace perfetto {
78 namespace trace_processor {
79 namespace {
80
RemoveWhitespace(const std::string & input)81 std::string RemoveWhitespace(const std::string& input) {
82 std::string str(input);
83 str.erase(std::remove_if(str.begin(), str.end(), ::isspace), str.end());
84 return str;
85 }
86
InitializeSqlite(sqlite3 * db)87 void InitializeSqlite(sqlite3* db) {
88 char* error = nullptr;
89 sqlite3_exec(db, "PRAGMA temp_store=2", 0, 0, &error);
90 if (error) {
91 PERFETTO_FATAL("Error setting pragma temp_store: %s", error);
92 }
93 sqlite3_str_split_init(db);
94 // In Android tree builds, we don't have the percentile module.
95 // Just don't include it.
96 #if !PERFETTO_BUILDFLAG(PERFETTO_ANDROID_BUILD)
97 sqlite3_percentile_init(db, &error, nullptr);
98 if (error) {
99 PERFETTO_ELOG("Error initializing: %s", error);
100 sqlite3_free(error);
101 }
102 #endif
103 }
104
BuildBoundsTable(sqlite3 * db,std::pair<int64_t,int64_t> bounds)105 void BuildBoundsTable(sqlite3* db, std::pair<int64_t, int64_t> bounds) {
106 char* error = nullptr;
107 sqlite3_exec(db, "DELETE FROM trace_bounds", nullptr, nullptr, &error);
108 if (error) {
109 PERFETTO_ELOG("Error deleting from bounds table: %s", error);
110 sqlite3_free(error);
111 return;
112 }
113
114 char* insert_sql = sqlite3_mprintf("INSERT INTO trace_bounds VALUES(%" PRId64
115 ", %" PRId64 ")",
116 bounds.first, bounds.second);
117
118 sqlite3_exec(db, insert_sql, 0, 0, &error);
119 sqlite3_free(insert_sql);
120 if (error) {
121 PERFETTO_ELOG("Error inserting bounds table: %s", error);
122 sqlite3_free(error);
123 }
124 }
125
CreateBuiltinTables(sqlite3 * db)126 void CreateBuiltinTables(sqlite3* db) {
127 char* error = nullptr;
128 sqlite3_exec(db, "CREATE TABLE perfetto_tables(name STRING)", 0, 0, &error);
129 if (error) {
130 PERFETTO_ELOG("Error initializing: %s", error);
131 sqlite3_free(error);
132 }
133 sqlite3_exec(db,
134 "CREATE TABLE trace_bounds(start_ts BIG INT, end_ts BIG INT)", 0,
135 0, &error);
136 if (error) {
137 PERFETTO_ELOG("Error initializing: %s", error);
138 sqlite3_free(error);
139 }
140
141 // Initialize the bounds table with some data so even before parsing any data,
142 // we still have a valid table.
143 BuildBoundsTable(db, std::make_pair(0, 0));
144 }
145
CreateBuiltinViews(sqlite3 * db)146 void CreateBuiltinViews(sqlite3* db) {
147 char* error = nullptr;
148 sqlite3_exec(db,
149 "CREATE VIEW counters AS "
150 "SELECT * FROM counter_values "
151 "INNER JOIN counter_definitions USING(counter_id);",
152 0, 0, &error);
153 if (error) {
154 PERFETTO_ELOG("Error initializing: %s", error);
155 sqlite3_free(error);
156 }
157
158 sqlite3_exec(db,
159 "CREATE VIEW slice AS "
160 "SELECT "
161 " *, "
162 " CASE ref_type "
163 " WHEN 'utid' THEN ref "
164 " ELSE NULL "
165 " END AS utid "
166 "FROM internal_slice;",
167 0, 0, &error);
168 if (error) {
169 PERFETTO_ELOG("Error initializing: %s", error);
170 sqlite3_free(error);
171 }
172
173 // Legacy view for "slice" table with a deprecated table name.
174 // TODO(eseckler): Remove this view when all users have switched to "slice".
175 sqlite3_exec(db,
176 "CREATE VIEW slices AS "
177 "SELECT * FROM slice;",
178 0, 0, &error);
179 if (error) {
180 PERFETTO_ELOG("Error initializing: %s", error);
181 sqlite3_free(error);
182 }
183 }
184
CreateMetricsFunctions(TraceProcessorImpl * tp,sqlite3 * db)185 void CreateMetricsFunctions(TraceProcessorImpl* tp, sqlite3* db) {
186 auto ret = sqlite3_create_function_v2(db, "RUN_METRIC", -1, SQLITE_UTF8, tp,
187 metrics::RunMetric, nullptr, nullptr,
188 sqlite_utils::kSqliteStatic);
189 if (ret) {
190 PERFETTO_ELOG("Error initializing RUN_METRIC");
191 }
192 }
193
194 // Fuchsia traces have a magic number as documented here:
195 // https://fuchsia.googlesource.com/fuchsia/+/HEAD/docs/development/tracing/trace-format/README.md#magic-number-record-trace-info-type-0
196 constexpr uint64_t kFuchsiaMagicNumber = 0x0016547846040010;
197
198 } // namespace
199
GuessTraceType(const uint8_t * data,size_t size)200 TraceType GuessTraceType(const uint8_t* data, size_t size) {
201 if (size == 0)
202 return kUnknownTraceType;
203 std::string start(reinterpret_cast<const char*>(data),
204 std::min<size_t>(size, 20));
205 std::string start_minus_white_space = RemoveWhitespace(start);
206 if (base::StartsWith(start_minus_white_space, "{\"traceEvents\":["))
207 return kJsonTraceType;
208 if (base::StartsWith(start_minus_white_space, "[{"))
209 return kJsonTraceType;
210 if (size >= 8) {
211 uint64_t first_word = *reinterpret_cast<const uint64_t*>(data);
212 if (first_word == kFuchsiaMagicNumber)
213 return kFuchsiaTraceType;
214 }
215 return ProtoTraceTokenizer::GuessProtoTraceType(data, size);
216 }
217
TraceProcessorImpl(const Config & cfg)218 TraceProcessorImpl::TraceProcessorImpl(const Config& cfg) : cfg_(cfg) {
219 sqlite3* db = nullptr;
220 PERFETTO_CHECK(sqlite3_open(":memory:", &db) == SQLITE_OK);
221 InitializeSqlite(db);
222 CreateBuiltinTables(db);
223 CreateBuiltinViews(db);
224 CreateMetricsFunctions(this, db);
225 db_.reset(std::move(db));
226
227 context_.storage.reset(new TraceStorage());
228 context_.args_tracker.reset(new ArgsTracker(&context_));
229 context_.slice_tracker.reset(new SliceTracker(&context_));
230 context_.event_tracker.reset(new EventTracker(&context_));
231 context_.process_tracker.reset(new ProcessTracker(&context_));
232 context_.syscall_tracker.reset(new SyscallTracker(&context_));
233 context_.clock_tracker.reset(new ClockTracker(&context_));
234 context_.heap_profile_tracker.reset(new HeapProfileTracker(&context_));
235
236 ArgsTable::RegisterTable(*db_, context_.storage.get());
237 ProcessTable::RegisterTable(*db_, context_.storage.get());
238 SchedSliceTable::RegisterTable(*db_, context_.storage.get());
239 SliceTable::RegisterTable(*db_, context_.storage.get());
240 SqlStatsTable::RegisterTable(*db_, context_.storage.get());
241 StringTable::RegisterTable(*db_, context_.storage.get());
242 ThreadTable::RegisterTable(*db_, context_.storage.get());
243 CounterDefinitionsTable::RegisterTable(*db_, context_.storage.get());
244 CounterValuesTable::RegisterTable(*db_, context_.storage.get());
245 SpanJoinOperatorTable::RegisterTable(*db_, context_.storage.get());
246 WindowOperatorTable::RegisterTable(*db_, context_.storage.get());
247 InstantsTable::RegisterTable(*db_, context_.storage.get());
248 StatsTable::RegisterTable(*db_, context_.storage.get());
249 AndroidLogsTable::RegisterTable(*db_, context_.storage.get());
250 RawTable::RegisterTable(*db_, context_.storage.get());
251 }
252
~TraceProcessorImpl()253 TraceProcessorImpl::~TraceProcessorImpl() {
254 for (auto* it : iterators_)
255 it->Reset();
256 }
257
Parse(std::unique_ptr<uint8_t[]> data,size_t size)258 bool TraceProcessorImpl::Parse(std::unique_ptr<uint8_t[]> data, size_t size) {
259 if (size == 0)
260 return true;
261 if (unrecoverable_parse_error_)
262 return false;
263
264 // If this is the first Parse() call, guess the trace type and create the
265 // appropriate parser.
266 if (!context_.chunk_reader) {
267 TraceType trace_type;
268 {
269 auto scoped_trace = context_.storage->TraceExecutionTimeIntoStats(
270 stats::guess_trace_type_duration_ns);
271 trace_type = GuessTraceType(data.get(), size);
272 }
273 int64_t window_size_ns = static_cast<int64_t>(cfg_.window_size_ns);
274 switch (trace_type) {
275 case kJsonTraceType:
276 PERFETTO_DLOG("Legacy JSON trace detected");
277 #if PERFETTO_BUILDFLAG(PERFETTO_STANDALONE_BUILD)
278 context_.chunk_reader.reset(new JsonTraceTokenizer(&context_));
279 // JSON traces have no guarantees about the order of events in them.
280 window_size_ns = std::numeric_limits<int64_t>::max();
281 context_.sorter.reset(new TraceSorter(&context_, window_size_ns));
282 context_.parser.reset(new JsonTraceParser(&context_));
283 #else
284 PERFETTO_FATAL("JSON traces only supported in standalone mode.");
285 #endif
286 break;
287 case kProtoWithTrackEventsTraceType:
288 case kProtoTraceType:
289 if (trace_type == kProtoWithTrackEventsTraceType) {
290 // TrackEvents can be ordered arbitrarily due to out-of-order absolute
291 // timestamps and cross-packet-sequence events (e.g. async events).
292 window_size_ns = std::numeric_limits<int64_t>::max();
293 }
294 context_.chunk_reader.reset(new ProtoTraceTokenizer(&context_));
295 context_.sorter.reset(new TraceSorter(&context_, window_size_ns));
296 context_.parser.reset(new ProtoTraceParser(&context_));
297 break;
298 case kFuchsiaTraceType:
299 context_.chunk_reader.reset(new FuchsiaTraceTokenizer(&context_));
300 context_.sorter.reset(new TraceSorter(&context_, window_size_ns));
301 context_.parser.reset(new FuchsiaTraceParser(&context_));
302 break;
303 case kUnknownTraceType:
304 return false;
305 }
306 }
307
308 auto scoped_trace = context_.storage->TraceExecutionTimeIntoStats(
309 stats::parse_trace_duration_ns);
310 bool res = context_.chunk_reader->Parse(std::move(data), size);
311 unrecoverable_parse_error_ |= !res;
312 return res;
313 }
314
NotifyEndOfFile()315 void TraceProcessorImpl::NotifyEndOfFile() {
316 if (unrecoverable_parse_error_ || !context_.chunk_reader)
317 return;
318
319 context_.sorter->ExtractEventsForced();
320 context_.event_tracker->FlushPendingEvents();
321 BuildBoundsTable(*db_, context_.storage->GetTraceTimestampBoundsNs());
322 }
323
ExecuteQuery(const std::string & sql,int64_t time_queued)324 TraceProcessor::Iterator TraceProcessorImpl::ExecuteQuery(
325 const std::string& sql,
326 int64_t time_queued) {
327 sqlite3_stmt* raw_stmt;
328 int err = sqlite3_prepare_v2(*db_, sql.c_str(), static_cast<int>(sql.size()),
329 &raw_stmt, nullptr);
330 base::Optional<std::string> error;
331 uint32_t col_count = 0;
332 if (err != SQLITE_OK) {
333 error = sqlite3_errmsg(*db_);
334 } else {
335 col_count = static_cast<uint32_t>(sqlite3_column_count(raw_stmt));
336 }
337
338 base::TimeNanos t_start = base::GetWallTimeNs();
339 uint32_t sql_stats_row =
340 context_.storage->mutable_sql_stats()->RecordQueryBegin(sql, time_queued,
341 t_start.count());
342
343 std::unique_ptr<IteratorImpl> impl(new IteratorImpl(
344 this, *db_, ScopedStmt(raw_stmt), col_count, error, sql_stats_row));
345 iterators_.emplace_back(impl.get());
346 return TraceProcessor::Iterator(std::move(impl));
347 }
348
InterruptQuery()349 void TraceProcessorImpl::InterruptQuery() {
350 if (!db_)
351 return;
352 query_interrupted_.store(true);
353 sqlite3_interrupt(db_.get());
354 }
355
ComputeMetric(const std::vector<std::string> & metric_names,std::vector<uint8_t> * metrics_proto)356 int TraceProcessorImpl::ComputeMetric(
357 const std::vector<std::string>& metric_names,
358 std::vector<uint8_t>* metrics_proto) {
359 return metrics::ComputeMetrics(this, metric_names, metrics_proto);
360 }
361
IteratorImpl(TraceProcessorImpl * trace_processor,sqlite3 * db,ScopedStmt stmt,uint32_t column_count,base::Optional<std::string> error,uint32_t sql_stats_row)362 TraceProcessor::IteratorImpl::IteratorImpl(TraceProcessorImpl* trace_processor,
363 sqlite3* db,
364 ScopedStmt stmt,
365 uint32_t column_count,
366 base::Optional<std::string> error,
367 uint32_t sql_stats_row)
368 : trace_processor_(trace_processor),
369 db_(db),
370 stmt_(std::move(stmt)),
371 column_count_(column_count),
372 error_(error),
373 sql_stats_row_(sql_stats_row) {}
374
~IteratorImpl()375 TraceProcessor::IteratorImpl::~IteratorImpl() {
376 if (trace_processor_) {
377 auto* its = &trace_processor_->iterators_;
378 auto it = std::find(its->begin(), its->end(), this);
379 PERFETTO_CHECK(it != its->end());
380 its->erase(it);
381
382 base::TimeNanos t_end = base::GetWallTimeNs();
383 auto* sql_stats = trace_processor_->context_.storage->mutable_sql_stats();
384 sql_stats->RecordQueryEnd(sql_stats_row_, t_end.count());
385 }
386 }
387
Reset()388 void TraceProcessor::IteratorImpl::Reset() {
389 *this = IteratorImpl(nullptr, nullptr, ScopedStmt(), 0, base::nullopt, 0);
390 }
391
RecordFirstNextInSqlStats()392 void TraceProcessor::IteratorImpl::RecordFirstNextInSqlStats() {
393 base::TimeNanos t_first_next = base::GetWallTimeNs();
394 auto* sql_stats = trace_processor_->context_.storage->mutable_sql_stats();
395 sql_stats->RecordQueryFirstNext(sql_stats_row_, t_first_next.count());
396 }
397
398 } // namespace trace_processor
399 } // namespace perfetto
400