1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/sqlite/sqlite_engine.h"
18
19 #include <sqlite3.h>
20 #include <cstdint>
21 #include <optional>
22 #include <string>
23 #include <utility>
24
25 #include "perfetto/base/build_config.h"
26 #include "perfetto/base/logging.h"
27 #include "perfetto/base/status.h"
28 #include "perfetto/public/compiler.h"
29 #include "src/trace_processor/sqlite/scoped_db.h"
30 #include "src/trace_processor/sqlite/sql_source.h"
31 #include "src/trace_processor/tp_metatrace.h"
32
33 #include "protos/perfetto/trace_processor/metatrace_categories.pbzero.h"
34
35 // In Android and Chromium tree builds, we don't have the percentile module.
36 // Just don't include it.
37 #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
38 // defined in sqlite_src/ext/misc/percentile.c
39 extern "C" int sqlite3_percentile_init(sqlite3* db,
40 char** error,
41 const sqlite3_api_routines* api);
42 #endif // PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
43
44 namespace perfetto::trace_processor {
45 namespace {
46
EnsureSqliteInitialized()47 void EnsureSqliteInitialized() {
48 // sqlite3_initialize isn't actually thread-safe in standalone builds because
49 // we build with SQLITE_THREADSAFE=0. Ensure it's only called from a single
50 // thread.
51 static bool init_once = [] {
52 // Enabling memstatus causes a lock to be taken on every malloc/free in
53 // SQLite to update the memory statistics. This can cause massive contention
54 // in trace processor when multiple instances are used in parallel.
55 // Fix this by disabling the memstatus API which we don't make use of in
56 // any case. See b/335019324 for more info on this.
57 PERFETTO_CHECK(sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0) == SQLITE_OK);
58 return sqlite3_initialize() == SQLITE_OK;
59 }();
60 PERFETTO_CHECK(init_once);
61 }
62
InitializeSqlite(sqlite3 * db)63 void InitializeSqlite(sqlite3* db) {
64 char* error = nullptr;
65 sqlite3_exec(db, "PRAGMA temp_store=2", nullptr, nullptr, &error);
66 if (error) {
67 PERFETTO_FATAL("Error setting pragma temp_store: %s", error);
68 }
69 // In Android tree builds, we don't have the percentile module.
70 #if PERFETTO_BUILDFLAG(PERFETTO_TP_PERCENTILE)
71 sqlite3_percentile_init(db, &error, nullptr);
72 if (error) {
73 PERFETTO_ELOG("Error initializing: %s", error);
74 sqlite3_free(error);
75 }
76 #endif
77 }
78
GetErrorOffsetDb(sqlite3 * db)79 std::optional<uint32_t> GetErrorOffsetDb(sqlite3* db) {
80 int offset = sqlite3_error_offset(db);
81 return offset == -1 ? std::nullopt
82 : std::make_optional(static_cast<uint32_t>(offset));
83 }
84
85 } // namespace
86
SqliteEngine()87 SqliteEngine::SqliteEngine() {
88 sqlite3* db = nullptr;
89 EnsureSqliteInitialized();
90
91 // Ensure that we open the database with mutexes disabled: this is because
92 // trace processor as a whole cannot be used from multiple threads so there is
93 // no point paying the (potentially significant) cost of mutexes at the SQLite
94 // level.
95 static constexpr int kSqliteOpenFlags =
96 SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX;
97 PERFETTO_CHECK(sqlite3_open_v2(":memory:", &db, kSqliteOpenFlags, nullptr) ==
98 SQLITE_OK);
99 InitializeSqlite(db);
100 db_.reset(db);
101 }
102
~SqliteEngine()103 SqliteEngine::~SqliteEngine() {
104 // It is important to unregister any functions that have been registered with
105 // the database before destroying it. This is because functions can hold onto
106 // prepared statements, which must be finalized before database destruction.
107 for (auto it = fn_ctx_.GetIterator(); it; ++it) {
108 int ret = sqlite3_create_function_v2(db_.get(), it.key().first.c_str(),
109 it.key().second, SQLITE_UTF8, nullptr,
110 nullptr, nullptr, nullptr, nullptr);
111 if (PERFETTO_UNLIKELY(ret != SQLITE_OK)) {
112 PERFETTO_FATAL("Failed to drop function: '%s'", it.key().first.c_str());
113 }
114 }
115 fn_ctx_.Clear();
116 }
117
PrepareStatement(SqlSource sql)118 SqliteEngine::PreparedStatement SqliteEngine::PrepareStatement(SqlSource sql) {
119 PERFETTO_TP_TRACE(metatrace::Category::QUERY_DETAILED, "QUERY_PREPARE");
120 sqlite3_stmt* raw_stmt = nullptr;
121 int err =
122 sqlite3_prepare_v2(db_.get(), sql.sql().c_str(), -1, &raw_stmt, nullptr);
123 PreparedStatement statement{ScopedStmt(raw_stmt), std::move(sql)};
124 if (err != SQLITE_OK) {
125 const char* errmsg = sqlite3_errmsg(db_.get());
126 std::string frame =
127 statement.sql_source_.AsTracebackForSqliteOffset(GetErrorOffset());
128 base::Status status = base::ErrStatus("%s%s", frame.c_str(), errmsg);
129 status.SetPayload("perfetto.dev/has_traceback", "true");
130
131 statement.status_ = std::move(status);
132 return statement;
133 }
134 if (!raw_stmt) {
135 statement.status_ = base::ErrStatus("No SQL to execute");
136 }
137 return statement;
138 }
139
RegisterFunction(const char * name,int argc,Fn * fn,void * ctx,FnCtxDestructor * destructor,bool deterministic)140 base::Status SqliteEngine::RegisterFunction(const char* name,
141 int argc,
142 Fn* fn,
143 void* ctx,
144 FnCtxDestructor* destructor,
145 bool deterministic) {
146 int flags = SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0);
147 int ret =
148 sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc), flags,
149 ctx, fn, nullptr, nullptr, destructor);
150 if (ret != SQLITE_OK) {
151 return base::ErrStatus("Unable to register function with name %s", name);
152 }
153 *fn_ctx_.Insert(std::make_pair(name, argc), ctx).first = ctx;
154 return base::OkStatus();
155 }
156
RegisterAggregateFunction(const char * name,int argc,AggregateFnStep * step,AggregateFnFinal * final,void * ctx,FnCtxDestructor * destructor,bool deterministic)157 base::Status SqliteEngine::RegisterAggregateFunction(
158 const char* name,
159 int argc,
160 AggregateFnStep* step,
161 AggregateFnFinal* final,
162 void* ctx,
163 FnCtxDestructor* destructor,
164 bool deterministic) {
165 int flags = SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0);
166 int ret =
167 sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc), flags,
168 ctx, nullptr, step, final, destructor);
169 if (ret != SQLITE_OK) {
170 return base::ErrStatus("Unable to register function with name %s", name);
171 }
172 return base::OkStatus();
173 }
174
RegisterWindowFunction(const char * name,int argc,WindowFnStep * step,WindowFnInverse * inverse,WindowFnValue * value,WindowFnFinal * final,void * ctx,FnCtxDestructor * destructor,bool deterministic)175 base::Status SqliteEngine::RegisterWindowFunction(const char* name,
176 int argc,
177 WindowFnStep* step,
178 WindowFnInverse* inverse,
179 WindowFnValue* value,
180 WindowFnFinal* final,
181 void* ctx,
182 FnCtxDestructor* destructor,
183 bool deterministic) {
184 int flags = SQLITE_UTF8 | (deterministic ? SQLITE_DETERMINISTIC : 0);
185 int ret = sqlite3_create_window_function(
186 db_.get(), name, static_cast<int>(argc), flags, ctx, step, final, value,
187 inverse, destructor);
188 if (ret != SQLITE_OK) {
189 return base::ErrStatus("Unable to register function with name %s", name);
190 }
191 return base::OkStatus();
192 }
193
UnregisterFunction(const char * name,int argc)194 base::Status SqliteEngine::UnregisterFunction(const char* name, int argc) {
195 int ret = sqlite3_create_function_v2(db_.get(), name, static_cast<int>(argc),
196 SQLITE_UTF8, nullptr, nullptr, nullptr,
197 nullptr, nullptr);
198 if (ret != SQLITE_OK) {
199 return base::ErrStatus("Unable to unregister function with name %s", name);
200 }
201 fn_ctx_.Erase({name, argc});
202 return base::OkStatus();
203 }
204
DeclareVirtualTable(const std::string & create_stmt)205 base::Status SqliteEngine::DeclareVirtualTable(const std::string& create_stmt) {
206 int res = sqlite3_declare_vtab(db_.get(), create_stmt.c_str());
207 if (res != SQLITE_OK) {
208 return base::ErrStatus("Declare vtab failed: %s",
209 sqlite3_errmsg(db_.get()));
210 }
211 return base::OkStatus();
212 }
213
GetFunctionContext(const std::string & name,int argc)214 void* SqliteEngine::GetFunctionContext(const std::string& name, int argc) {
215 auto* res = fn_ctx_.Find(std::make_pair(name, argc));
216 return res ? *res : nullptr;
217 }
218
GetErrorOffset() const219 std::optional<uint32_t> SqliteEngine::GetErrorOffset() const {
220 return GetErrorOffsetDb(db_.get());
221 }
222
PreparedStatement(ScopedStmt stmt,SqlSource source)223 SqliteEngine::PreparedStatement::PreparedStatement(ScopedStmt stmt,
224 SqlSource source)
225 : stmt_(std::move(stmt)),
226 expanded_sql_(sqlite3_expanded_sql(stmt_.get())),
227 sql_source_(std::move(source)) {}
228
Step()229 bool SqliteEngine::PreparedStatement::Step() {
230 PERFETTO_TP_TRACE(metatrace::Category::QUERY_DETAILED, "STMT_STEP",
231 [this](metatrace::Record* record) {
232 record->AddArg("Original SQL", original_sql());
233 record->AddArg("Executed SQL", sql());
234 });
235
236 // Now step once into |cur_stmt| so that when we prepare the next statment
237 // we will have executed any dependent bytecode in this one.
238 int err = sqlite3_step(stmt_.get());
239 if (err == SQLITE_ROW) {
240 return true;
241 }
242 if (err == SQLITE_DONE) {
243 return false;
244 }
245 sqlite3* db = sqlite3_db_handle(stmt_.get());
246 std::string frame =
247 sql_source_.AsTracebackForSqliteOffset(GetErrorOffsetDb(db));
248 const char* errmsg = sqlite3_errmsg(db);
249 status_ = base::ErrStatus("%s%s", frame.c_str(), errmsg);
250 return false;
251 }
252
IsDone() const253 bool SqliteEngine::PreparedStatement::IsDone() const {
254 return !sqlite3_stmt_busy(stmt_.get());
255 }
256
original_sql() const257 const char* SqliteEngine::PreparedStatement::original_sql() const {
258 return sql_source_.original_sql().c_str();
259 }
260
sql() const261 const char* SqliteEngine::PreparedStatement::sql() const {
262 return expanded_sql_.get();
263 }
264
265 } // namespace perfetto::trace_processor
266