1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/sqlite/sqlite_table.h"
18
19 #include <string.h>
20 #include <algorithm>
21 #include <cinttypes>
22 #include <map>
23 #include <memory>
24
25 #include "perfetto/base/logging.h"
26 #include "perfetto/base/status.h"
27 #include "perfetto/ext/base/status_or.h"
28 #include "perfetto/ext/base/string_view.h"
29 #include "sqlite3.h"
30 #include "src/trace_processor/sqlite/sqlite_engine.h"
31 #include "src/trace_processor/tp_metatrace.h"
32 #include "src/trace_processor/util/status_macros.h"
33
34 namespace perfetto {
35 namespace trace_processor {
36
37 namespace {
38
TypeToSqlString(SqlValue::Type type)39 std::string TypeToSqlString(SqlValue::Type type) {
40 switch (type) {
41 case SqlValue::Type::kString:
42 return "TEXT";
43 case SqlValue::Type::kLong:
44 return "BIGINT";
45 case SqlValue::Type::kDouble:
46 return "DOUBLE";
47 case SqlValue::Type::kBytes:
48 return "BLOB";
49 case SqlValue::Type::kNull:
50 PERFETTO_FATAL("Cannot map unknown column type");
51 }
52 PERFETTO_FATAL("Not reached"); // For gcc
53 }
54
OpToDebugString(int op)55 std::string OpToDebugString(int op) {
56 switch (op) {
57 case SQLITE_INDEX_CONSTRAINT_EQ:
58 return "=";
59 case SQLITE_INDEX_CONSTRAINT_NE:
60 return "!=";
61 case SQLITE_INDEX_CONSTRAINT_GE:
62 return ">=";
63 case SQLITE_INDEX_CONSTRAINT_GT:
64 return ">";
65 case SQLITE_INDEX_CONSTRAINT_LE:
66 return "<=";
67 case SQLITE_INDEX_CONSTRAINT_LT:
68 return "<";
69 case SQLITE_INDEX_CONSTRAINT_LIKE:
70 return "like";
71 case SQLITE_INDEX_CONSTRAINT_ISNULL:
72 return "is null";
73 case SQLITE_INDEX_CONSTRAINT_ISNOTNULL:
74 return "is not null";
75 case SQLITE_INDEX_CONSTRAINT_GLOB:
76 return "glob";
77 case SQLITE_INDEX_CONSTRAINT_LIMIT:
78 return "limit";
79 case SQLITE_INDEX_CONSTRAINT_OFFSET:
80 return "offset";
81 case SqliteTable::CustomFilterOpcode::kSourceGeqOpCode:
82 return "source_geq";
83 default:
84 PERFETTO_FATAL("Operator to string conversion not impemented for %d", op);
85 }
86 }
87
ConstraintsToString(const QueryConstraints & qc,const SqliteTable::Schema & schema,std::string & out)88 void ConstraintsToString(const QueryConstraints& qc,
89 const SqliteTable::Schema& schema,
90 std::string& out) {
91 bool is_first = true;
92 for (const auto& cs : qc.constraints()) {
93 if (!is_first) {
94 out.append(",");
95 }
96 out.append(schema.columns()[static_cast<size_t>(cs.column)].name());
97 out.append(" ");
98 out.append(OpToDebugString(cs.op));
99 is_first = false;
100 }
101 }
102
OrderByToString(const QueryConstraints & qc,const SqliteTable::Schema & schema,std::string & out)103 void OrderByToString(const QueryConstraints& qc,
104 const SqliteTable::Schema& schema,
105 std::string& out) {
106 bool is_first = true;
107 for (const auto& ob : qc.order_by()) {
108 if (!is_first) {
109 out.append(",");
110 }
111 out.append(schema.columns()[static_cast<size_t>(ob.iColumn)].name());
112 out.append(" ");
113 out.append(std::to_string(ob.desc));
114 is_first = false;
115 }
116 }
117
QcDebugStr(const QueryConstraints & qc,const SqliteTable::Schema & schema)118 std::string QcDebugStr(const QueryConstraints& qc,
119 const SqliteTable::Schema& schema) {
120 std::string str_result;
121 str_result.reserve(512);
122
123 str_result.append("C");
124 str_result.append(std::to_string(qc.constraints().size()));
125 str_result.append(",");
126 ConstraintsToString(qc, schema, str_result);
127 str_result.append(";");
128
129 str_result.append("O");
130 str_result.append(std::to_string(qc.order_by().size()));
131 str_result.append(",");
132 OrderByToString(qc, schema, str_result);
133 str_result.append(";");
134
135 str_result.append("U");
136 str_result.append(std::to_string(qc.cols_used()));
137
138 return str_result;
139 }
140
WriteQueryConstraintsToMetatrace(metatrace::Record * r,const QueryConstraints & qc,const SqliteTable::Schema & schema)141 void WriteQueryConstraintsToMetatrace(metatrace::Record* r,
142 const QueryConstraints& qc,
143 const SqliteTable::Schema& schema) {
144 r->AddArg("constraint_count", std::to_string(qc.constraints().size()));
145 std::string constraints;
146 ConstraintsToString(qc, schema, constraints);
147 r->AddArg("constraints", constraints);
148 r->AddArg("order_by_count", std::to_string(qc.order_by().size()));
149 std::string order_by;
150 OrderByToString(qc, schema, order_by);
151 r->AddArg("order_by", order_by);
152 r->AddArg("columns_used", std::to_string(qc.cols_used()));
153 }
154
155 } // namespace
156
157 // static
158 bool SqliteTable::debug = false;
159
160 SqliteTable::SqliteTable() = default;
161 SqliteTable::~SqliteTable() = default;
162
ModifyConstraints(QueryConstraints *)163 base::Status SqliteTable::ModifyConstraints(QueryConstraints*) {
164 return base::OkStatus();
165 }
166
FindFunction(const char *,FindFunctionFn *,void **)167 int SqliteTable::FindFunction(const char*, FindFunctionFn*, void**) {
168 return 0;
169 }
170
Update(int,sqlite3_value **,sqlite3_int64 *)171 base::Status SqliteTable::Update(int, sqlite3_value**, sqlite3_int64*) {
172 return base::ErrStatus("Updating not supported");
173 }
174
ReadConstraints(int idxNum,const char * idxStr,int argc)175 bool SqliteTable::ReadConstraints(int idxNum, const char* idxStr, int argc) {
176 bool cache_hit = true;
177 if (idxNum != qc_hash_) {
178 qc_cache_ = QueryConstraints::FromString(idxStr);
179 qc_hash_ = idxNum;
180 cache_hit = false;
181 }
182
183 PERFETTO_TP_TRACE(metatrace::Category::QUERY, "SQLITE_TABLE_READ_CONSTRAINTS",
184 [&](metatrace::Record* r) {
185 r->AddArg("cache_hit", std::to_string(cache_hit));
186 r->AddArg("name", name_);
187 WriteQueryConstraintsToMetatrace(r, qc_cache_, schema_);
188 r->AddArg("raw_constraints", idxStr);
189 r->AddArg("argc", std::to_string(argc));
190 });
191
192 // Logging this every ReadConstraints just leads to log spam on joins making
193 // it unusable. Instead, only print this out when we miss the cache (which
194 // happens precisely when the constraint set from SQLite changes.)
195 if (SqliteTable::debug && !cache_hit) {
196 PERFETTO_LOG("[%s::ParseConstraints] constraints=%s argc=%d", name_.c_str(),
197 QcDebugStr(qc_cache_, schema_).c_str(), argc);
198 }
199 return cache_hit;
200 }
201
202 ////////////////////////////////////////////////////////////////////////////////
203 // SqliteTable::BaseCursor implementation
204 ////////////////////////////////////////////////////////////////////////////////
205
BaseCursor(SqliteTable * table)206 SqliteTable::BaseCursor::BaseCursor(SqliteTable* table) : table_(table) {
207 // This is required to prevent us from leaving this field uninitialised if
208 // we ever move construct the Cursor.
209 pVtab = table;
210 }
211 SqliteTable::BaseCursor::~BaseCursor() = default;
212
213 ////////////////////////////////////////////////////////////////////////////////
214 // SqliteTable::Column implementation
215 ////////////////////////////////////////////////////////////////////////////////
216
Column(size_t index,std::string name,SqlValue::Type type,bool hidden)217 SqliteTable::Column::Column(size_t index,
218 std::string name,
219 SqlValue::Type type,
220 bool hidden)
221 : index_(index), name_(name), type_(type), hidden_(hidden) {}
222
223 ////////////////////////////////////////////////////////////////////////////////
224 // SqliteTable::Schema implementation
225 ////////////////////////////////////////////////////////////////////////////////
226
227 SqliteTable::Schema::Schema() = default;
228
Schema(std::vector<Column> columns,std::vector<size_t> primary_keys)229 SqliteTable::Schema::Schema(std::vector<Column> columns,
230 std::vector<size_t> primary_keys)
231 : columns_(std::move(columns)), primary_keys_(std::move(primary_keys)) {
232 for (size_t i = 0; i < columns_.size(); i++) {
233 PERFETTO_CHECK(columns_[i].index() == i);
234 }
235 for (auto key : primary_keys_) {
236 PERFETTO_CHECK(key < columns_.size());
237 }
238 }
239
240 SqliteTable::Schema::Schema(const Schema&) = default;
241 SqliteTable::Schema& SqliteTable::Schema::operator=(const Schema&) = default;
242
ToCreateTableStmt() const243 std::string SqliteTable::Schema::ToCreateTableStmt() const {
244 std::string stmt = "CREATE TABLE x(";
245 for (size_t i = 0; i < columns_.size(); ++i) {
246 const Column& col = columns_[i];
247 stmt += " " + col.name();
248
249 if (col.type() != SqlValue::Type::kNull) {
250 stmt += " " + TypeToSqlString(col.type());
251 } else if (std::find(primary_keys_.begin(), primary_keys_.end(), i) !=
252 primary_keys_.end()) {
253 PERFETTO_FATAL("Unknown type for primary key column %s",
254 col.name().c_str());
255 }
256 if (col.hidden()) {
257 stmt += " HIDDEN";
258 }
259 stmt += ",";
260 }
261 stmt += " PRIMARY KEY(";
262 for (size_t i = 0; i < primary_keys_.size(); i++) {
263 if (i != 0)
264 stmt += ", ";
265 stmt += columns_[primary_keys_[i]].name();
266 }
267 stmt += ")) WITHOUT ROWID;";
268 return stmt;
269 }
270
271 ////////////////////////////////////////////////////////////////////////////////
272 // TypedSqliteTableBase implementation
273 ////////////////////////////////////////////////////////////////////////////////
274
275 TypedSqliteTableBase::~TypedSqliteTableBase() = default;
276
DeclareAndAssignVtab(std::unique_ptr<SqliteTable> table,sqlite3_vtab ** tab)277 base::Status TypedSqliteTableBase::DeclareAndAssignVtab(
278 std::unique_ptr<SqliteTable> table,
279 sqlite3_vtab** tab) {
280 auto create_stmt = table->schema().ToCreateTableStmt();
281 PERFETTO_DLOG("Create table statement: %s", create_stmt.c_str());
282 RETURN_IF_ERROR(table->engine_->DeclareVirtualTable(create_stmt));
283 *tab = table.release();
284 return base::OkStatus();
285 }
286
xDestroy(sqlite3_vtab * t)287 int TypedSqliteTableBase::xDestroy(sqlite3_vtab* t) {
288 delete static_cast<SqliteTable*>(t);
289 return SQLITE_OK;
290 }
291
xDestroyFatal(sqlite3_vtab *)292 int TypedSqliteTableBase::xDestroyFatal(sqlite3_vtab*) {
293 PERFETTO_FATAL("xDestroy should not be called");
294 }
295
xConnectRestoreTable(sqlite3 *,void * arg,int,const char * const * argv,sqlite3_vtab ** tab,char ** pzErr)296 int TypedSqliteTableBase::xConnectRestoreTable(sqlite3*,
297 void* arg,
298 int,
299 const char* const* argv,
300 sqlite3_vtab** tab,
301 char** pzErr) {
302 auto* xArg = static_cast<BaseModuleArg*>(arg);
303
304 // SQLite guarantees that argv[2] contains the name of the table.
305 std::string table_name = argv[2];
306 base::StatusOr<std::unique_ptr<SqliteTable>> table =
307 xArg->engine->RestoreSqliteTable(table_name);
308 if (!table.status().ok()) {
309 *pzErr = sqlite3_mprintf("%s", table.status().c_message());
310 return SQLITE_ERROR;
311 }
312 base::Status status = DeclareAndAssignVtab(std::move(table.value()), tab);
313 if (!status.ok()) {
314 *pzErr = sqlite3_mprintf("%s", status.c_message());
315 return SQLITE_ERROR;
316 }
317 return SQLITE_OK;
318 }
319
xDisconnectSaveTable(sqlite3_vtab * t)320 int TypedSqliteTableBase::xDisconnectSaveTable(sqlite3_vtab* t) {
321 auto* table = static_cast<TypedSqliteTableBase*>(t);
322 base::Status status = table->engine_->SaveSqliteTable(
323 table->name(), std::unique_ptr<SqliteTable>(table));
324 return table->SetStatusAndReturn(status);
325 }
326
InitInternal(SqliteEngine * engine,int argc,const char * const * argv)327 base::Status TypedSqliteTableBase::InitInternal(SqliteEngine* engine,
328 int argc,
329 const char* const* argv) {
330 // Set the engine to allow saving into it later.
331 engine_ = engine;
332
333 // SQLite guarantees that argv[0] will be the "module" name: this is the
334 // same as |table_name| passed to the Register function.
335 module_name_ = argv[0];
336
337 // SQLite guarantees that argv[2] contains the name of the table: for
338 // non-arg taking tables, this will be the same as |table_name| but for
339 // arg-taking tables, this will be the table name as defined by the
340 // user in the CREATE VIRTUAL TABLE call.
341 name_ = argv[2];
342
343 Schema schema;
344 RETURN_IF_ERROR(Init(argc, argv, &schema));
345 schema_ = std::move(schema);
346 return base::OkStatus();
347 }
348
xOpen(sqlite3_vtab * t,sqlite3_vtab_cursor ** ppCursor)349 int TypedSqliteTableBase::xOpen(sqlite3_vtab* t,
350 sqlite3_vtab_cursor** ppCursor) {
351 auto* table = static_cast<TypedSqliteTableBase*>(t);
352 *ppCursor =
353 static_cast<sqlite3_vtab_cursor*>(table->CreateCursor().release());
354 return SQLITE_OK;
355 }
356
xBestIndex(sqlite3_vtab * t,sqlite3_index_info * idx)357 int TypedSqliteTableBase::xBestIndex(sqlite3_vtab* t, sqlite3_index_info* idx) {
358 auto* table = static_cast<TypedSqliteTableBase*>(t);
359
360 QueryConstraints qc(idx->colUsed);
361
362 for (int i = 0; i < idx->nConstraint; i++) {
363 const auto& cs = idx->aConstraint[i];
364 if (!cs.usable)
365 continue;
366 qc.AddConstraint(cs.iColumn, cs.op, i);
367 }
368
369 for (int i = 0; i < idx->nOrderBy; i++) {
370 int column = idx->aOrderBy[i].iColumn;
371 bool desc = idx->aOrderBy[i].desc;
372 qc.AddOrderBy(column, desc);
373 }
374
375 int ret = table->SetStatusAndReturn(table->ModifyConstraints(&qc));
376 if (ret != SQLITE_OK)
377 return ret;
378
379 BestIndexInfo info;
380 info.estimated_cost = idx->estimatedCost;
381 info.estimated_rows = idx->estimatedRows;
382 info.sqlite_omit_constraint.resize(qc.constraints().size());
383
384 ret = table->BestIndex(qc, &info);
385
386 if (ret != SQLITE_OK)
387 return ret;
388
389 idx->orderByConsumed = qc.order_by().empty() || info.sqlite_omit_order_by;
390 idx->estimatedCost = info.estimated_cost;
391 idx->estimatedRows = info.estimated_rows;
392
393 // First pass: mark all constraints as omitted to ensure that any pruned
394 // constraints are not checked for by SQLite.
395 for (int i = 0; i < idx->nConstraint; ++i) {
396 auto& u = idx->aConstraintUsage[i];
397 u.omit = true;
398 }
399
400 // Second pass: actually set the correct omit and index values for all
401 // retained constraints.
402 for (uint32_t i = 0; i < qc.constraints().size(); ++i) {
403 auto& u = idx->aConstraintUsage[qc.constraints()[i].a_constraint_idx];
404 u.omit = info.sqlite_omit_constraint[i];
405 u.argvIndex = static_cast<int>(i) + 1;
406 }
407
408 PERFETTO_TP_TRACE(
409 metatrace::Category::QUERY, "SQLITE_TABLE_BEST_INDEX",
410 [&](metatrace::Record* r) {
411 r->AddArg("name", table->name());
412 WriteQueryConstraintsToMetatrace(r, qc, table->schema());
413 r->AddArg("order_by_consumed", std::to_string(idx->orderByConsumed));
414 r->AddArg("estimated_cost", std::to_string(idx->estimatedCost));
415 r->AddArg("estimated_rows",
416 std::to_string(static_cast<int64_t>(idx->estimatedRows)));
417 });
418
419 auto out_qc_str = qc.ToNewSqlite3String();
420 if (SqliteTable::debug) {
421 PERFETTO_LOG(
422 "[%s::BestIndex] constraints=%s orderByConsumed=%d estimatedCost=%f "
423 "estimatedRows=%" PRId64,
424 table->name().c_str(), QcDebugStr(qc, table->schema()).c_str(),
425 idx->orderByConsumed, idx->estimatedCost,
426 static_cast<int64_t>(idx->estimatedRows));
427 }
428
429 idx->idxStr = out_qc_str.release();
430 idx->needToFreeIdxStr = true;
431 idx->idxNum = ++table->best_index_num_;
432
433 return SQLITE_OK;
434 }
435
436 } // namespace trace_processor
437 } // namespace perfetto
438