1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/prelude/functions/create_view_function.h"
18
19 #include <numeric>
20
21 #include "perfetto/base/status.h"
22 #include "perfetto/ext/base/string_utils.h"
23 #include "perfetto/ext/base/string_view.h"
24 #include "perfetto/trace_processor/basic_types.h"
25 #include "src/trace_processor/prelude/functions/create_function_internal.h"
26 #include "src/trace_processor/sqlite/scoped_db.h"
27 #include "src/trace_processor/sqlite/sqlite_engine.h"
28 #include "src/trace_processor/sqlite/sqlite_table.h"
29 #include "src/trace_processor/sqlite/sqlite_utils.h"
30 #include "src/trace_processor/tp_metatrace.h"
31 #include "src/trace_processor/util/status_macros.h"
32
33 namespace perfetto {
34 namespace trace_processor {
35
36 namespace {
37
38 class CreatedViewFunction final
39 : public TypedSqliteTable<CreatedViewFunction, void*> {
40 public:
41 class Cursor final : public SqliteTable::BaseCursor {
42 public:
43 explicit Cursor(CreatedViewFunction* table);
44 ~Cursor() final;
45
46 base::Status Filter(const QueryConstraints& qc,
47 sqlite3_value**,
48 FilterHistory);
49 base::Status Next();
50 bool Eof();
51 base::Status Column(sqlite3_context* context, int N);
52
53 private:
54 ScopedStmt scoped_stmt_;
55 sqlite3_stmt* stmt_ = nullptr;
56 CreatedViewFunction* table_ = nullptr;
57 bool is_eof_ = false;
58 int next_call_count_ = 0;
59 };
60
61 CreatedViewFunction(sqlite3*, void*);
62 ~CreatedViewFunction() final;
63
64 base::Status Init(int argc, const char* const* argv, Schema*) final;
65 std::unique_ptr<SqliteTable::BaseCursor> CreateCursor() final;
66 int BestIndex(const QueryConstraints& qc, BestIndexInfo* info) final;
67
68 private:
69 Schema CreateSchema();
70
IsReturnValueColumn(size_t i) const71 bool IsReturnValueColumn(size_t i) const {
72 PERFETTO_DCHECK(i < schema().columns().size());
73 return i < return_values_.size();
74 }
75
IsArgumentColumn(size_t i) const76 bool IsArgumentColumn(size_t i) const {
77 PERFETTO_DCHECK(i < schema().columns().size());
78 return i >= return_values_.size() &&
79 (i - return_values_.size()) < prototype_.arguments.size();
80 }
81
IsPrimaryKeyColumn(size_t i) const82 bool IsPrimaryKeyColumn(size_t i) const {
83 PERFETTO_DCHECK(i < schema().columns().size());
84 return i == (return_values_.size() + prototype_.arguments.size());
85 }
86
87 sqlite3* db_ = nullptr;
88
89 Prototype prototype_;
90 std::vector<sql_argument::ArgumentDefinition> return_values_;
91
92 std::string prototype_str_;
93 std::string sql_defn_str_;
94 };
95
CreatedViewFunction(sqlite3 * db,void *)96 CreatedViewFunction::CreatedViewFunction(sqlite3* db, void*) : db_(db) {}
97 CreatedViewFunction::~CreatedViewFunction() = default;
98
Init(int argc,const char * const * argv,Schema * schema)99 base::Status CreatedViewFunction::Init(int argc,
100 const char* const* argv,
101 Schema* schema) {
102 // The first three args are SQLite ones which we ignore.
103 PERFETTO_CHECK(argc == 6);
104
105 prototype_str_ = argv[3];
106 std::string return_prototype_str = argv[4];
107 sql_defn_str_ = argv[5];
108
109 // SQLite gives us strings with quotes included (i.e. 'string'). Strip these
110 // from the front and back.
111 prototype_str_ = prototype_str_.substr(1, prototype_str_.size() - 2);
112 return_prototype_str =
113 return_prototype_str.substr(1, return_prototype_str.size() - 2);
114 sql_defn_str_ = sql_defn_str_.substr(1, sql_defn_str_.size() - 2);
115
116 // Parse all the arguments into a more friendly form.
117 base::Status status =
118 ParsePrototype(base::StringView(prototype_str_), prototype_);
119 if (!status.ok()) {
120 return base::ErrStatus("CREATE_VIEW_FUNCTION[prototype=%s]: %s",
121 prototype_str_.c_str(), status.c_message());
122 }
123
124 // Parse the return type into a enum format.
125 status = sql_argument::ParseArgumentDefinitions(return_prototype_str,
126 return_values_);
127 if (!status.ok()) {
128 return base::ErrStatus(
129 "CREATE_VIEW_FUNCTION[prototype=%s, return=%s]: unknown return type "
130 "specified",
131 prototype_str_.c_str(), return_prototype_str.c_str());
132 }
133
134 // Verify that the provided SQL prepares to a statement correctly.
135 ScopedStmt stmt;
136 sqlite3_stmt* raw_stmt = nullptr;
137 int ret = sqlite3_prepare_v2(db_, sql_defn_str_.data(),
138 static_cast<int>(sql_defn_str_.size()),
139 &raw_stmt, nullptr);
140 stmt.reset(raw_stmt);
141 if (ret != SQLITE_OK) {
142 return base::ErrStatus(
143 "%s: Failed to prepare SQL statement for function. "
144 "Check the SQL defintion this function for syntax errors.\n%s",
145 prototype_.function_name.c_str(),
146 sqlite_utils::FormatErrorMessage(
147 raw_stmt, base::StringView(sql_defn_str_), db_, ret)
148 .c_message());
149 }
150
151 // Verify that every argument name in the function appears in the
152 // argument list.
153 //
154 // We intentionally loop from 1 to |used_param_count| because SQL
155 // parameters are 1-indexed *not* 0-indexed.
156 int used_param_count = sqlite3_bind_parameter_count(stmt.get());
157 for (int i = 1; i <= used_param_count; ++i) {
158 const char* name = sqlite3_bind_parameter_name(stmt.get(), i);
159
160 if (!name) {
161 return base::ErrStatus(
162 "%s: \"Nameless\" SQL parameters cannot be used in the SQL "
163 "statements of view functions.",
164 prototype_.function_name.c_str());
165 }
166
167 if (!base::StringView(name).StartsWith("$")) {
168 return base::ErrStatus(
169 "%s: invalid parameter name %s used in the SQL definition of "
170 "the view function: all parameters must be prefixed with '$' not ':' "
171 "or '@'.",
172 prototype_.function_name.c_str(), name);
173 }
174
175 auto it =
176 std::find_if(prototype_.arguments.begin(), prototype_.arguments.end(),
177 [name](const sql_argument::ArgumentDefinition& arg) {
178 return arg.dollar_name() == name;
179 });
180 if (it == prototype_.arguments.end()) {
181 return base::ErrStatus(
182 "%s: parameter %s does not appear in the list of arguments in the "
183 "prototype of the view function.",
184 prototype_.function_name.c_str(), name);
185 }
186 }
187
188 // Verify that the prepared statement column count matches the return
189 // count.
190 uint32_t col_count = static_cast<uint32_t>(sqlite3_column_count(stmt.get()));
191 if (col_count != return_values_.size()) {
192 return base::ErrStatus(
193 "%s: number of return values %u does not match SQL statement column "
194 "count %zu.",
195 prototype_.function_name.c_str(), col_count, return_values_.size());
196 }
197
198 // Verify that the return names matches the prepared statment column names.
199 for (uint32_t i = 0; i < col_count; ++i) {
200 const char* name = sqlite3_column_name(stmt.get(), static_cast<int>(i));
201 if (name != return_values_[i].name()) {
202 return base::ErrStatus(
203 "%s: column %s at index %u does not match return value name %s.",
204 prototype_.function_name.c_str(), name, i,
205 return_values_[i].name().c_str());
206 }
207 }
208
209 // Now we've parsed prototype and return values, create the schema.
210 *schema = CreateSchema();
211
212 return base::OkStatus();
213 }
214
CreateSchema()215 SqliteTable::Schema CreatedViewFunction::CreateSchema() {
216 std::vector<Column> columns;
217 for (size_t i = 0; i < return_values_.size(); ++i) {
218 const auto& ret = return_values_[i];
219 columns.push_back(Column(columns.size(), ret.name().ToStdString(),
220 sql_argument::TypeToSqlValueType(ret.type())));
221 }
222 for (size_t i = 0; i < prototype_.arguments.size(); ++i) {
223 const auto& arg = prototype_.arguments[i];
224
225 // Add the "in_" prefix to every argument param to avoid clashes between the
226 // output and input parameters.
227 columns.push_back(Column(columns.size(), "in_" + arg.name().ToStdString(),
228 sql_argument::TypeToSqlValueType(arg.type()),
229 true));
230 }
231
232 std::vector<size_t> primary_keys;
233
234 // Add the "primary key" column. SQLite requires that we provide a column
235 // which is non-null and unique. Unfortunately, we have no restrictions on
236 // the subqueries so we cannot rely on this constriant being held there.
237 // Therefore, we create a "primary key" column which exists purely for SQLite
238 // primary key purposes and is equal to the row number.
239 columns.push_back(
240 Column(columns.size(), "_primary_key", SqlValue::kLong, true));
241 primary_keys.emplace_back(columns.size() - 1);
242
243 return SqliteTable::Schema(std::move(columns), std::move(primary_keys));
244 }
245
CreateCursor()246 std::unique_ptr<SqliteTable::BaseCursor> CreatedViewFunction::CreateCursor() {
247 return std::unique_ptr<Cursor>(new Cursor(this));
248 }
249
BestIndex(const QueryConstraints & qc,BestIndexInfo * info)250 int CreatedViewFunction::BestIndex(const QueryConstraints& qc,
251 BestIndexInfo* info) {
252 // Only accept constraint sets where every input parameter has a value.
253 size_t seen_argument_constraints = 0;
254 for (size_t i = 0; i < qc.constraints().size(); ++i) {
255 const auto& cs = qc.constraints()[i];
256 seen_argument_constraints +=
257 IsArgumentColumn(static_cast<size_t>(cs.column));
258 }
259 if (seen_argument_constraints < prototype_.arguments.size())
260 return SQLITE_CONSTRAINT;
261
262 for (size_t i = 0; i < info->sqlite_omit_constraint.size(); ++i) {
263 size_t col = static_cast<size_t>(qc.constraints()[i].column);
264 if (IsArgumentColumn(col)) {
265 info->sqlite_omit_constraint[i] = true;
266 }
267 }
268 return SQLITE_OK;
269 }
270
Cursor(CreatedViewFunction * table)271 CreatedViewFunction::Cursor::Cursor(CreatedViewFunction* table)
272 : SqliteTable::BaseCursor(table), table_(table) {}
273
274 CreatedViewFunction::Cursor::~Cursor() = default;
275
Filter(const QueryConstraints & qc,sqlite3_value ** argv,FilterHistory)276 base::Status CreatedViewFunction::Cursor::Filter(const QueryConstraints& qc,
277 sqlite3_value** argv,
278 FilterHistory) {
279 PERFETTO_TP_TRACE(metatrace::Category::FUNCTION, "CREATE_VIEW_FUNCTION",
280 [this](metatrace::Record* r) {
281 r->AddArg("Function",
282 table_->prototype_.function_name.c_str());
283 });
284
285 auto col_to_arg_idx = [this](int col) {
286 return static_cast<uint32_t>(col) -
287 static_cast<uint32_t>(table_->return_values_.size());
288 };
289
290 size_t seen_argument_constraints = 0;
291 for (size_t i = 0; i < qc.constraints().size(); ++i) {
292 const auto& cs = qc.constraints()[i];
293
294 // Only consider argument columns (i.e. input parameters) as we're
295 // delegating the rest to SQLite.
296 if (!table_->IsArgumentColumn(static_cast<size_t>(cs.column)))
297 continue;
298
299 // We only support equality constraints as we're expecting "input arguments"
300 // to our "function".
301 if (!sqlite_utils::IsOpEq(cs.op)) {
302 return base::ErrStatus("%s: non-equality constraint passed",
303 table_->prototype_.function_name.c_str());
304 }
305
306 const auto& arg = table_->prototype_.arguments[col_to_arg_idx(cs.column)];
307 base::Status status = sqlite_utils::TypeCheckSqliteValue(
308 argv[i], sql_argument::TypeToSqlValueType(arg.type()),
309 sql_argument::TypeToHumanFriendlyString(arg.type()));
310 if (!status.ok()) {
311 return base::ErrStatus("%s: argument %s (index %zu) %s",
312 table_->prototype_.function_name.c_str(),
313 arg.name().c_str(), i, status.c_message());
314 }
315
316 seen_argument_constraints++;
317 }
318
319 // Verify that we saw one valid constriant for every input argument.
320 if (seen_argument_constraints < table_->prototype_.arguments.size()) {
321 return base::ErrStatus(
322 "%s: missing value for input argument. Saw %zu arguments but expected "
323 "%zu",
324 table_->prototype_.function_name.c_str(), seen_argument_constraints,
325 table_->prototype_.arguments.size());
326 }
327
328 // Prepare the SQL definition as a statement using SQLite.
329 // TODO(lalitm): see if we can reuse this prepared statement rather than
330 // creating it very time.
331 // TODO(lalitm): measure and implement whether it would be a good idea to
332 // forward constraints here when we build the nested query.
333 int ret = sqlite3_prepare_v2(table_->db_, table_->sql_defn_str_.data(),
334 static_cast<int>(table_->sql_defn_str_.size()),
335 &stmt_, nullptr);
336 scoped_stmt_.reset(stmt_);
337 PERFETTO_CHECK(ret == SQLITE_OK);
338
339 // Bind all the arguments to the appropriate places in the function.
340 for (size_t i = 0; i < qc.constraints().size(); ++i) {
341 const auto& cs = qc.constraints()[i];
342
343 // Don't deal with any constraints on the output parameters for simplicty.
344 // TODO(lalitm): reconsider this decision to allow more efficient queries:
345 // we would need to wrap the query in a SELECT * FROM (...) WHERE constraint
346 // like we do for SPAN JOIN.
347 if (!table_->IsArgumentColumn(static_cast<size_t>(cs.column)))
348 continue;
349
350 uint32_t index = col_to_arg_idx(cs.column);
351 PERFETTO_DCHECK(index < table_->prototype_.arguments.size());
352
353 const auto& arg = table_->prototype_.arguments[index];
354 auto status = MaybeBindArgument(stmt_, table_->prototype_.function_name,
355 arg, argv[i]);
356 RETURN_IF_ERROR(status);
357 }
358
359 // Reset the next call count - this is necessary because the same cursor
360 // can be used for multiple filter operations.
361 next_call_count_ = 0;
362 return Next();
363 }
364
Next()365 base::Status CreatedViewFunction::Cursor::Next() {
366 int ret = sqlite3_step(stmt_);
367 is_eof_ = ret == SQLITE_DONE;
368 next_call_count_++;
369 if (ret != SQLITE_ROW && ret != SQLITE_DONE) {
370 return base::ErrStatus(
371 "%s: SQLite error while stepping statement: %s",
372 table_->prototype_.function_name.c_str(),
373 sqlite_utils::FormatErrorMessage(stmt_, std::nullopt, table_->db_, ret)
374 .c_message());
375 }
376 return base::OkStatus();
377 }
378
Eof()379 bool CreatedViewFunction::Cursor::Eof() {
380 return is_eof_;
381 }
382
Column(sqlite3_context * ctx,int i)383 base::Status CreatedViewFunction::Cursor::Column(sqlite3_context* ctx, int i) {
384 size_t idx = static_cast<size_t>(i);
385 if (table_->IsReturnValueColumn(idx)) {
386 sqlite3_result_value(ctx, sqlite3_column_value(stmt_, i));
387 } else if (table_->IsArgumentColumn(idx)) {
388 // TODO(lalitm): it may be more appropriate to keep a note of the arguments
389 // which we passed in and return them here. Not doing this to because it
390 // doesn't seem necessary for any useful thing but something which may need
391 // to be changed in the future.
392 sqlite3_result_null(ctx);
393 } else {
394 PERFETTO_DCHECK(table_->IsPrimaryKeyColumn(idx));
395 sqlite3_result_int(ctx, next_call_count_);
396 }
397 return base::OkStatus();
398 }
399
400 } // namespace
401
Run(CreateViewFunction::Context * ctx,size_t argc,sqlite3_value ** argv,SqlValue &,Destructors &)402 base::Status CreateViewFunction::Run(CreateViewFunction::Context* ctx,
403 size_t argc,
404 sqlite3_value** argv,
405 SqlValue&,
406 Destructors&) {
407 if (argc != 3) {
408 return base::ErrStatus(
409 "CREATE_VIEW_FUNCTION: invalid number of args; expected %u, received "
410 "%zu",
411 3u, argc);
412 }
413
414 sqlite3_value* prototype_value = argv[0];
415 sqlite3_value* return_prototype_value = argv[1];
416 sqlite3_value* sql_defn_value = argv[2];
417
418 // Type check all the arguments.
419 {
420 auto type_check = [prototype_value](sqlite3_value* value,
421 SqlValue::Type type, const char* desc) {
422 base::Status status = sqlite_utils::TypeCheckSqliteValue(value, type);
423 if (!status.ok()) {
424 return base::ErrStatus("CREATE_VIEW_FUNCTION[prototype=%s]: %s %s",
425 sqlite3_value_text(prototype_value), desc,
426 status.c_message());
427 }
428 return base::OkStatus();
429 };
430
431 RETURN_IF_ERROR(type_check(prototype_value, SqlValue::Type::kString,
432 "function prototype (first argument)"));
433 RETURN_IF_ERROR(type_check(return_prototype_value, SqlValue::Type::kString,
434 "return prototype (second argument)"));
435 RETURN_IF_ERROR(type_check(sql_defn_value, SqlValue::Type::kString,
436 "SQL definition (third argument)"));
437 }
438
439 // Extract the arguments from the value wrappers.
440 auto extract_string = [](sqlite3_value* value) -> const char* {
441 return reinterpret_cast<const char*>(sqlite3_value_text(value));
442 };
443 const char* prototype_str = extract_string(prototype_value);
444 const char* return_prototype_str = extract_string(return_prototype_value);
445 const char* sql_defn_str = extract_string(sql_defn_value);
446
447 base::StringView function_name;
448 RETURN_IF_ERROR(ParseFunctionName(prototype_str, function_name));
449
450 static constexpr char kSqlTemplate[] = R"""(
451 DROP TABLE IF EXISTS %s;
452
453 CREATE VIRTUAL TABLE %s
454 USING INTERNAL_VIEW_FUNCTION_IMPL('%s', '%s', '%s');
455 )""";
456 std::string function_name_str = function_name.ToStdString();
457
458 ScopedSqliteString errmsg;
459 char* errmsg_raw = nullptr;
460 int ret;
461
462 NullTermStringView sql_defn(sql_defn_str);
463 if (sql_defn.size() < 512) {
464 base::StackString<1024> sql(kSqlTemplate, function_name_str.c_str(),
465 function_name_str.c_str(), prototype_str,
466 return_prototype_str, sql_defn_str);
467 ret = sqlite3_exec(ctx->db, sql.c_str(), nullptr, nullptr, &errmsg_raw);
468 } else {
469 std::vector<char> formatted_sql(sql_defn.size() + 1024);
470 base::SprintfTrunc(formatted_sql.data(), formatted_sql.size(), kSqlTemplate,
471 function_name_str.c_str(), function_name_str.c_str(),
472 prototype_str, return_prototype_str, sql_defn_str);
473 ret = sqlite3_exec(ctx->db, formatted_sql.data(), nullptr, nullptr,
474 &errmsg_raw);
475 }
476
477 errmsg.reset(errmsg_raw);
478 if (ret != SQLITE_OK)
479 return base::ErrStatus("%s", errmsg.get());
480
481 // CREATE_VIEW_FUNCTION doesn't have a return value so just don't sent |out|.
482 return base::OkStatus();
483 }
484
RegisterCreateViewFunctionModule(SqliteEngine * engine)485 void RegisterCreateViewFunctionModule(SqliteEngine* engine) {
486 engine->RegisterVirtualTableModule<CreatedViewFunction>(
487 "internal_view_function_impl", nullptr,
488 SqliteTable::TableType::kExplicitCreate, false);
489 }
490
491 } // namespace trace_processor
492 } // namespace perfetto
493