1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_ENGINE_H_
18 #define SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_ENGINE_H_
19
20 #include <cstddef>
21 #include <cstdint>
22 #include <memory>
23 #include <string>
24 #include <string_view>
25 #include <utility>
26 #include <vector>
27
28 #include "perfetto/base/logging.h"
29 #include "perfetto/base/status.h"
30 #include "perfetto/ext/base/flat_hash_map.h"
31 #include "perfetto/ext/base/status_or.h"
32 #include "perfetto/trace_processor/basic_types.h"
33 #include "src/trace_processor/containers/string_pool.h"
34 #include "src/trace_processor/db/runtime_table.h"
35 #include "src/trace_processor/db/table.h"
36 #include "src/trace_processor/perfetto_sql/engine/runtime_table_function.h"
37 #include "src/trace_processor/perfetto_sql/intrinsics/functions/sql_function.h"
38 #include "src/trace_processor/perfetto_sql/intrinsics/table_functions/static_table_function.h"
39 #include "src/trace_processor/perfetto_sql/parser/function_util.h"
40 #include "src/trace_processor/perfetto_sql/parser/perfetto_sql_parser.h"
41 #include "src/trace_processor/perfetto_sql/preprocessor/perfetto_sql_preprocessor.h"
42 #include "src/trace_processor/sqlite/bindings/sqlite_result.h"
43 #include "src/trace_processor/sqlite/bindings/sqlite_window_function.h"
44 #include "src/trace_processor/sqlite/db_sqlite_table.h"
45 #include "src/trace_processor/sqlite/sql_source.h"
46 #include "src/trace_processor/sqlite/sqlite_engine.h"
47 #include "src/trace_processor/sqlite/sqlite_utils.h"
48 #include "src/trace_processor/util/sql_argument.h"
49 #include "src/trace_processor/util/sql_modules.h"
50
51 namespace perfetto::trace_processor {
52
53 // Intermediary class which translates high-level concepts and algorithms used
54 // in trace processor into lower-level concepts and functions can be understood
55 // by and executed against SQLite.
56 class PerfettoSqlEngine {
57 public:
58 struct ExecutionStats {
59 uint32_t column_count = 0;
60 uint32_t statement_count = 0;
61 uint32_t statement_count_with_output = 0;
62 };
63 struct ExecutionResult {
64 SqliteEngine::PreparedStatement stmt;
65 ExecutionStats stats;
66 };
67
68 PerfettoSqlEngine(StringPool* pool, bool enable_extra_checks);
69
70 // Executes all the statements in |sql| and returns a |ExecutionResult|
71 // object. The metadata will reference all the statements executed and the
72 // |ScopedStmt| be empty.
73 //
74 // Returns an error if the execution of any statement failed or if there was
75 // no valid SQL to run.
76 base::StatusOr<ExecutionStats> Execute(SqlSource sql);
77
78 // Executes all the statements in |sql| fully until the final statement and
79 // returns a |ExecutionResult| object containing a |ScopedStmt| for the final
80 // statement (which has been stepped once) and metadata about all statements
81 // executed.
82 //
83 // Returns an error if the execution of any statement failed or if there was
84 // no valid SQL to run.
85 base::StatusOr<ExecutionResult> ExecuteUntilLastStatement(SqlSource sql);
86
87 // Prepares a single SQLite statement in |sql| and returns a
88 // |PreparedStatement| object.
89 //
90 // Returns an error if the preparation of the statement failed or if there was
91 // no valid SQL to run.
92 base::StatusOr<SqliteEngine::PreparedStatement> PrepareSqliteStatement(
93 SqlSource sql);
94
95 // Registers a trace processor C++ function to be runnable from SQL.
96 //
97 // The format of the function is given by the |SqlFunction|.
98 //
99 // |name|: name of the function in SQL.
100 // |argc|: number of arguments for this function. This can be -1 if
101 // the number of arguments is variable.
102 // |ctx|: context object for the function (see SqlFunction::Run);
103 // this object *must* outlive the function so should likely
104 // be either static or scoped to the lifetime of
105 // TraceProcessor.
106 // |deterministic|: whether this function has deterministic output given the
107 // same set of arguments.
108 template <typename Function = SqlFunction>
109 base::Status RegisterStaticFunction(const char* name,
110 int argc,
111 typename Function::Context* ctx,
112 bool deterministic = true);
113
114 // Registers a trace processor C++ function to be runnable from SQL.
115 //
116 // This function is the same as the above except allows a unique_ptr to be
117 // passed for the context; this allows for SQLite to manage the lifetime of
118 // this pointer instead of the essentially static requirement of the context
119 // pointer above.
120 template <typename Function>
121 base::Status RegisterStaticFunction(
122 const char* name,
123 int argc,
124 std::unique_ptr<typename Function::Context> ctx,
125 bool deterministic = true);
126
127 // Registers a trace processor C++ function to be runnable from SQL.
128 //
129 // The format of the function is given by the |SqliteFunction|.
130 //
131 // |ctx|: context object for the function; this object *must*
132 // outlive the function so should likely be either static or
133 // scoped to the lifetime of TraceProcessor.
134 // |deterministic|: whether this function has deterministic output given the
135 // same set of arguments.
136 template <typename Function>
137 base::Status RegisterSqliteFunction(typename Function::UserDataContext* ctx,
138 bool deterministic = true);
139 template <typename Function>
140 base::Status RegisterSqliteFunction(
141 std::unique_ptr<typename Function::UserDataContext> ctx,
142 bool deterministic = true);
143
144 // Registers a trace processor C++ aggregate function to be runnable from SQL.
145 //
146 // The format of the function is given by the |SqliteAggregateFunction|.
147 //
148 // |ctx|: context object for the function; this object *must*
149 // outlive the function so should likely be either static or
150 // scoped to the lifetime of TraceProcessor.
151 // |deterministic|: whether this function has deterministic output given the
152 // same set of arguments.
153 template <typename Function>
154 base::Status RegisterSqliteAggregateFunction(
155 typename Function::UserDataContext* ctx,
156 bool deterministic = true);
157
158 // Registers a trace processor C++ window function to be runnable from SQL.
159 //
160 // The format of the function is given by the |SqliteWindowFunction|.
161 //
162 // |name|: name of the function in SQL.
163 // |argc|: number of arguments for this function. This can be -1 if
164 // the number of arguments is variable.
165 // |ctx|: context object for the function; this object *must*
166 // outlive the function so should likely be either static or
167 // scoped to the lifetime of TraceProcessor.
168 // |deterministic|: whether this function has deterministic output given the
169 // same set of arguments.
170 template <typename Function = SqliteWindowFunction>
171 base::Status RegisterSqliteWindowFunction(const char* name,
172 int argc,
173 typename Function::Context* ctx,
174 bool deterministic = true);
175
176 // Registers a function with the prototype |prototype| which returns a value
177 // of |return_type| and is implemented by executing the SQL statement |sql|.
178 base::Status RegisterRuntimeFunction(bool replace,
179 const FunctionPrototype& prototype,
180 sql_argument::Type return_type,
181 SqlSource sql);
182
183 // Enables memoization for the given SQL function.
184 base::Status EnableSqlFunctionMemoization(const std::string& name);
185
186 // Registers a trace processor C++ table with SQLite with an SQL name of
187 // |name|.
188 void RegisterStaticTable(Table*,
189 const std::string& name,
190 Table::Schema schema);
191
192 // Registers a trace processor C++ table function with SQLite.
193 void RegisterStaticTableFunction(std::unique_ptr<StaticTableFunction> fn);
194
sqlite_engine()195 SqliteEngine* sqlite_engine() { return engine_.get(); }
196
197 // Makes new SQL package available to include.
RegisterPackage(const std::string & name,sql_modules::RegisteredPackage package)198 void RegisterPackage(const std::string& name,
199 sql_modules::RegisteredPackage package) {
200 packages_.Erase(name);
201 packages_.Insert(name, std::move(package));
202 }
203
204 // Fetches registered SQL package.
FindPackage(const std::string & name)205 sql_modules::RegisteredPackage* FindPackage(const std::string& name) {
206 return packages_.Find(name);
207 }
208
209 // Returns the number of objects (tables, views, functions etc) registered
210 // with SQLite.
SqliteRegisteredObjectCount()211 uint64_t SqliteRegisteredObjectCount() {
212 // This query will return all the tables, views, indexes and table functions
213 // SQLite knows about.
214 constexpr char kAllTablesQuery[] =
215 "SELECT COUNT() FROM (SELECT * FROM sqlite_master "
216 "UNION ALL SELECT * FROM sqlite_temp_master)";
217 auto stmt = ExecuteUntilLastStatement(
218 SqlSource::FromTraceProcessorImplementation(kAllTablesQuery));
219 if (!stmt.ok()) {
220 PERFETTO_FATAL("%s", stmt.status().c_message());
221 }
222 uint32_t query_count =
223 static_cast<uint32_t>(sqlite3_column_int(stmt->stmt.sqlite_stmt(), 0));
224 PERFETTO_CHECK(!stmt->stmt.Step());
225 PERFETTO_CHECK(stmt->stmt.status().ok());
226
227 // The missing objects from the above query are static functions, runtime
228 // functions and macros. Add those in now.
229 return query_count + static_function_count_ +
230 static_window_function_count_ + static_aggregate_function_count_ +
231 runtime_function_count_ + macros_.size();
232 }
233
234 // Find table (Static or Runtime) registered with engine with provided name.
GetTableOrNull(std::string_view name)235 const Table* GetTableOrNull(std::string_view name) const {
236 if (const auto* r = GetRuntimeTableOrNull(name); r) {
237 return r;
238 }
239 return GetStaticTableOrNull(name);
240 }
241
242 private:
243 base::Status ExecuteCreateFunction(const PerfettoSqlParser::CreateFunction&);
244
245 base::Status ExecuteInclude(const PerfettoSqlParser::Include&,
246 const PerfettoSqlParser& parser);
247
248 // Creates a runtime table and registers it with SQLite.
249 base::Status ExecuteCreateTable(
250 const PerfettoSqlParser::CreateTable& create_table);
251
252 base::Status ExecuteCreateView(const PerfettoSqlParser::CreateView&);
253
254 base::Status ExecuteCreateMacro(const PerfettoSqlParser::CreateMacro&);
255
256 base::Status ExecuteCreateIndex(const PerfettoSqlParser::CreateIndex&);
257
258 base::Status ExecuteDropIndex(const PerfettoSqlParser::DropIndex&);
259
260 enum class CreateTableType {
261 kCreateTable,
262 // For now, bytes columns are not supported in CREATE PERFETTO TABLE,
263 // but supported in CREATE PERFETTO VIEW, so we skip them when validating
264 // views.
265 kValidateOnly
266 };
267 // |effective_schema| should have been normalised and its column order
268 // should match |column_names|.
269 base::StatusOr<std::unique_ptr<RuntimeTable>> CreateTableImpl(
270 const char* tag,
271 const std::string& name,
272 SqliteEngine::PreparedStatement source,
273 const std::vector<std::string>& column_names,
274 const std::vector<sql_argument::ArgumentDefinition>& effective_schema,
275 CreateTableType type);
276
277 template <typename Function>
278 base::Status RegisterFunctionWithSqlite(
279 const char* name,
280 int argc,
281 std::unique_ptr<typename Function::Context> ctx,
282 bool deterministic = true);
283
284 // Given a package and a key, include the correct file(s) from the package.
285 // The key can contain a wildcard to include all files in the module with the
286 // matching prefix.
287 base::Status IncludePackageImpl(sql_modules::RegisteredPackage&,
288 const std::string& key,
289 const PerfettoSqlParser&);
290
291 // Include a given module.
292 base::Status IncludeModuleImpl(sql_modules::RegisteredPackage::ModuleFile&,
293 const std::string& key,
294 const PerfettoSqlParser&);
295
296 // Find table (Static or Runtime) registered with engine with provided name.
GetTableOrNull(std::string_view name)297 Table* GetTableOrNull(std::string_view name) {
298 if (auto* maybe_runtime = GetRuntimeTableOrNull(name); maybe_runtime) {
299 return maybe_runtime;
300 }
301 return GetStaticTableOrNull(name);
302 }
303
304 // Find RuntimeTable registered with engine with provided name.
305 RuntimeTable* GetRuntimeTableOrNull(std::string_view);
306
307 // Find static table registered with engine with provided name.
308 Table* GetStaticTableOrNull(std::string_view);
309
310 // Find RuntimeTable registered with engine with provided name.
311 const RuntimeTable* GetRuntimeTableOrNull(std::string_view) const;
312
313 // Find static table registered with engine with provided name.
314 const Table* GetStaticTableOrNull(std::string_view) const;
315
316 StringPool* pool_ = nullptr;
317 // If true, engine will perform additional consistency checks when e.g.
318 // creating tables and views.
319 const bool enable_extra_checks_;
320
321 uint64_t static_function_count_ = 0;
322 uint64_t static_aggregate_function_count_ = 0;
323 uint64_t static_window_function_count_ = 0;
324 uint64_t runtime_function_count_ = 0;
325
326 RuntimeTableFunctionModule::Context* runtime_table_fn_context_ = nullptr;
327 DbSqliteModule::Context* runtime_table_context_ = nullptr;
328 DbSqliteModule::Context* static_table_context_ = nullptr;
329 DbSqliteModule::Context* static_table_fn_context_ = nullptr;
330 base::FlatHashMap<std::string, sql_modules::RegisteredPackage> packages_;
331 base::FlatHashMap<std::string, PerfettoSqlPreprocessor::Macro> macros_;
332 std::unique_ptr<SqliteEngine> engine_;
333 };
334
335 // The rest of this file is just implementation details which we need
336 // in the header file because it is templated code. We separate it out
337 // like this to keep the API people actually care about easy to read.
338
339 namespace perfetto_sql_internal {
340
341 // RAII type to call Function::Cleanup when destroyed.
342 template <typename Function>
343 struct ScopedCleanup {
344 typename Function::Context* ctx;
~ScopedCleanupScopedCleanup345 ~ScopedCleanup() { Function::Cleanup(ctx); }
346 };
347
348 template <typename Function>
WrapSqlFunction(sqlite3_context * ctx,int argc,sqlite3_value ** argv)349 void WrapSqlFunction(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
350 using Context = typename Function::Context;
351 auto* ud = static_cast<Context*>(sqlite3_user_data(ctx));
352
353 ScopedCleanup<Function> scoped_cleanup{ud};
354 SqlValue value{};
355 SqlFunction::Destructors destructors{};
356 base::Status status =
357 Function::Run(ud, static_cast<size_t>(argc), argv, value, destructors);
358 if (!status.ok()) {
359 sqlite::result::Error(ctx, status.c_message());
360 return;
361 }
362
363 if (Function::kVoidReturn) {
364 if (!value.is_null()) {
365 sqlite::result::Error(ctx, "void SQL function returned value");
366 return;
367 }
368
369 // If the function doesn't want to return anything, set the "VOID"
370 // pointer type to a non-null value. Note that because of the weird
371 // way |sqlite3_value_pointer| works, we need to set some value even
372 // if we don't actually read it - just set it to a pointer to an empty
373 // string for this reason.
374 static char kVoidValue[] = "";
375 sqlite::result::StaticPointer(ctx, kVoidValue, "VOID");
376 } else {
377 sqlite::utils::ReportSqlValue(ctx, value, destructors.string_destructor,
378 destructors.bytes_destructor);
379 }
380
381 status = Function::VerifyPostConditions(ud);
382 if (!status.ok()) {
383 sqlite::result::Error(ctx, status.c_message());
384 return;
385 }
386 }
387
388 } // namespace perfetto_sql_internal
389
390 template <typename Function>
RegisterStaticFunction(const char * name,int argc,typename Function::Context * ctx,bool deterministic)391 base::Status PerfettoSqlEngine::RegisterStaticFunction(
392 const char* name,
393 int argc,
394 typename Function::Context* ctx,
395 bool deterministic) {
396 // Metric proto builder functions can be reregistered: don't double count when
397 // this happens.
398 if (!engine_->GetFunctionContext(name, argc)) {
399 static_function_count_++;
400 }
401 return engine_->RegisterFunction(
402 name, argc, perfetto_sql_internal::WrapSqlFunction<Function>, ctx,
403 nullptr, deterministic);
404 }
405
406 template <typename Function>
RegisterSqliteFunction(typename Function::UserDataContext * ctx,bool deterministic)407 base::Status PerfettoSqlEngine::RegisterSqliteFunction(
408 typename Function::UserDataContext* ctx,
409 bool deterministic) {
410 static_function_count_++;
411 return engine_->RegisterFunction(Function::kName, Function::kArgCount,
412 Function::Step, ctx, nullptr, deterministic);
413 }
414
415 template <typename Function>
RegisterSqliteFunction(std::unique_ptr<typename Function::UserDataContext> ctx,bool deterministic)416 base::Status PerfettoSqlEngine::RegisterSqliteFunction(
417 std::unique_ptr<typename Function::UserDataContext> ctx,
418 bool deterministic) {
419 static_function_count_++;
420 return engine_->RegisterFunction(
421 Function::kName, Function::kArgCount, Function::Step, ctx.release(),
422 [](void* ptr) {
423 std::unique_ptr<typename Function::UserDataContext>(
424 static_cast<typename Function::UserDataContext*>(ptr));
425 },
426 deterministic);
427 }
428
429 template <typename Function>
RegisterSqliteAggregateFunction(typename Function::UserDataContext * ctx,bool deterministic)430 base::Status PerfettoSqlEngine::RegisterSqliteAggregateFunction(
431 typename Function::UserDataContext* ctx,
432 bool deterministic) {
433 static_aggregate_function_count_++;
434 return engine_->RegisterAggregateFunction(
435 Function::kName, Function::kArgCount, Function::Step, Function::Final,
436 ctx, nullptr, deterministic);
437 }
438
439 template <typename Function>
RegisterSqliteWindowFunction(const char * name,int argc,typename Function::Context * ctx,bool deterministic)440 base::Status PerfettoSqlEngine::RegisterSqliteWindowFunction(
441 const char* name,
442 int argc,
443 typename Function::Context* ctx,
444 bool deterministic) {
445 static_window_function_count_++;
446 return engine_->RegisterWindowFunction(
447 name, argc, Function::Step, Function::Inverse, Function::Value,
448 Function::Final, ctx, nullptr, deterministic);
449 }
450
451 template <typename Function>
RegisterStaticFunction(const char * name,int argc,std::unique_ptr<typename Function::Context> ctx,bool deterministic)452 base::Status PerfettoSqlEngine::RegisterStaticFunction(
453 const char* name,
454 int argc,
455 std::unique_ptr<typename Function::Context> ctx,
456 bool deterministic) {
457 // Metric proto builder functions can be reregistered: don't double count when
458 // this happens.
459 if (!engine_->GetFunctionContext(name, argc)) {
460 static_function_count_++;
461 }
462 return RegisterFunctionWithSqlite<Function>(name, argc, std::move(ctx),
463 deterministic);
464 }
465
466 template <typename Function>
RegisterFunctionWithSqlite(const char * name,int argc,std::unique_ptr<typename Function::Context> ctx,bool deterministic)467 base::Status PerfettoSqlEngine::RegisterFunctionWithSqlite(
468 const char* name,
469 int argc,
470 std::unique_ptr<typename Function::Context> ctx,
471 bool deterministic) {
472 auto ctx_destructor = [](void* ptr) {
473 delete static_cast<typename Function::Context*>(ptr);
474 };
475 return engine_->RegisterFunction(
476 name, argc, perfetto_sql_internal::WrapSqlFunction<Function>,
477 ctx.release(), ctx_destructor, deterministic);
478 }
479
480 } // namespace perfetto::trace_processor
481
482 #endif // SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_ENGINE_H_
483