1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_ENGINE_H_
18 #define SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_ENGINE_H_
19
20 #include <cstdint>
21 #include <memory>
22 #include <string>
23 #include <string_view>
24 #include <utility>
25 #include <vector>
26
27 #include "perfetto/base/logging.h"
28 #include "perfetto/base/status.h"
29 #include "perfetto/ext/base/flat_hash_map.h"
30 #include "perfetto/ext/base/status_or.h"
31 #include "perfetto/trace_processor/basic_types.h"
32 #include "src/trace_processor/containers/string_pool.h"
33 #include "src/trace_processor/db/runtime_table.h"
34 #include "src/trace_processor/db/table.h"
35 #include "src/trace_processor/perfetto_sql/engine/function_util.h"
36 #include "src/trace_processor/perfetto_sql/engine/perfetto_sql_parser.h"
37 #include "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.h"
38 #include "src/trace_processor/perfetto_sql/engine/runtime_table_function.h"
39 #include "src/trace_processor/perfetto_sql/intrinsics/functions/sql_function.h"
40 #include "src/trace_processor/perfetto_sql/intrinsics/table_functions/static_table_function.h"
41 #include "src/trace_processor/sqlite/bindings/sqlite_result.h"
42 #include "src/trace_processor/sqlite/bindings/sqlite_window_function.h"
43 #include "src/trace_processor/sqlite/db_sqlite_table.h"
44 #include "src/trace_processor/sqlite/sql_source.h"
45 #include "src/trace_processor/sqlite/sqlite_engine.h"
46 #include "src/trace_processor/sqlite/sqlite_utils.h"
47 #include "src/trace_processor/util/sql_argument.h"
48 #include "src/trace_processor/util/sql_modules.h"
49
50 namespace perfetto::trace_processor {
51
52 // Intermediary class which translates high-level concepts and algorithms used
53 // in trace processor into lower-level concepts and functions can be understood
54 // by and executed against SQLite.
55 class PerfettoSqlEngine {
56 public:
57 struct ExecutionStats {
58 uint32_t column_count = 0;
59 uint32_t statement_count = 0;
60 uint32_t statement_count_with_output = 0;
61 };
62 struct ExecutionResult {
63 SqliteEngine::PreparedStatement stmt;
64 ExecutionStats stats;
65 };
66
67 explicit PerfettoSqlEngine(StringPool* pool);
68
69 // Executes all the statements in |sql| and returns a |ExecutionResult|
70 // object. The metadata will reference all the statements executed and the
71 // |ScopedStmt| be empty.
72 //
73 // Returns an error if the execution of any statement failed or if there was
74 // no valid SQL to run.
75 base::StatusOr<ExecutionStats> Execute(SqlSource sql);
76
77 // Executes all the statements in |sql| fully until the final statement and
78 // returns a |ExecutionResult| object containing a |ScopedStmt| for the final
79 // statement (which has been stepped once) and metadata about all statements
80 // executed.
81 //
82 // Returns an error if the execution of any statement failed or if there was
83 // no valid SQL to run.
84 base::StatusOr<ExecutionResult> ExecuteUntilLastStatement(SqlSource sql);
85
86 // Registers a trace processor C++ function to be runnable from SQL.
87 //
88 // The format of the function is given by the |SqlFunction|.
89 //
90 // |name|: name of the function in SQL.
91 // |argc|: number of arguments for this function. This can be -1 if
92 // the number of arguments is variable.
93 // |ctx|: context object for the function (see SqlFunction::Run);
94 // this object *must* outlive the function so should likely
95 // be either static or scoped to the lifetime of
96 // TraceProcessor.
97 // |deterministic|: whether this function has deterministic output given the
98 // same set of arguments.
99 template <typename Function = SqlFunction>
100 base::Status RegisterStaticFunction(const char* name,
101 int argc,
102 typename Function::Context* ctx,
103 bool deterministic = true);
104
105 // Registers a trace processor C++ function to be runnable from SQL.
106 //
107 // This function is the same as the above except allows a unique_ptr to be
108 // passed for the context; this allows for SQLite to manage the lifetime of
109 // this pointer instead of the essentially static requirement of the context
110 // pointer above.
111 template <typename Function>
112 base::Status RegisterStaticFunction(
113 const char* name,
114 int argc,
115 std::unique_ptr<typename Function::Context> ctx,
116 bool deterministic = true);
117
118 // Registers a trace processor C++ aggregate function to be runnable from SQL.
119 //
120 // The format of the function is given by the |SqliteAggregateFunction|.
121 //
122 // |ctx|: context object for the function; this object *must*
123 // outlive the function so should likely be either static or
124 // scoped to the lifetime of TraceProcessor.
125 // |deterministic|: whether this function has deterministic output given the
126 // same set of arguments.
127 template <typename Function>
128 base::Status RegisterSqliteAggregateFunction(
129 typename Function::UserDataContext* ctx,
130 bool deterministic = true);
131
132 // Registers a trace processor C++ window function to be runnable from SQL.
133 //
134 // The format of the function is given by the |SqliteWindowFunction|.
135 //
136 // |name|: name of the function in SQL.
137 // |argc|: number of arguments for this function. This can be -1 if
138 // the number of arguments is variable.
139 // |ctx|: context object for the function; this object *must*
140 // outlive the function so should likely be either static or
141 // scoped to the lifetime of TraceProcessor.
142 // |deterministic|: whether this function has deterministic output given the
143 // same set of arguments.
144 template <typename Function = SqliteWindowFunction>
145 base::Status RegisterSqliteWindowFunction(const char* name,
146 int argc,
147 typename Function::Context* ctx,
148 bool deterministic = true);
149
150 // Registers a function with the prototype |prototype| which returns a value
151 // of |return_type| and is implemented by executing the SQL statement |sql|.
152 base::Status RegisterRuntimeFunction(bool replace,
153 const FunctionPrototype& prototype,
154 const std::string& return_type,
155 SqlSource sql);
156
157 // Enables memoization for the given SQL function.
158 base::Status EnableSqlFunctionMemoization(const std::string& name);
159
160 // Registers a trace processor C++ table with SQLite with an SQL name of
161 // |name|.
162 void RegisterStaticTable(const Table&,
163 const std::string& name,
164 Table::Schema schema);
165
166 // Registers a trace processor C++ table function with SQLite.
167 void RegisterStaticTableFunction(std::unique_ptr<StaticTableFunction> fn);
168
sqlite_engine()169 SqliteEngine* sqlite_engine() { return engine_.get(); }
170
171 // Makes new SQL module available to import.
RegisterModule(const std::string & name,sql_modules::RegisteredModule module)172 void RegisterModule(const std::string& name,
173 sql_modules::RegisteredModule module) {
174 modules_.Erase(name);
175 modules_.Insert(name, std::move(module));
176 }
177
178 // Fetches registered SQL module.
FindModule(const std::string & name)179 sql_modules::RegisteredModule* FindModule(const std::string& name) {
180 return modules_.Find(name);
181 }
182
183 // Returns the number of objects (tables, views, functions etc) registered
184 // with SQLite.
SqliteRegisteredObjectCount()185 uint64_t SqliteRegisteredObjectCount() {
186 // This query will return all the tables, views, indexes and table functions
187 // SQLite knows about.
188 constexpr char kAllTablesQuery[] =
189 "SELECT COUNT() FROM (SELECT * FROM sqlite_master "
190 "UNION ALL SELECT * FROM sqlite_temp_master)";
191 auto stmt = ExecuteUntilLastStatement(
192 SqlSource::FromTraceProcessorImplementation(kAllTablesQuery));
193 PERFETTO_CHECK(stmt.ok());
194 uint32_t query_count =
195 static_cast<uint32_t>(sqlite3_column_int(stmt->stmt.sqlite_stmt(), 0));
196 PERFETTO_CHECK(!stmt->stmt.Step());
197 PERFETTO_CHECK(stmt->stmt.status().ok());
198
199 // The missing objects from the above query are static functions, runtime
200 // functions and macros. Add those in now.
201 return query_count + static_function_count_ +
202 static_window_function_count_ + static_aggregate_function_count_ +
203 runtime_function_count_ + macros_.size();
204 }
205
206 // Find static table (Static or Runtime) registered with engine with provided
207 // name.
GetTableOrNull(std::string_view name)208 const Table* GetTableOrNull(std::string_view name) const {
209 if (auto maybe_runtime = GetRuntimeTableOrNull(name); maybe_runtime) {
210 return maybe_runtime;
211 }
212 return GetStaticTableOrNull(name);
213 }
214
215 // Find RuntimeTable registered with engine with provided name.
216 const RuntimeTable* GetRuntimeTableOrNull(std::string_view) const;
217
218 // Find static table registered with engine with provided name.
219 const Table* GetStaticTableOrNull(std::string_view) const;
220
221 private:
222 base::Status ExecuteCreateFunction(const PerfettoSqlParser::CreateFunction&);
223
224 base::Status ExecuteInclude(const PerfettoSqlParser::Include&,
225 const PerfettoSqlParser& parser);
226
227 // Creates a runtime table and registers it with SQLite.
228 base::Status ExecuteCreateTable(
229 const PerfettoSqlParser::CreateTable& create_table);
230
231 base::Status ExecuteCreateView(const PerfettoSqlParser::CreateView&);
232
233 base::Status ExecuteCreateMacro(const PerfettoSqlParser::CreateMacro&);
234
235 template <typename Function>
236 base::Status RegisterFunctionWithSqlite(
237 const char* name,
238 int argc,
239 std::unique_ptr<typename Function::Context> ctx,
240 bool deterministic = true);
241
242 // Get the column names from a statement.
243 // |operator_name| is used in the error message if the statement is invalid.
244 static base::StatusOr<std::vector<std::string>>
245 GetColumnNamesFromSelectStatement(const SqliteEngine::PreparedStatement& stmt,
246 const char* tag);
247
248 // Validates that the column names in |column_names| match the |schema|.
249 // |operator_name| is used in the error message if the statement is invalid.
250 static base::Status ValidateColumnNames(
251 const std::vector<std::string>& column_names,
252 const std::vector<sql_argument::ArgumentDefinition>& schema,
253 const char* operator_name);
254
255 // Given a module and a key, include the correct file(s) from the module.
256 // The key can contain a wildcard to include all files in the module with the
257 // matching prefix.
258 base::Status IncludeModuleImpl(sql_modules::RegisteredModule& module,
259 const std::string& key,
260 const PerfettoSqlParser& parser);
261
262 // Import a given file.
263 base::Status IncludeFileImpl(
264 sql_modules::RegisteredModule::ModuleFile& module,
265 const std::string& key,
266 const PerfettoSqlParser& parser);
267
268 StringPool* pool_ = nullptr;
269
270 uint64_t static_function_count_ = 0;
271 uint64_t static_aggregate_function_count_ = 0;
272 uint64_t static_window_function_count_ = 0;
273 uint64_t runtime_function_count_ = 0;
274
275 RuntimeTableFunctionModule::Context* runtime_table_fn_context_ = nullptr;
276 DbSqliteModule::Context* runtime_table_context_ = nullptr;
277 DbSqliteModule::Context* static_table_context_ = nullptr;
278 DbSqliteModule::Context* static_table_fn_context_ = nullptr;
279 base::FlatHashMap<std::string, sql_modules::RegisteredModule> modules_;
280 base::FlatHashMap<std::string, PerfettoSqlPreprocessor::Macro> macros_;
281 std::unique_ptr<SqliteEngine> engine_;
282 };
283
284 // The rest of this file is just implementation details which we need
285 // in the header file because it is templated code. We separate it out
286 // like this to keep the API people actually care about easy to read.
287
288 namespace perfetto_sql_internal {
289
290 // RAII type to call Function::Cleanup when destroyed.
291 template <typename Function>
292 struct ScopedCleanup {
293 typename Function::Context* ctx;
~ScopedCleanupScopedCleanup294 ~ScopedCleanup() { Function::Cleanup(ctx); }
295 };
296
297 template <typename Function>
WrapSqlFunction(sqlite3_context * ctx,int argc,sqlite3_value ** argv)298 void WrapSqlFunction(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
299 using Context = typename Function::Context;
300 auto* ud = static_cast<Context*>(sqlite3_user_data(ctx));
301
302 ScopedCleanup<Function> scoped_cleanup{ud};
303 SqlValue value{};
304 SqlFunction::Destructors destructors{};
305 base::Status status =
306 Function::Run(ud, static_cast<size_t>(argc), argv, value, destructors);
307 if (!status.ok()) {
308 sqlite::result::Error(ctx, status.c_message());
309 return;
310 }
311
312 if (Function::kVoidReturn) {
313 if (!value.is_null()) {
314 sqlite::result::Error(ctx, "void SQL function returned value");
315 return;
316 }
317
318 // If the function doesn't want to return anything, set the "VOID"
319 // pointer type to a non-null value. Note that because of the weird
320 // way |sqlite3_value_pointer| works, we need to set some value even
321 // if we don't actually read it - just set it to a pointer to an empty
322 // string for this reason.
323 static char kVoidValue[] = "";
324 sqlite::result::StaticPointer(ctx, kVoidValue, "VOID");
325 } else {
326 sqlite::utils::ReportSqlValue(ctx, value, destructors.string_destructor,
327 destructors.bytes_destructor);
328 }
329
330 status = Function::VerifyPostConditions(ud);
331 if (!status.ok()) {
332 sqlite::result::Error(ctx, status.c_message());
333 return;
334 }
335 }
336
337 } // namespace perfetto_sql_internal
338
339 template <typename Function>
RegisterStaticFunction(const char * name,int argc,typename Function::Context * ctx,bool deterministic)340 base::Status PerfettoSqlEngine::RegisterStaticFunction(
341 const char* name,
342 int argc,
343 typename Function::Context* ctx,
344 bool deterministic) {
345 // Metric proto builder functions can be reregistered: don't double count when
346 // this happens.
347 if (!engine_->GetFunctionContext(name, argc)) {
348 static_function_count_++;
349 }
350 return engine_->RegisterFunction(
351 name, argc, perfetto_sql_internal::WrapSqlFunction<Function>, ctx,
352 nullptr, deterministic);
353 }
354
355 template <typename Function>
RegisterSqliteAggregateFunction(typename Function::UserDataContext * ctx,bool deterministic)356 base::Status PerfettoSqlEngine::RegisterSqliteAggregateFunction(
357 typename Function::UserDataContext* ctx,
358 bool deterministic) {
359 static_aggregate_function_count_++;
360 return engine_->RegisterAggregateFunction(
361 Function::kName, Function::kArgCount, Function::Step, Function::Final,
362 ctx, nullptr, deterministic);
363 }
364
365 template <typename Function>
RegisterSqliteWindowFunction(const char * name,int argc,typename Function::Context * ctx,bool deterministic)366 base::Status PerfettoSqlEngine::RegisterSqliteWindowFunction(
367 const char* name,
368 int argc,
369 typename Function::Context* ctx,
370 bool deterministic) {
371 static_window_function_count_++;
372 return engine_->RegisterWindowFunction(
373 name, argc, Function::Step, Function::Inverse, Function::Value,
374 Function::Final, ctx, nullptr, deterministic);
375 }
376
377 template <typename Function>
RegisterStaticFunction(const char * name,int argc,std::unique_ptr<typename Function::Context> ctx,bool deterministic)378 base::Status PerfettoSqlEngine::RegisterStaticFunction(
379 const char* name,
380 int argc,
381 std::unique_ptr<typename Function::Context> ctx,
382 bool deterministic) {
383 // Metric proto builder functions can be reregistered: don't double count when
384 // this happens.
385 if (!engine_->GetFunctionContext(name, argc)) {
386 static_function_count_++;
387 }
388 return RegisterFunctionWithSqlite<Function>(name, argc, std::move(ctx),
389 deterministic);
390 }
391
392 template <typename Function>
RegisterFunctionWithSqlite(const char * name,int argc,std::unique_ptr<typename Function::Context> ctx,bool deterministic)393 base::Status PerfettoSqlEngine::RegisterFunctionWithSqlite(
394 const char* name,
395 int argc,
396 std::unique_ptr<typename Function::Context> ctx,
397 bool deterministic) {
398 auto ctx_destructor = [](void* ptr) {
399 delete static_cast<typename Function::Context*>(ptr);
400 };
401 return engine_->RegisterFunction(
402 name, argc, perfetto_sql_internal::WrapSqlFunction<Function>,
403 ctx.release(), ctx_destructor, deterministic);
404 }
405
406 } // namespace perfetto::trace_processor
407
408 #endif // SRC_TRACE_PROCESSOR_PERFETTO_SQL_ENGINE_PERFETTO_SQL_ENGINE_H_
409