1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/perfetto_sql/engine/perfetto_sql_preprocessor.h"
18
19 #include <cstdint>
20 #include <optional>
21 #include <string>
22 #include <unordered_map>
23 #include <unordered_set>
24 #include <utility>
25
26 #include "perfetto/base/logging.h"
27 #include "perfetto/base/status.h"
28 #include "perfetto/ext/base/flat_hash_map.h"
29 #include "perfetto/ext/base/status_or.h"
30 #include "perfetto/ext/base/string_utils.h"
31 #include "src/trace_processor/sqlite/sql_source.h"
32 #include "src/trace_processor/sqlite/sqlite_tokenizer.h"
33 #include "src/trace_processor/util/status_macros.h"
34
35 namespace perfetto::trace_processor {
36 namespace {
37
ErrorAtToken(const SqliteTokenizer & tokenizer,const SqliteTokenizer::Token & token,const char * error)38 base::Status ErrorAtToken(const SqliteTokenizer& tokenizer,
39 const SqliteTokenizer::Token& token,
40 const char* error) {
41 std::string traceback = tokenizer.AsTraceback(token);
42 return base::ErrStatus("%s%s", traceback.c_str(), error);
43 }
44
45 struct InvocationArg {
46 std::optional<SqlSource> arg;
47 bool has_more;
48 };
49
ParseMacroInvocationArg(SqliteTokenizer & tokenizer,SqliteTokenizer::Token & tok,bool has_prev_args)50 base::StatusOr<InvocationArg> ParseMacroInvocationArg(
51 SqliteTokenizer& tokenizer,
52 SqliteTokenizer::Token& tok,
53 bool has_prev_args) {
54 uint32_t nested_parens = 0;
55 bool seen_token_in_arg = false;
56 auto start = tokenizer.NextNonWhitespace();
57 for (tok = start;; tok = tokenizer.NextNonWhitespace()) {
58 if (tok.IsTerminal()) {
59 if (tok.token_type == SqliteTokenType::TK_SEMI) {
60 // TODO(b/290185551): add a link to macro documentation.
61 return ErrorAtToken(tokenizer, tok,
62 "Semi-colon is not allowed in macro invocation");
63 }
64 // TODO(b/290185551): add a link to macro documentation.
65 return ErrorAtToken(tokenizer, tok, "Macro invocation not complete");
66 }
67
68 bool is_arg_terminator = tok.token_type == SqliteTokenType::TK_RP ||
69 tok.token_type == SqliteTokenType::TK_COMMA;
70 if (nested_parens == 0 && is_arg_terminator) {
71 bool token_required =
72 has_prev_args || tok.token_type != SqliteTokenType::TK_RP;
73 if (!seen_token_in_arg && token_required) {
74 // TODO(b/290185551): add a link to macro documentation.
75 return ErrorAtToken(tokenizer, tok, "Macro arg is empty");
76 }
77 return InvocationArg{
78 seen_token_in_arg ? std::make_optional(tokenizer.Substr(start, tok))
79 : std::optional<SqlSource>(std::nullopt),
80 tok.token_type == SqliteTokenType::TK_COMMA,
81 };
82 }
83 seen_token_in_arg = true;
84
85 if (tok.token_type == SqliteTokenType::TK_LP) {
86 nested_parens++;
87 continue;
88 }
89 if (tok.token_type == SqliteTokenType::TK_RP) {
90 nested_parens--;
91 continue;
92 }
93 }
94 }
95
96 } // namespace
97
PerfettoSqlPreprocessor(SqlSource source,const base::FlatHashMap<std::string,Macro> & macros)98 PerfettoSqlPreprocessor::PerfettoSqlPreprocessor(
99 SqlSource source,
100 const base::FlatHashMap<std::string, Macro>& macros)
101 : global_tokenizer_(std::move(source)), macros_(¯os) {}
102
NextStatement()103 bool PerfettoSqlPreprocessor::NextStatement() {
104 PERFETTO_CHECK(status_.ok());
105
106 // Skip through any number of semi-colons (representing empty statements).
107 SqliteTokenizer::Token tok = global_tokenizer_.NextNonWhitespace();
108 while (tok.token_type == SqliteTokenType::TK_SEMI) {
109 tok = global_tokenizer_.NextNonWhitespace();
110 }
111
112 // If we still see a terminal token at this point, we must have hit EOF.
113 if (tok.IsTerminal()) {
114 PERFETTO_DCHECK(tok.token_type != SqliteTokenType::TK_SEMI);
115 return false;
116 }
117
118 SqlSource stmt =
119 global_tokenizer_.Substr(tok, global_tokenizer_.NextTerminal());
120 auto stmt_or = RewriteInternal(stmt, {});
121 if (stmt_or.ok()) {
122 statement_ = std::move(*stmt_or);
123 return true;
124 }
125 status_ = stmt_or.status();
126 return false;
127 }
128
RewriteInternal(const SqlSource & source,const std::unordered_map<std::string,SqlSource> & arg_bindings)129 base::StatusOr<SqlSource> PerfettoSqlPreprocessor::RewriteInternal(
130 const SqlSource& source,
131 const std::unordered_map<std::string, SqlSource>& arg_bindings) {
132 SqlSource::Rewriter rewriter(source);
133 SqliteTokenizer tokenizer(source);
134 for (SqliteTokenizer::Token tok = tokenizer.NextNonWhitespace(), prev;;
135 prev = tok, tok = tokenizer.NextNonWhitespace()) {
136 if (tok.IsTerminal()) {
137 break;
138 }
139 if (tok.token_type == SqliteTokenType::TK_VARIABLE &&
140 !seen_macros_.empty()) {
141 PERFETTO_CHECK(tok.str.size() >= 2);
142 if (tok.str[0] != '$') {
143 return ErrorAtToken(tokenizer, tok, "Variables must start with $");
144 }
145 auto binding_it = arg_bindings.find(std::string(tok.str.substr(1)));
146 if (binding_it == arg_bindings.end()) {
147 return ErrorAtToken(tokenizer, tok, "Variable not found");
148 }
149 tokenizer.RewriteToken(rewriter, tok, binding_it->second);
150 continue;
151 }
152 if (tok.token_type != SqliteTokenType::TK_ILLEGAL || tok.str != "!") {
153 continue;
154 }
155
156 base::StatusOr<MacroInvocation> invocation_or =
157 ParseMacroInvocation(tokenizer, tok, prev, arg_bindings);
158 RETURN_IF_ERROR(invocation_or.status());
159
160 seen_macros_.emplace(invocation_or->macro->name);
161 auto source_or =
162 RewriteInternal(invocation_or->macro->sql, invocation_or->arg_bindings);
163 RETURN_IF_ERROR(source_or.status());
164 seen_macros_.erase(invocation_or->macro->name);
165
166 tokenizer.Rewrite(rewriter, prev, tok, std::move(*source_or),
167 SqliteTokenizer::EndToken::kInclusive);
168 }
169 return std::move(rewriter).Build();
170 }
171
172 base::StatusOr<PerfettoSqlPreprocessor::MacroInvocation>
ParseMacroInvocation(SqliteTokenizer & tokenizer,SqliteTokenizer::Token & tok,const SqliteTokenizer::Token & name_token,const std::unordered_map<std::string,SqlSource> & arg_bindings)173 PerfettoSqlPreprocessor::ParseMacroInvocation(
174 SqliteTokenizer& tokenizer,
175 SqliteTokenizer::Token& tok,
176 const SqliteTokenizer::Token& name_token,
177 const std::unordered_map<std::string, SqlSource>& arg_bindings) {
178 if (name_token.token_type == SqliteTokenType::TK_VARIABLE) {
179 // TODO(b/290185551): add a link to macro documentation.
180 return ErrorAtToken(tokenizer, name_token,
181 "Macro name cannot contain a variable");
182 }
183 if (name_token.token_type != SqliteTokenType::TK_ID) {
184 // TODO(b/290185551): add a link to macro documentation.
185 return ErrorAtToken(tokenizer, name_token, "Macro invocation is invalid");
186 }
187
188 // Get the opening left parenthesis.
189 tok = tokenizer.NextNonWhitespace();
190 if (tok.token_type != SqliteTokenType::TK_LP) {
191 // TODO(b/290185551): add a link to macro documentation.
192 return ErrorAtToken(tokenizer, tok, "( expected to open macro invocation");
193 }
194
195 std::string macro_name(name_token.str);
196 Macro* macro = macros_->Find(macro_name);
197 if (!macro) {
198 // TODO(b/290185551): add a link to macro documentation.
199 base::StackString<1024> err("Macro %s does not exist", macro_name.c_str());
200 return ErrorAtToken(tokenizer, name_token, err.c_str());
201 }
202
203 if (seen_macros_.count(macro_name)) {
204 // TODO(b/290185551): add a link to macro documentation.
205 return ErrorAtToken(tokenizer, name_token,
206 "Macros cannot be recursive or mutually recursive");
207 }
208
209 std::unordered_map<std::string, SqlSource> inner_bindings;
210 for (bool has_more = true; has_more;) {
211 base::StatusOr<InvocationArg> source_or =
212 ParseMacroInvocationArg(tokenizer, tok, !inner_bindings.empty());
213 RETURN_IF_ERROR(source_or.status());
214 if (source_or->arg) {
215 base::StatusOr<SqlSource> res =
216 RewriteInternal(source_or->arg.value(), arg_bindings);
217 RETURN_IF_ERROR(res.status());
218 if (macro->args.size() <= inner_bindings.size()) {
219 // TODO(lalitm): add a link to macro documentation.
220 return ErrorAtToken(tokenizer, name_token,
221 "Macro invoked with too many args");
222 }
223 inner_bindings.emplace(macro->args[inner_bindings.size()], *res);
224 }
225 has_more = source_or->has_more;
226 }
227
228 if (inner_bindings.size() < macro->args.size()) {
229 // TODO(lalitm): add a link to macro documentation.
230 return ErrorAtToken(tokenizer, name_token,
231 "Macro invoked with too few args");
232 }
233 PERFETTO_CHECK(inner_bindings.size() == macro->args.size());
234 return MacroInvocation{macro, std::move(inner_bindings)};
235 }
236
237 } // namespace perfetto::trace_processor
238