1 /*
2 * Copyright (C) 2023 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/trace_processor/perfetto_sql/tokenizer/sqlite_tokenizer.h"
18
19 #include <cstdint>
20 #include <string>
21 #include <string_view>
22 #include <utility>
23
24 #include "perfetto/base/logging.h"
25 #include "src/trace_processor/sqlite/sql_source.h"
26
27 namespace perfetto::trace_processor {
28 extern "C" {
29 int sqlite3GetToken(const unsigned char* z, int* tokenType);
30 int sqliteTokenizeInternalAnalyzeWindowKeyword(const unsigned char* z);
31 int sqliteTokenizeInternalAnalyzeOverKeyword(const unsigned char* z,
32 int lastToken);
33 int sqliteTokenizeInternalAnalyzeFilterKeyword(const unsigned char* z,
34 int lastToken);
35 }
36
SqliteTokenizer(SqlSource sql)37 SqliteTokenizer::SqliteTokenizer(SqlSource sql) : source_(std::move(sql)) {}
38
Next()39 SqliteTokenizer::Token SqliteTokenizer::Next() {
40 Token token;
41 const char* start = source_.sql().data() + offset_;
42 int n = sqlite3GetToken(reinterpret_cast<const unsigned char*>(start),
43 &token.token_type);
44 if (token.token_type == TK_WINDOW) {
45 token.token_type = sqliteTokenizeInternalAnalyzeWindowKeyword(
46 reinterpret_cast<const unsigned char*>(start + n));
47 } else if (token.token_type == TK_OVER) {
48 token.token_type = sqliteTokenizeInternalAnalyzeOverKeyword(
49 reinterpret_cast<const unsigned char*>(start + n),
50 last_non_space_token_);
51 } else if (token.token_type == TK_FILTER) {
52 token.token_type = sqliteTokenizeInternalAnalyzeFilterKeyword(
53 reinterpret_cast<const unsigned char*>(start + n),
54 last_non_space_token_);
55 }
56 offset_ += static_cast<uint32_t>(n);
57 token.str = std::string_view(start, static_cast<uint32_t>(n));
58 if (token.token_type != TK_SPACE) {
59 last_non_space_token_ = token.token_type;
60 }
61 return token;
62 }
63
NextNonWhitespace()64 SqliteTokenizer::Token SqliteTokenizer::NextNonWhitespace() {
65 Token t;
66 for (t = Next(); t.token_type == TK_SPACE; t = Next()) {
67 }
68 return t;
69 }
70
NextTerminal()71 SqliteTokenizer::Token SqliteTokenizer::NextTerminal() {
72 Token tok = Next();
73 while (!tok.IsTerminal()) {
74 tok = Next();
75 }
76 return tok;
77 }
78
Substr(const Token & start,const Token & end,EndToken end_token) const79 SqlSource SqliteTokenizer::Substr(const Token& start,
80 const Token& end,
81 EndToken end_token) const {
82 auto offset = static_cast<uint32_t>(start.str.data() - source_.sql().c_str());
83 const char* e =
84 end.str.data() +
85 (end_token == SqliteTokenizer::EndToken::kInclusive ? end.str.size() : 0);
86 auto len = static_cast<uint32_t>(e - start.str.data());
87 return source_.Substr(offset, len);
88 }
89
SubstrToken(const Token & token) const90 SqlSource SqliteTokenizer::SubstrToken(const Token& token) const {
91 auto offset = static_cast<uint32_t>(token.str.data() - source_.sql().c_str());
92 auto len = static_cast<uint32_t>(token.str.size());
93 return source_.Substr(offset, len);
94 }
95
AsTraceback(const Token & token) const96 std::string SqliteTokenizer::AsTraceback(const Token& token) const {
97 PERFETTO_CHECK(source_.sql().c_str() <= token.str.data());
98 PERFETTO_CHECK(token.str.data() <=
99 source_.sql().c_str() + source_.sql().size());
100 auto offset = static_cast<uint32_t>(token.str.data() - source_.sql().c_str());
101 return source_.AsTraceback(offset);
102 }
103
Rewrite(SqlSource::Rewriter & rewriter,const Token & start,const Token & end,SqlSource rewrite,EndToken end_token) const104 void SqliteTokenizer::Rewrite(SqlSource::Rewriter& rewriter,
105 const Token& start,
106 const Token& end,
107 SqlSource rewrite,
108 EndToken end_token) const {
109 auto s_off = static_cast<uint32_t>(start.str.data() - source_.sql().c_str());
110 auto e_off = static_cast<uint32_t>(end.str.data() - source_.sql().c_str());
111 uint32_t e_diff = end_token == EndToken::kInclusive
112 ? static_cast<uint32_t>(end.str.size())
113 : 0;
114 rewriter.Rewrite(s_off, e_off + e_diff, std::move(rewrite));
115 }
116
RewriteToken(SqlSource::Rewriter & rewriter,const Token & token,SqlSource rewrite) const117 void SqliteTokenizer::RewriteToken(SqlSource::Rewriter& rewriter,
118 const Token& token,
119 SqlSource rewrite) const {
120 auto s_off = static_cast<uint32_t>(token.str.data() - source_.sql().c_str());
121 auto e_off = static_cast<uint32_t>(token.str.data() + token.str.size() -
122 source_.sql().c_str());
123 rewriter.Rewrite(s_off, e_off, std::move(rewrite));
124 }
125
126 } // namespace perfetto::trace_processor
127