1 /**
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "keywordsUtil.h"
17
18 #include <gen/keywords.h>
19 #include <unicode/uchar.h>
20
21 namespace panda::es2panda::lexer {
22
23 enum class AsciiFlags : uint8_t {
24 NONE = 0,
25 ID_START = 1 << 0,
26 ID_CONTINUE = 1 << 1,
27 };
28
operator |(AsciiFlags a,AsciiFlags b)29 constexpr AsciiFlags operator|(AsciiFlags a, AsciiFlags b)
30 {
31 using utype = std::underlying_type_t<AsciiFlags>;
32 return static_cast<AsciiFlags>(static_cast<utype>(a) | static_cast<utype>(b));
33 }
34
operator &(AsciiFlags a,AsciiFlags b)35 inline std::underlying_type_t<AsciiFlags> operator&(AsciiFlags a, AsciiFlags b)
36 {
37 using utype = std::underlying_type_t<AsciiFlags>;
38 /* NOLINTNEXTLINE(hicpp-signed-bitwise) */
39 return static_cast<utype>(static_cast<utype>(a) & static_cast<utype>(b));
40 }
41
42 constexpr std::array<AsciiFlags, 128> ASCII_FLAGS = {{
43 AsciiFlags::NONE, /* NUL */
44 AsciiFlags::NONE, /* SOH */
45 AsciiFlags::NONE, /* STX */
46 AsciiFlags::NONE, /* ETX */
47 AsciiFlags::NONE, /* EOT */
48 AsciiFlags::NONE, /* ENQ */
49 AsciiFlags::NONE, /* ACK */
50 AsciiFlags::NONE, /* BEL */
51 AsciiFlags::NONE, /* BS */
52 AsciiFlags::NONE, /* TAB */
53 AsciiFlags::NONE, /* LF */
54 AsciiFlags::NONE, /* VT */
55 AsciiFlags::NONE, /* FF */
56 AsciiFlags::NONE, /* CR */
57 AsciiFlags::NONE, /* SO */
58 AsciiFlags::NONE, /* SI */
59 AsciiFlags::NONE, /* DLE */
60 AsciiFlags::NONE, /* DC1 */
61 AsciiFlags::NONE, /* DC2 */
62 AsciiFlags::NONE, /* DC3 */
63 AsciiFlags::NONE, /* DC4 */
64 AsciiFlags::NONE, /* NAK */
65 AsciiFlags::NONE, /* SYN */
66 AsciiFlags::NONE, /* ETB */
67 AsciiFlags::NONE, /* CAN */
68 AsciiFlags::NONE, /* EM */
69 AsciiFlags::NONE, /* SUB */
70 AsciiFlags::NONE, /* ESC */
71 AsciiFlags::NONE, /* FS */
72 AsciiFlags::NONE, /* GS */
73 AsciiFlags::NONE, /* RS */
74 AsciiFlags::NONE, /* US */
75 AsciiFlags::NONE, /* Space */
76 AsciiFlags::NONE, /* ! */
77 AsciiFlags::NONE, /* " */
78 AsciiFlags::NONE, /* # */
79 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* $ */
80 AsciiFlags::NONE, /* % */
81 AsciiFlags::NONE, /* & */
82 AsciiFlags::NONE, /* ' */
83 AsciiFlags::NONE, /* ( */
84 AsciiFlags::NONE, /* ) */
85 AsciiFlags::NONE, /* * */
86 AsciiFlags::NONE, /* + */
87 AsciiFlags::NONE, /* , */
88 AsciiFlags::NONE, /* - */
89 AsciiFlags::NONE, /* . */
90 AsciiFlags::NONE, /* / */
91 AsciiFlags::ID_CONTINUE, /* 0 */
92 AsciiFlags::ID_CONTINUE, /* 1 */
93 AsciiFlags::ID_CONTINUE, /* 2 */
94 AsciiFlags::ID_CONTINUE, /* 3 */
95 AsciiFlags::ID_CONTINUE, /* 4 */
96 AsciiFlags::ID_CONTINUE, /* 5 */
97 AsciiFlags::ID_CONTINUE, /* 6 */
98 AsciiFlags::ID_CONTINUE, /* 7 */
99 AsciiFlags::ID_CONTINUE, /* 8 */
100 AsciiFlags::ID_CONTINUE, /* 9 */
101 AsciiFlags::NONE, /* : */
102 AsciiFlags::NONE, /* ; */
103 AsciiFlags::NONE, /* < */
104 AsciiFlags::NONE, /* = */
105 AsciiFlags::NONE, /* > */
106 AsciiFlags::NONE, /* ? */
107 AsciiFlags::NONE, /* @ */
108 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* A */
109 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* B */
110 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* C */
111 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* D */
112 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* E */
113 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* F */
114 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* G */
115 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* H */
116 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* I */
117 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* J */
118 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* K */
119 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* L */
120 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* M */
121 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* N */
122 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* O */
123 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* P */
124 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* Q */
125 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* R */
126 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* S */
127 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* T */
128 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* U */
129 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* V */
130 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* W */
131 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* X */
132 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* Y */
133 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* Z */
134 AsciiFlags::NONE, /* [ */
135 AsciiFlags::NONE, /* \ */
136 AsciiFlags::NONE, /* ] */
137 AsciiFlags::NONE, /* ^ */
138 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* _ */
139 AsciiFlags::NONE, /* ` */
140 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* a */
141 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* b */
142 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* c */
143 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* d */
144 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* e */
145 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* f */
146 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* g */
147 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* h */
148 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* i */
149 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* j */
150 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* k */
151 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* l */
152 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* m */
153 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* n */
154 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* o */
155 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* p */
156 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* q */
157 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* r */
158 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* s */
159 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* t */
160 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* u */
161 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* v */
162 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* w */
163 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* x */
164 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* y */
165 AsciiFlags::ID_START | AsciiFlags::ID_CONTINUE, /* z */
166 AsciiFlags::NONE, /* { */
167 AsciiFlags::NONE, /* | */
168 AsciiFlags::NONE, /* } */
169 AsciiFlags::NONE, /* ~ */
170 AsciiFlags::NONE /* DEL */
171 }};
172
IsIdentifierStart(char32_t cp)173 bool KeywordsUtil::IsIdentifierStart(char32_t cp)
174 {
175 if (cp < LEX_ASCII_MAX_BITS) {
176 return (ASCII_FLAGS[cp] & AsciiFlags::ID_START) != 0;
177 }
178 // Unicode {xxxxx} may consist of 4 bytes information and cannot be forcibly converted to 2 bytes
179 auto uchar = static_cast<UChar32>(cp);
180 return u_hasBinaryProperty(uchar, UCHAR_ID_START);
181 }
182
IsIdentifierPart(char32_t cp)183 bool KeywordsUtil::IsIdentifierPart(char32_t cp)
184 {
185 if (cp < LEX_ASCII_MAX_BITS) {
186 return (ASCII_FLAGS[cp] & AsciiFlags::ID_CONTINUE) != 0;
187 }
188
189 /**
190 * u_isIDPart or Other_ID_Continue characters or ZWJ/ZWNJ.
191 * Unicode {xxxxx} may consist of 4 bytes information and cannot be forcibly converted to 2 bytes
192 */
193 auto uchar = static_cast<UChar32>(cp);
194 return (u_hasBinaryProperty(uchar, UCHAR_ID_CONTINUE) || cp == LEX_CHAR_ZWNJ || cp == LEX_CHAR_ZWJ);
195 }
196
ScanIdentifierStart(char32_t cp)197 void KeywordsUtil::ScanIdentifierStart(char32_t cp)
198 {
199 if (!KeywordsUtil::IsIdentifierStart(cp)) {
200 lexer_->ThrowError("Expected an identifier");
201 }
202
203 cp_ = cp;
204 const auto map = KeywordsMap::Map(cp);
205 ScanIdContinueMaybeKeyword(map);
206 }
207
ScanIdContinue()208 void KeywordsUtil::ScanIdContinue()
209 {
210 util::UString ident(lexer_->Allocator());
211 size_t startPos = lexer_->GetToken().Start().index;
212
213 if (HasEscape()) {
214 ident.Append(cp_);
215 startPos = Iterator().Index();
216 }
217
218 auto escapeEnd = startPos;
219
220 do {
221 if (Iterator().Peek() == LEX_CHAR_BACKSLASH) {
222 ident.Append(lexer_->SourceView(escapeEnd, Iterator().Index()));
223
224 auto cp = ScanUnicodeEscapeSequence();
225 if (!IsIdentifierPart(cp)) {
226 lexer_->ThrowError("Invalid identifier part");
227 }
228
229 escapeEnd = Iterator().Index();
230 ident.Append(cp);
231 continue;
232 }
233
234 size_t cpSize {};
235 auto cp = Iterator().PeekCp(&cpSize);
236 if (!IsIdentifierPart(cp)) {
237 break;
238 }
239
240 Iterator().Forward(cpSize);
241 } while (true);
242
243 lexer_->GetToken().type_ = TokenType::LITERAL_IDENT;
244 lexer_->GetToken().keywordType_ = TokenType::EOS;
245
246 if (HasEscape()) {
247 ident.Append(lexer_->SourceView(escapeEnd, Iterator().Index()));
248 lexer_->GetToken().src_ = ident.View();
249 } else {
250 lexer_->GetToken().src_ = lexer_->SourceView(startPos, Iterator().Index());
251 }
252 }
253
ScanIdContinueMaybeKeyword(Span<const KeywordString> map)254 void KeywordsUtil::ScanIdContinueMaybeKeyword(Span<const KeywordString> map)
255 {
256 ScanIdContinue();
257
258 if (!HasEscape() || map.empty()) {
259 return;
260 }
261
262 const auto &str = lexer_->GetToken().Ident().Utf8();
263
264 int start = 0;
265 int end = static_cast<int>(map.size());
266 int middle = end / 2;
267
268 while (true) {
269 const auto &kws = map[middle];
270
271 int relation = str.compare(kws.str);
272 if (relation == 0) {
273 Keywords::SetKeyword(this, kws);
274 }
275
276 if (relation > 0) {
277 start = middle + 1;
278 } else {
279 end = middle;
280 }
281
282 middle = (start + end) / 2;
283
284 if (start >= end) {
285 return;
286 }
287 }
288 }
289
ScanUnicodeEscapeSequence()290 char32_t KeywordsUtil::ScanUnicodeEscapeSequence()
291 {
292 ASSERT(Iterator().Peek() == LEX_CHAR_BACKSLASH);
293
294 lexer_->GetToken().flags_ |= lexer::TokenFlags::HAS_ESCAPE;
295
296 Iterator().Forward(1);
297
298 if (Iterator().Peek() != LEX_CHAR_LOWERCASE_U) {
299 return util::StringView::Iterator::INVALID_CP;
300 }
301
302 return lexer_->ScanUnicodeEscapeSequence();
303 }
304
305 } // namespace panda::es2panda::lexer
306