1 /**
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "lexer.h"
17
18 #include "es2panda.h"
19 #include "generated/keywords.h"
20 #include "lexer/token/letters.h"
21 #include "lexer/token/tokenType.h"
22 #include "parser/context/parserContext.h"
23 #include "parser/program/program.h"
24
25 #include <array>
26
27 namespace ark::es2panda::lexer {
LexerPosition(const util::StringView & source)28 LexerPosition::LexerPosition(const util::StringView &source) : iterator_(source) {}
29
Lexer(const parser::ParserContext * parserContext,bool startLexer)30 Lexer::Lexer(const parser::ParserContext *parserContext, bool startLexer)
31 : allocator_(parserContext->GetProgram()->Allocator()),
32 parserContext_(parserContext),
33 source_(parserContext->GetProgram()->SourceCode()),
34 pos_(source_)
35 {
36 if (startLexer) {
37 SkipWhiteSpaces();
38 }
39 }
40
ScanUnicodeEscapeSequence()41 char32_t Lexer::ScanUnicodeEscapeSequence()
42 {
43 ASSERT(Iterator().Peek() == LEX_CHAR_LOWERCASE_U);
44 auto constexpr UNICODE_ESCAPE_SEQUENCE_LENGTH = 4;
45
46 Iterator().Forward(1);
47
48 if (Iterator().Peek() == LEX_CHAR_LEFT_BRACE) {
49 Iterator().Forward(1);
50 return ScanUnicodeCodePointEscape();
51 }
52
53 if (parserContext_->GetProgram()->Extension() == ScriptExtension::AS) {
54 return ScanHexEscape<UNICODE_ESCAPE_SEQUENCE_LENGTH, true>();
55 }
56
57 return ScanHexEscape<UNICODE_ESCAPE_SEQUENCE_LENGTH>();
58 }
59
60 // '/u{...}' escape sequence should have at least one hex digit inside brackets!
ScanUnicodeCodePointEscape()61 char32_t Lexer::ScanUnicodeCodePointEscape()
62 {
63 double code = 0;
64 char32_t cp = Iterator().Peek();
65 if (!IsHexDigit(cp)) {
66 ThrowError("Invalid unicode escape sequence");
67 }
68
69 while (true) {
70 Iterator().Forward(1);
71
72 constexpr auto MULTIPLIER = 16;
73 code = code * MULTIPLIER + HexValue(cp);
74 if (code > UNICODE_CODE_POINT_MAX) {
75 ThrowError("Invalid unicode escape sequence");
76 }
77
78 cp = Iterator().Peek();
79 if (!IsHexDigit(cp)) {
80 break;
81 }
82 }
83
84 if (cp != LEX_CHAR_RIGHT_BRACE) {
85 ThrowError("Invalid unicode escape sequence");
86 }
87
88 Iterator().Forward(1);
89 return static_cast<char32_t>(code);
90 }
91
Allocator()92 ArenaAllocator *Lexer::Allocator()
93 {
94 return allocator_;
95 }
96
GetToken()97 Token &Lexer::GetToken()
98 {
99 return pos_.token_;
100 }
101
GetToken() const102 const Token &Lexer::GetToken() const
103 {
104 return pos_.token_;
105 }
106
Line() const107 size_t Lexer::Line() const
108 {
109 return pos_.line_;
110 }
111
Save() const112 LexerPosition Lexer::Save() const
113 {
114 return pos_;
115 }
116
BackwardToken(TokenType type,size_t offset)117 void Lexer::BackwardToken(TokenType type, size_t offset)
118 {
119 pos_.token_.type_ = type;
120 pos_.iterator_.Reset(GetToken().End().index - offset);
121 pos_.nextTokenLine_ = 0;
122 }
123
ForwardToken(TokenType type,size_t offset)124 void Lexer::ForwardToken(TokenType type, size_t offset)
125 {
126 pos_.token_.type_ = type;
127 pos_.iterator_.Forward(offset);
128 SkipWhiteSpaces();
129 }
130
Rewind(const LexerPosition & pos)131 void Lexer::Rewind(const LexerPosition &pos)
132 {
133 pos_ = pos;
134 }
135
Lookahead()136 char32_t Lexer::Lookahead()
137 {
138 return Iterator().Peek();
139 }
140
SourceView(const util::StringView::Iterator & begin,const util::StringView::Iterator & end) const141 util::StringView Lexer::SourceView(const util::StringView::Iterator &begin, const util::StringView::Iterator &end) const
142 {
143 return SourceView(begin.Index(), end.Index());
144 }
145
SourceView(size_t begin,size_t end) const146 util::StringView Lexer::SourceView(size_t begin, size_t end) const
147 {
148 return source_.Substr(begin, end);
149 }
150
SkipMultiLineComment()151 void Lexer::SkipMultiLineComment()
152 {
153 while (true) {
154 switch (Iterator().Next()) {
155 case util::StringView::Iterator::INVALID_CP: {
156 ThrowError("Unterminated multi-line comment");
157 }
158 case LEX_CHAR_LF:
159 case LEX_CHAR_CR:
160 case LEX_CHAR_LS:
161 case LEX_CHAR_PS: {
162 pos_.nextTokenLine_++;
163 continue;
164 }
165 case LEX_CHAR_ASTERISK: {
166 if (Iterator().Peek() == LEX_CHAR_SLASH) {
167 Iterator().Forward(1);
168 return;
169 }
170
171 break;
172 }
173 default: {
174 break;
175 }
176 }
177 }
178 }
179
180 /* New line character is not processed */
SkipSingleLineComment()181 void Lexer::SkipSingleLineComment()
182 {
183 while (true) {
184 switch (Iterator().Next()) {
185 case util::StringView::Iterator::INVALID_CP:
186 case LEX_CHAR_CR: {
187 if (Iterator().Peek() == LEX_CHAR_LF) {
188 Iterator().Forward(1);
189 }
190
191 [[fallthrough]];
192 }
193 case LEX_CHAR_LF:
194 case LEX_CHAR_LS:
195 case LEX_CHAR_PS: {
196 pos_.nextTokenLine_++;
197 return;
198 }
199 default: {
200 break;
201 }
202 }
203 }
204 }
205
ThrowUnexpectedToken(lexer::TokenType tokenType) const206 [[noreturn]] void Lexer::ThrowUnexpectedToken(lexer::TokenType tokenType) const
207 {
208 std::stringstream ss;
209 ss << "Unexpected token: '" << TokenToString(tokenType) << "'.";
210 ThrowError(ss.str());
211 }
212
ThrowError(std::string_view message) const213 [[noreturn]] void Lexer::ThrowError(std::string_view message) const
214 {
215 lexer::LineIndex index(source_);
216 lexer::SourceLocation loc = index.GetLocation(SourcePosition(Iterator().Index(), pos_.line_));
217
218 throw Error(es2panda::ErrorType::SYNTAX, parserContext_->GetProgram()->SourceFilePath().Utf8(), message, loc.line,
219 loc.col);
220 }
221
CheckNumberLiteralEnd()222 void Lexer::CheckNumberLiteralEnd()
223 {
224 if (Iterator().Peek() == LEX_CHAR_LOWERCASE_N) {
225 GetToken().flags_ |= TokenFlags::NUMBER_BIGINT;
226 GetToken().src_ = SourceView(GetToken().Start().index, Iterator().Index());
227 Iterator().Forward(1);
228 } else {
229 GetToken().src_ = SourceView(GetToken().Start().index, Iterator().Index());
230 }
231
232 const auto nextCp = Iterator().PeekCp();
233 if (KeywordsUtil::IsIdentifierStart(nextCp) || IsDecimalDigit(nextCp)) {
234 ThrowError("Invalid numeric literal");
235 }
236 }
237
ScanDecimalNumbers()238 void Lexer::ScanDecimalNumbers()
239 {
240 bool allowNumericOnNext = true;
241
242 while (true) {
243 switch (Iterator().Peek()) {
244 case LEX_CHAR_0:
245 case LEX_CHAR_1:
246 case LEX_CHAR_2:
247 case LEX_CHAR_3:
248 case LEX_CHAR_4:
249 case LEX_CHAR_5:
250 case LEX_CHAR_6:
251 case LEX_CHAR_7:
252 case LEX_CHAR_8:
253 case LEX_CHAR_9: {
254 Iterator().Forward(1);
255 allowNumericOnNext = true;
256 break;
257 }
258 case LEX_CHAR_UNDERSCORE: {
259 Iterator().Backward(1);
260
261 if (Iterator().Peek() == LEX_CHAR_DOT || !allowNumericOnNext) {
262 Iterator().Forward(1);
263 ThrowError("Invalid numeric separator");
264 }
265
266 GetToken().flags_ |= TokenFlags::NUMBER_HAS_UNDERSCORE;
267 Iterator().Forward(2U);
268 allowNumericOnNext = false;
269 break;
270 }
271 default: {
272 if (!allowNumericOnNext) {
273 ThrowError("Numeric separators are not allowed at the end of numeric literals");
274 }
275 return;
276 }
277 }
278 }
279 }
280
ConvertNumber(const std::string & utf8,NumberFlags flags)281 void Lexer::ConvertNumber(const std::string &utf8, [[maybe_unused]] NumberFlags flags)
282 {
283 ConversionResult res;
284 const long double temp = StrToNumeric(&std::strtold, utf8.c_str(), res);
285 if (res == ConversionResult::SUCCESS) {
286 GetToken().number_ = Number(GetToken().src_, static_cast<double>(temp));
287 } else if (res == ConversionResult::INVALID_ARGUMENT) {
288 ThrowError("Invalid number");
289 } else if (res == ConversionResult::OUT_OF_RANGE) {
290 GetToken().number_ = Number(GetToken().src_, std::numeric_limits<double>::infinity());
291 }
292 }
293
ScanNumber(bool allowBigInt)294 void Lexer::ScanNumber(bool allowBigInt)
295 {
296 const bool isPeriod = GetToken().type_ == TokenType::PUNCTUATOR_PERIOD;
297 GetToken().type_ = TokenType::LITERAL_NUMBER;
298 GetToken().keywordType_ = TokenType::LITERAL_NUMBER;
299
300 if (!isPeriod) {
301 ScanDecimalNumbers();
302 }
303
304 size_t exponentSignPos = std::numeric_limits<size_t>::max();
305 bool parseExponent = true;
306 auto flags = NumberFlags::NONE;
307
308 if (Iterator().Peek() == LEX_CHAR_DOT || isPeriod) {
309 flags |= NumberFlags::DECIMAL_POINT;
310 allowBigInt = false;
311 if (!isPeriod) {
312 Iterator().Forward(1);
313 }
314
315 auto cp = Iterator().Peek();
316 if (IsDecimalDigit(cp) || cp == LEX_CHAR_LOWERCASE_E || cp == LEX_CHAR_UPPERCASE_E) {
317 ScanDecimalNumbers();
318 } else {
319 parseExponent = false;
320 }
321 }
322
323 std::tie(exponentSignPos, allowBigInt, flags) = ScanCharLex(allowBigInt, parseExponent, flags);
324
325 CheckNumberLiteralEnd();
326
327 if ((GetToken().flags_ & TokenFlags::NUMBER_BIGINT) != 0) {
328 if (!allowBigInt) {
329 ThrowError("Invalid BigInt number");
330 }
331
332 return;
333 }
334
335 util::StringView sv = SourceView(GetToken().Start().index, Iterator().Index());
336 std::string utf8 = std::string {sv.Utf8()};
337 bool needConversion = false;
338
339 if (exponentSignPos != std::numeric_limits<size_t>::max()) {
340 utf8.insert(exponentSignPos, 1, '+');
341 needConversion = true;
342 }
343
344 if ((GetToken().flags_ & TokenFlags::NUMBER_HAS_UNDERSCORE) != 0U) {
345 utf8.erase(std::remove(utf8.begin(), utf8.end(), LEX_CHAR_UNDERSCORE), utf8.end());
346 needConversion = true;
347 }
348
349 if (needConversion) {
350 util::UString converted(utf8, Allocator());
351 GetToken().src_ = converted.View();
352 } else {
353 GetToken().src_ = sv;
354 }
355
356 ConvertNumber(utf8, flags);
357 }
358
ScanCharLex(bool allowBigInt,bool parseExponent,NumberFlags flags)359 std::tuple<size_t, bool, NumberFlags> Lexer::ScanCharLex(bool allowBigInt, bool parseExponent, NumberFlags flags)
360 {
361 size_t exponentSignPos = std::numeric_limits<size_t>::max();
362 switch (Iterator().Peek()) {
363 case LEX_CHAR_LOWERCASE_E:
364 case LEX_CHAR_UPPERCASE_E: {
365 allowBigInt = false;
366
367 if (!parseExponent) {
368 break;
369 }
370
371 flags |= NumberFlags::EXPONENT;
372
373 Iterator().Forward(1);
374
375 exponentSignPos = ScanSignOfNumber();
376
377 if (!IsDecimalDigit(Iterator().Peek())) {
378 ThrowError("Invalid numeric literal");
379 }
380 ScanDecimalNumbers();
381 break;
382 }
383 default: {
384 break;
385 }
386 }
387 return {exponentSignPos, allowBigInt, flags};
388 }
389
ScanSignOfNumber()390 size_t Lexer::ScanSignOfNumber()
391 {
392 size_t exponentSignPos = std::numeric_limits<size_t>::max();
393 switch (Iterator().Peek()) {
394 case LEX_CHAR_UNDERSCORE: {
395 break;
396 }
397 case LEX_CHAR_PLUS:
398 case LEX_CHAR_MINUS: {
399 Iterator().Forward(1);
400 break;
401 }
402 default: {
403 exponentSignPos = Iterator().Index() - GetToken().Start().index;
404 break;
405 }
406 }
407 return exponentSignPos;
408 }
409
PushTemplateContext(TemplateLiteralParserContext * ctx)410 void Lexer::PushTemplateContext(TemplateLiteralParserContext *ctx)
411 {
412 tlCtx_ = ctx;
413 }
414
ScanTemplateStringEnd()415 void Lexer::ScanTemplateStringEnd()
416 {
417 ASSERT(Iterator().Peek() == LEX_CHAR_BACK_TICK);
418 Iterator().Forward(1);
419 SetTokenEnd();
420 SkipWhiteSpaces();
421 }
422
ScanTemplateString()423 LexerTemplateString Lexer::ScanTemplateString()
424 {
425 LexerTemplateString templateStr(Allocator());
426 size_t cpSize = 0U;
427
428 auto const checkOctalDigit = [this](char32_t const nextCp) -> void {
429 if (IsOctalDigit(nextCp)) {
430 Iterator().Forward(1);
431
432 if (Iterator().Peek() != LEX_CHAR_BACK_TICK) {
433 ThrowError("Octal escape sequences are not allowed in template strings");
434 }
435
436 Iterator().Backward(1);
437 }
438 };
439
440 while (true) {
441 char32_t cp = Iterator().PeekCp(&cpSize);
442
443 switch (cp) {
444 case util::StringView::Iterator::INVALID_CP: {
445 ThrowError("Unexpected token, expected '${' or '`'");
446 break;
447 }
448 case LEX_CHAR_BACK_TICK: {
449 templateStr.end = Iterator().Index();
450 return templateStr;
451 }
452 case LEX_CHAR_CR: {
453 Iterator().Forward(1);
454
455 if (Iterator().Peek() != LEX_CHAR_LF) {
456 Iterator().Backward(1);
457 }
458
459 [[fallthrough]];
460 }
461 case LEX_CHAR_LF: {
462 pos_.line_++;
463 templateStr.str.Append(LEX_CHAR_LF);
464 Iterator().Forward(1);
465 continue;
466 }
467 case LEX_CHAR_BACKSLASH: {
468 Iterator().Forward(1);
469
470 char32_t nextCp = Iterator().Peek();
471 checkOctalDigit(nextCp);
472
473 if (nextCp == LEX_CHAR_BACK_TICK || nextCp == LEX_CHAR_BACKSLASH || nextCp == LEX_CHAR_DOLLAR_SIGN) {
474 templateStr.str.Append(cp);
475 templateStr.str.Append(nextCp);
476 Iterator().Forward(1);
477 continue;
478 }
479
480 Iterator().Backward(1);
481 break;
482 }
483 case LEX_CHAR_DOLLAR_SIGN: {
484 templateStr.end = Iterator().Index();
485 Iterator().Forward(1);
486
487 if (Iterator().Peek() == LEX_CHAR_LEFT_BRACE) {
488 Iterator().Forward(1);
489 templateStr.scanExpression = true;
490 SkipWhiteSpaces();
491 return templateStr;
492 }
493
494 templateStr.str.Append(cp);
495 continue;
496 }
497 default: {
498 break;
499 }
500 }
501
502 templateStr.str.Append(cp);
503 Iterator().Forward(cpSize);
504 }
505
506 UNREACHABLE();
507 return templateStr;
508 }
509
ResetTokenEnd()510 void Lexer::ResetTokenEnd()
511 {
512 SetTokenStart();
513 pos_.iterator_.Reset(GetToken().End().index);
514 pos_.line_ = GetToken().End().line;
515 pos_.nextTokenLine_ = 0;
516 }
517
ScanStringUnicodePart(util::UString * str)518 void Lexer::ScanStringUnicodePart(util::UString *str)
519 {
520 if (auto cp = ScanUnicodeCharacter(); cp != util::StringView::Iterator::INVALID_CP) {
521 str->Append(cp);
522 }
523 }
524
ScanUnicodeCharacter()525 char32_t Lexer::ScanUnicodeCharacter()
526 {
527 size_t cpSize {};
528 char32_t cp = Iterator().PeekCp(&cpSize);
529
530 switch (cp) {
531 case util::StringView::Iterator::INVALID_CP: {
532 ThrowError("Unterminated string");
533 break;
534 }
535 case LEX_CHAR_CR: {
536 Iterator().Forward(1);
537 if (Iterator().Peek() != LEX_CHAR_LF) {
538 Iterator().Backward(1);
539 }
540
541 [[fallthrough]];
542 }
543 case LEX_CHAR_LS:
544 case LEX_CHAR_PS:
545 case LEX_CHAR_LF: {
546 pos_.line_++;
547 Iterator().Forward(cpSize);
548 return util::StringView::Iterator::INVALID_CP;
549 }
550 case LEX_CHAR_LOWERCASE_B: {
551 cp = LEX_CHAR_BS;
552 break;
553 }
554 case LEX_CHAR_LOWERCASE_T: {
555 cp = LEX_CHAR_TAB;
556 break;
557 }
558 case LEX_CHAR_LOWERCASE_N: {
559 cp = LEX_CHAR_LF;
560 break;
561 }
562 case LEX_CHAR_LOWERCASE_V: {
563 cp = LEX_CHAR_VT;
564 break;
565 }
566 case LEX_CHAR_LOWERCASE_F: {
567 cp = LEX_CHAR_FF;
568 break;
569 }
570 case LEX_CHAR_LOWERCASE_R: {
571 cp = LEX_CHAR_CR;
572 break;
573 }
574 case LEX_CHAR_LOWERCASE_X: {
575 Iterator().Forward(1);
576 cp = ScanHexEscape<2U>();
577 return cp;
578 }
579 case LEX_CHAR_LOWERCASE_U: {
580 cp = ScanUnicodeEscapeSequence();
581 return cp;
582 }
583 case LEX_CHAR_0: {
584 Iterator().Forward(1);
585 bool isDecimal = IsDecimalDigit(Iterator().Peek());
586 Iterator().Backward(1);
587
588 if (!isDecimal) {
589 cp = LEX_CHAR_NULL;
590 break;
591 }
592
593 [[fallthrough]];
594 }
595 default: {
596 if (IsDecimalDigit(Iterator().Peek())) {
597 ThrowError("Invalid character escape sequence in strict mode");
598 }
599
600 break;
601 }
602 }
603
604 Iterator().Forward(cpSize);
605 return cp;
606 }
607
ScanQuestionPunctuator()608 void Lexer::ScanQuestionPunctuator()
609 {
610 GetToken().type_ = TokenType::PUNCTUATOR_QUESTION_MARK;
611
612 switch (Iterator().Peek()) {
613 case LEX_CHAR_QUESTION: {
614 GetToken().type_ = TokenType::PUNCTUATOR_NULLISH_COALESCING;
615 Iterator().Forward(1);
616
617 switch (Iterator().Peek()) {
618 case LEX_CHAR_EQUALS: {
619 GetToken().type_ = TokenType::PUNCTUATOR_LOGICAL_NULLISH_EQUAL;
620 Iterator().Forward(1);
621 break;
622 }
623 default: {
624 break;
625 }
626 }
627
628 break;
629 }
630 case LEX_CHAR_DOT: {
631 Iterator().Forward(1);
632
633 if (!IsDecimalDigit(Iterator().Peek())) {
634 GetToken().type_ = TokenType::PUNCTUATOR_QUESTION_DOT;
635 return;
636 }
637
638 Iterator().Backward(1);
639 break;
640 }
641 default: {
642 break;
643 }
644 }
645 }
646
ScanLessThanPunctuator()647 void Lexer::ScanLessThanPunctuator()
648 {
649 GetToken().type_ = TokenType::PUNCTUATOR_LESS_THAN;
650
651 switch (Iterator().Peek()) {
652 case LEX_CHAR_LESS_THAN: {
653 GetToken().type_ = TokenType::PUNCTUATOR_LEFT_SHIFT;
654 Iterator().Forward(1);
655
656 if (Iterator().Peek() == LEX_CHAR_EQUALS) {
657 GetToken().type_ = TokenType::PUNCTUATOR_LEFT_SHIFT_EQUAL;
658 Iterator().Forward(1);
659 }
660 break;
661 }
662 case LEX_CHAR_EQUALS: {
663 GetToken().type_ = TokenType::PUNCTUATOR_LESS_THAN_EQUAL;
664 Iterator().Forward(1);
665 break;
666 }
667 default: {
668 break;
669 }
670 }
671 }
672
ScanGreaterThanPunctuator()673 void Lexer::ScanGreaterThanPunctuator()
674 {
675 GetToken().type_ = TokenType::PUNCTUATOR_GREATER_THAN;
676
677 switch (Iterator().Peek()) {
678 case LEX_CHAR_GREATER_THAN: {
679 GetToken().type_ = TokenType::PUNCTUATOR_RIGHT_SHIFT;
680 Iterator().Forward(1);
681
682 switch (Iterator().Peek()) {
683 case LEX_CHAR_GREATER_THAN: {
684 GetToken().type_ = TokenType::PUNCTUATOR_UNSIGNED_RIGHT_SHIFT;
685 Iterator().Forward(1);
686
687 if (Iterator().Peek() == LEX_CHAR_EQUALS) {
688 GetToken().type_ = TokenType::PUNCTUATOR_UNSIGNED_RIGHT_SHIFT_EQUAL;
689 Iterator().Forward(1);
690 }
691 break;
692 }
693 case LEX_CHAR_EQUALS: {
694 GetToken().type_ = TokenType::PUNCTUATOR_RIGHT_SHIFT_EQUAL;
695 Iterator().Forward(1);
696 break;
697 }
698 default: {
699 break;
700 }
701 }
702 break;
703 }
704 case LEX_CHAR_EQUALS: {
705 GetToken().type_ = TokenType::PUNCTUATOR_GREATER_THAN_EQUAL;
706 Iterator().Forward(1);
707 break;
708 }
709 default: {
710 break;
711 }
712 }
713 }
714
ScanEqualsPunctuator()715 void Lexer::ScanEqualsPunctuator()
716 {
717 GetToken().type_ = TokenType::PUNCTUATOR_SUBSTITUTION;
718
719 switch (Iterator().Peek()) {
720 case LEX_CHAR_EQUALS: {
721 GetToken().type_ = TokenType::PUNCTUATOR_EQUAL;
722 Iterator().Forward(1);
723
724 if (Iterator().Peek() == LEX_CHAR_EQUALS) {
725 GetToken().type_ = TokenType::PUNCTUATOR_STRICT_EQUAL;
726 Iterator().Forward(1);
727 }
728 break;
729 }
730 case LEX_CHAR_GREATER_THAN: {
731 GetToken().type_ = TokenType::PUNCTUATOR_ARROW;
732 Iterator().Forward(1);
733 break;
734 }
735 default: {
736 break;
737 }
738 }
739 }
740
ScanExclamationPunctuator()741 void Lexer::ScanExclamationPunctuator()
742 {
743 GetToken().type_ = TokenType::PUNCTUATOR_EXCLAMATION_MARK;
744
745 switch (Iterator().Peek()) {
746 case LEX_CHAR_EQUALS: {
747 GetToken().type_ = TokenType::PUNCTUATOR_NOT_EQUAL;
748 Iterator().Forward(1);
749
750 if (Iterator().Peek() == LEX_CHAR_EQUALS) {
751 GetToken().type_ = TokenType::PUNCTUATOR_NOT_STRICT_EQUAL;
752 Iterator().Forward(1);
753 }
754 break;
755 }
756 default: {
757 break;
758 }
759 }
760 }
761
ScanAmpersandPunctuator()762 void Lexer::ScanAmpersandPunctuator()
763 {
764 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_AND;
765
766 switch (Iterator().Peek()) {
767 case LEX_CHAR_AMPERSAND: {
768 GetToken().type_ = TokenType::PUNCTUATOR_LOGICAL_AND;
769 Iterator().Forward(1);
770
771 switch (Iterator().Peek()) {
772 case LEX_CHAR_EQUALS: {
773 GetToken().type_ = TokenType::PUNCTUATOR_LOGICAL_AND_EQUAL;
774 Iterator().Forward(1);
775 break;
776 }
777 default: {
778 break;
779 }
780 }
781
782 break;
783 }
784 case LEX_CHAR_EQUALS: {
785 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_AND_EQUAL;
786 Iterator().Forward(1);
787 break;
788 }
789 default: {
790 break;
791 }
792 }
793 }
794
ScanAtPunctuator()795 void Lexer::ScanAtPunctuator()
796 {
797 GetToken().type_ = TokenType::PUNCTUATOR_AT;
798
799 if (Iterator().Peek() == LEX_CHAR_AT) {
800 GetToken().type_ = TokenType::PUNCTUATOR_FORMAT;
801 Iterator().Forward(1U);
802 }
803 }
804
ScanVLinePunctuator()805 void Lexer::ScanVLinePunctuator()
806 {
807 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_OR;
808
809 switch (Iterator().Peek()) {
810 case LEX_CHAR_VLINE: {
811 GetToken().type_ = TokenType::PUNCTUATOR_LOGICAL_OR;
812 Iterator().Forward(1);
813
814 switch (Iterator().Peek()) {
815 case LEX_CHAR_EQUALS: {
816 GetToken().type_ = TokenType::PUNCTUATOR_LOGICAL_OR_EQUAL;
817 Iterator().Forward(1);
818 break;
819 }
820 default: {
821 break;
822 }
823 }
824
825 break;
826 }
827 case LEX_CHAR_EQUALS: {
828 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_OR_EQUAL;
829 Iterator().Forward(1);
830 break;
831 }
832 default: {
833 break;
834 }
835 }
836 }
837
ScanCircumflexPunctuator()838 void Lexer::ScanCircumflexPunctuator()
839 {
840 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_XOR;
841
842 switch (Iterator().Peek()) {
843 case LEX_CHAR_EQUALS: {
844 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_XOR_EQUAL;
845 Iterator().Forward(1);
846 break;
847 }
848 default: {
849 break;
850 }
851 }
852 }
853
ScanPlusPunctuator()854 void Lexer::ScanPlusPunctuator()
855 {
856 GetToken().type_ = TokenType::PUNCTUATOR_PLUS;
857
858 switch (Iterator().Peek()) {
859 case LEX_CHAR_PLUS: {
860 GetToken().type_ = TokenType::PUNCTUATOR_PLUS_PLUS;
861 Iterator().Forward(1);
862 break;
863 }
864 case LEX_CHAR_EQUALS: {
865 GetToken().type_ = TokenType::PUNCTUATOR_PLUS_EQUAL;
866 Iterator().Forward(1);
867 break;
868 }
869 default: {
870 break;
871 }
872 }
873 }
874
ScanMinusPunctuator()875 void Lexer::ScanMinusPunctuator()
876 {
877 GetToken().type_ = TokenType::PUNCTUATOR_MINUS;
878
879 switch (Iterator().Peek()) {
880 case LEX_CHAR_MINUS: {
881 GetToken().type_ = TokenType::PUNCTUATOR_MINUS_MINUS;
882 Iterator().Forward(1);
883 break;
884 }
885 case LEX_CHAR_EQUALS: {
886 GetToken().type_ = TokenType::PUNCTUATOR_MINUS_EQUAL;
887 Iterator().Forward(1);
888 break;
889 }
890 default: {
891 break;
892 }
893 }
894 }
895
ScanSlashPunctuator()896 void Lexer::ScanSlashPunctuator()
897 {
898 GetToken().type_ = TokenType::PUNCTUATOR_DIVIDE;
899
900 switch (Iterator().Peek()) {
901 case LEX_CHAR_EQUALS: {
902 GetToken().type_ = TokenType::PUNCTUATOR_DIVIDE_EQUAL;
903 Iterator().Forward(1);
904 break;
905 }
906 default: {
907 break;
908 }
909 }
910 }
911
ScanDotPunctuator()912 void Lexer::ScanDotPunctuator()
913 {
914 GetToken().type_ = TokenType::PUNCTUATOR_PERIOD;
915
916 switch (Iterator().Peek()) {
917 case LEX_CHAR_0:
918 case LEX_CHAR_1:
919 case LEX_CHAR_2:
920 case LEX_CHAR_3:
921 case LEX_CHAR_4:
922 case LEX_CHAR_5:
923 case LEX_CHAR_6:
924 case LEX_CHAR_7:
925 case LEX_CHAR_8:
926 case LEX_CHAR_9: {
927 ScanNumber();
928 break;
929 }
930 case LEX_CHAR_QUESTION: {
931 GetToken().type_ = TokenType::PUNCTUATOR_PERIOD_QUESTION;
932 Iterator().Forward(1);
933 break;
934 }
935 case LEX_CHAR_DOT: {
936 Iterator().Forward(1);
937
938 if (Iterator().Peek() == LEX_CHAR_DOT) {
939 GetToken().type_ = TokenType::PUNCTUATOR_PERIOD_PERIOD_PERIOD;
940 Iterator().Forward(1);
941 break;
942 }
943
944 Iterator().Backward(1);
945 break;
946 }
947 default: {
948 break;
949 }
950 }
951 }
952
ScanAsteriskPunctuator()953 void Lexer::ScanAsteriskPunctuator()
954 {
955 GetToken().type_ = TokenType::PUNCTUATOR_MULTIPLY;
956
957 switch (Iterator().Peek()) {
958 case LEX_CHAR_ASTERISK: {
959 GetToken().type_ = TokenType::PUNCTUATOR_EXPONENTIATION;
960 Iterator().Forward(1);
961
962 if (Iterator().Peek() == LEX_CHAR_EQUALS) {
963 GetToken().type_ = TokenType::PUNCTUATOR_EXPONENTIATION_EQUAL;
964 Iterator().Forward(1);
965 }
966 break;
967 }
968 case LEX_CHAR_EQUALS: {
969 GetToken().type_ = TokenType::PUNCTUATOR_MULTIPLY_EQUAL;
970 Iterator().Forward(1);
971 break;
972 }
973 default: {
974 break;
975 }
976 }
977 }
978
ScanPercentPunctuator()979 void Lexer::ScanPercentPunctuator()
980 {
981 GetToken().type_ = TokenType::PUNCTUATOR_MOD;
982
983 switch (Iterator().Peek()) {
984 case LEX_CHAR_EQUALS: {
985 GetToken().type_ = TokenType::PUNCTUATOR_MOD_EQUAL;
986 Iterator().Forward(1);
987 break;
988 }
989 default: {
990 break;
991 }
992 }
993 }
994
IsLineTerminatorOrEos() const995 bool Lexer::IsLineTerminatorOrEos() const
996 {
997 switch (Iterator().PeekCp()) {
998 case util::StringView::Iterator::INVALID_CP:
999 case LEX_CHAR_LF:
1000 case LEX_CHAR_CR:
1001 case LEX_CHAR_LS:
1002 case LEX_CHAR_PS: {
1003 return true;
1004 }
1005 default: {
1006 break;
1007 }
1008 }
1009
1010 return false;
1011 }
1012
ScanRegExpPattern()1013 void Lexer::ScanRegExpPattern()
1014 {
1015 bool isCharClass = false;
1016 size_t cpSize {};
1017
1018 while (true) {
1019 switch (Iterator().PeekCp(&cpSize)) {
1020 case util::StringView::Iterator::INVALID_CP:
1021 case LEX_CHAR_LF:
1022 case LEX_CHAR_CR:
1023 case LEX_CHAR_LS:
1024 case LEX_CHAR_PS: {
1025 ThrowError("Unterminated RegExp");
1026 }
1027 case LEX_CHAR_SLASH: {
1028 if (!isCharClass) {
1029 return;
1030 }
1031
1032 break;
1033 }
1034 case LEX_CHAR_LEFT_SQUARE: {
1035 isCharClass = true;
1036 break;
1037 }
1038 case LEX_CHAR_RIGHT_SQUARE: {
1039 isCharClass = false;
1040 break;
1041 }
1042 case LEX_CHAR_BACKSLASH: {
1043 Iterator().Forward(1);
1044
1045 if (IsLineTerminatorOrEos()) {
1046 continue;
1047 }
1048
1049 break;
1050 }
1051 default: {
1052 break;
1053 }
1054 }
1055
1056 Iterator().Forward(cpSize);
1057 }
1058 }
1059
ScanRegExpFlags()1060 RegExpFlags Lexer::ScanRegExpFlags()
1061 {
1062 RegExpFlags resultFlags = RegExpFlags::EMPTY;
1063
1064 while (true) {
1065 size_t cpSize {};
1066 auto cp = Iterator().PeekCp(&cpSize);
1067 if (!KeywordsUtil::IsIdentifierPart(cp)) {
1068 break;
1069 }
1070
1071 Iterator().Forward(cpSize);
1072
1073 RegExpFlags flag = RegExpFlags::EMPTY;
1074
1075 switch (cp) {
1076 case LEX_CHAR_LOWERCASE_G: {
1077 flag = RegExpFlags::GLOBAL;
1078 break;
1079 }
1080 case LEX_CHAR_LOWERCASE_I: {
1081 flag = RegExpFlags::IGNORE_CASE;
1082 break;
1083 }
1084 case LEX_CHAR_LOWERCASE_M: {
1085 flag = RegExpFlags::MULTILINE;
1086 break;
1087 }
1088 case LEX_CHAR_LOWERCASE_S: {
1089 flag = RegExpFlags::DOTALL;
1090 break;
1091 }
1092 case LEX_CHAR_LOWERCASE_U: {
1093 flag = RegExpFlags::UNICODE;
1094 break;
1095 }
1096 case LEX_CHAR_LOWERCASE_Y: {
1097 flag = RegExpFlags::STICKY;
1098 break;
1099 }
1100 case LEX_CHAR_SP: {
1101 return resultFlags;
1102 }
1103 default: {
1104 ThrowError("Invalid RegExp flag");
1105 }
1106 }
1107
1108 if (flag == RegExpFlags::EMPTY || (resultFlags & flag) != 0) {
1109 ThrowError("Invalid RegExp flag");
1110 }
1111
1112 resultFlags = resultFlags | flag;
1113 }
1114
1115 return resultFlags;
1116 }
1117
ScanRegExp()1118 RegExp Lexer::ScanRegExp()
1119 {
1120 // for proper handling such regexps as /=/
1121 if (GetToken().Type() == lexer::TokenType::PUNCTUATOR_DIVIDE_EQUAL) {
1122 Iterator().Backward(1);
1123 }
1124
1125 GetToken().type_ = TokenType::LITERAL_REGEXP;
1126 GetToken().keywordType_ = TokenType::LITERAL_REGEXP;
1127
1128 const auto patternStart = Iterator().Index();
1129 ScanRegExpPattern();
1130 const auto pattern = SourceView(patternStart, Iterator().Index());
1131
1132 ASSERT(Iterator().Peek() == LEX_CHAR_SLASH);
1133 Iterator().Forward(1);
1134
1135 const auto flagsStart = Iterator().Index();
1136 RegExpFlags resultFlags = ScanRegExpFlags();
1137 const auto flags = SourceView(flagsStart, Iterator().Index());
1138
1139 SkipWhiteSpaces();
1140 SetTokenEnd();
1141
1142 return {pattern, flags, resultFlags};
1143 }
1144
CheckArrow()1145 bool Lexer::CheckArrow()
1146 {
1147 if (Iterator().Peek() != LEX_CHAR_EQUALS) {
1148 return false;
1149 }
1150 Iterator().Forward(1);
1151
1152 bool res = Iterator().Peek() == LEX_CHAR_GREATER_THAN;
1153 Iterator().Backward(1);
1154
1155 return res;
1156 }
1157
SetTokenStart()1158 void Lexer::SetTokenStart()
1159 {
1160 if (pos_.nextTokenLine_ != 0) {
1161 pos_.line_ += pos_.nextTokenLine_;
1162 pos_.nextTokenLine_ = 0;
1163 GetToken().flags_ = TokenFlags::NEW_LINE;
1164 } else {
1165 GetToken().flags_ = TokenFlags::NONE;
1166 }
1167
1168 pos_.token_.loc_.start = SourcePosition {Iterator().Index(), pos_.line_};
1169 GetToken().keywordType_ = TokenType::EOS;
1170 }
1171
SetTokenEnd()1172 void Lexer::SetTokenEnd()
1173 {
1174 pos_.token_.loc_.end = SourcePosition {Iterator().Index(), pos_.line_};
1175 }
1176
SkipWhiteSpaces()1177 void Lexer::SkipWhiteSpaces()
1178 {
1179 while (true) {
1180 auto cp = Iterator().Peek();
1181
1182 switch (cp) {
1183 case LEX_CHAR_CR: {
1184 Iterator().Forward(1);
1185
1186 if (Iterator().Peek() != LEX_CHAR_LF) {
1187 Iterator().Backward(1);
1188 }
1189
1190 [[fallthrough]];
1191 }
1192 case LEX_CHAR_LF: {
1193 Iterator().Forward(1);
1194 pos_.nextTokenLine_++;
1195 continue;
1196 }
1197 case LEX_CHAR_VT:
1198 case LEX_CHAR_FF:
1199 case LEX_CHAR_SP:
1200 case LEX_CHAR_TAB: {
1201 Iterator().Forward(1);
1202 continue;
1203 }
1204 case LEX_CHAR_SLASH: {
1205 Iterator().Forward(1);
1206 cp = Iterator().Peek();
1207 if (cp == LEX_CHAR_SLASH) {
1208 Iterator().Forward(1);
1209 SkipSingleLineComment();
1210 continue;
1211 }
1212 if (cp == LEX_CHAR_ASTERISK) {
1213 Iterator().Forward(1);
1214 SkipMultiLineComment();
1215 continue;
1216 }
1217
1218 Iterator().Backward(1);
1219 return;
1220 }
1221 default: {
1222 if (cp < LEX_ASCII_MAX_BITS) {
1223 return;
1224 }
1225
1226 size_t cpSize {};
1227 cp = Iterator().PeekCp(&cpSize);
1228
1229 switch (cp) {
1230 case LEX_CHAR_LS:
1231 case LEX_CHAR_PS: {
1232 pos_.nextTokenLine_++;
1233 [[fallthrough]];
1234 }
1235 case LEX_CHAR_NBSP:
1236 case LEX_CHAR_ZWNBSP: {
1237 Iterator().Forward(cpSize);
1238 continue;
1239 }
1240 default: {
1241 return;
1242 }
1243 }
1244 }
1245 }
1246 }
1247 }
1248
ScanHashMark()1249 void Lexer::ScanHashMark()
1250 {
1251 GetToken().type_ = TokenType::PUNCTUATOR_HASH_MARK;
1252 }
1253
ScanBackTick()1254 void Lexer::ScanBackTick()
1255 {
1256 GetToken().type_ = TokenType::PUNCTUATOR_BACK_TICK;
1257 SetTokenEnd();
1258 }
1259
1260 // NOLINTNEXTLINE(google-default-arguments)
NextToken(NextTokenFlags flags)1261 void Lexer::NextToken(NextTokenFlags flags)
1262 {
1263 JSKeywords kws(this, flags);
1264 NextToken(&kws);
1265 }
1266
ScanColonPunctuator()1267 void Lexer::ScanColonPunctuator()
1268 {
1269 GetToken().type_ = TokenType::PUNCTUATOR_COLON;
1270 }
1271
ScanDollarPunctuator()1272 bool Lexer::ScanDollarPunctuator()
1273 {
1274 return false;
1275 }
1276
1277 // NOLINTNEXTLINE(readability-function-size)
NextToken(Keywords * kws)1278 void Lexer::NextToken(Keywords *kws)
1279 {
1280 KeywordsUtil &kwu = kws->Util();
1281
1282 SetTokenStart();
1283
1284 auto cp = Iterator().Peek();
1285 Iterator().Forward(1);
1286
1287 switch (cp) {
1288 case LEX_CHAR_EXCLAMATION: {
1289 ScanExclamationPunctuator();
1290 break;
1291 }
1292 case LEX_CHAR_SINGLE_QUOTE: {
1293 ScanString<LEX_CHAR_SINGLE_QUOTE>();
1294 break;
1295 }
1296 case LEX_CHAR_DOUBLE_QUOTE: {
1297 ScanString<LEX_CHAR_DOUBLE_QUOTE>();
1298 break;
1299 }
1300 case LEX_CHAR_HASH_MARK: {
1301 ScanHashMark();
1302 break;
1303 }
1304 case LEX_CHAR_PERCENT: {
1305 ScanPercentPunctuator();
1306 break;
1307 }
1308 case LEX_CHAR_AMPERSAND: {
1309 ScanAmpersandPunctuator();
1310 break;
1311 }
1312 case LEX_CHAR_LEFT_PAREN: {
1313 GetToken().type_ = TokenType::PUNCTUATOR_LEFT_PARENTHESIS;
1314 break;
1315 }
1316 case LEX_CHAR_RIGHT_PAREN: {
1317 GetToken().type_ = TokenType::PUNCTUATOR_RIGHT_PARENTHESIS;
1318 break;
1319 }
1320 case LEX_CHAR_ASTERISK: {
1321 ScanAsteriskPunctuator();
1322 break;
1323 }
1324 case LEX_CHAR_PLUS: {
1325 ScanPlusPunctuator();
1326 break;
1327 }
1328 case LEX_CHAR_COMMA: {
1329 GetToken().type_ = TokenType::PUNCTUATOR_COMMA;
1330 break;
1331 }
1332 case LEX_CHAR_MINUS: {
1333 ScanMinusPunctuator();
1334 break;
1335 }
1336 case LEX_CHAR_DOT: {
1337 ScanDotPunctuator();
1338 break;
1339 }
1340 case LEX_CHAR_SLASH: {
1341 ScanSlashPunctuator();
1342 break;
1343 }
1344 case LEX_CHAR_0: {
1345 ScanNumberLeadingZero();
1346 break;
1347 }
1348 case LEX_CHAR_1:
1349 case LEX_CHAR_2:
1350 case LEX_CHAR_3:
1351 case LEX_CHAR_4:
1352 case LEX_CHAR_5:
1353 case LEX_CHAR_6:
1354 case LEX_CHAR_7:
1355 case LEX_CHAR_8:
1356 case LEX_CHAR_9: {
1357 ScanNumber();
1358 break;
1359 }
1360 case LEX_CHAR_COLON: {
1361 ScanColonPunctuator();
1362 break;
1363 }
1364 case LEX_CHAR_SEMICOLON: {
1365 GetToken().type_ = TokenType::PUNCTUATOR_SEMI_COLON;
1366 break;
1367 }
1368 case LEX_CHAR_LESS_THAN: {
1369 ScanLessThanPunctuator();
1370 break;
1371 }
1372 case LEX_CHAR_EQUALS: {
1373 ScanEqualsPunctuator();
1374 break;
1375 }
1376 case LEX_CHAR_GREATER_THAN: {
1377 ScanGreaterThanPunctuator();
1378 break;
1379 }
1380 case LEX_CHAR_QUESTION: {
1381 ScanQuestionPunctuator();
1382 break;
1383 }
1384 case LEX_CHAR_AT: {
1385 ScanAtPunctuator();
1386 break;
1387 }
1388 case LEX_CHAR_DOLLAR_SIGN: {
1389 if (ScanDollarPunctuator()) {
1390 break;
1391 }
1392
1393 [[fallthrough]];
1394 }
1395 case LEX_CHAR_UPPERCASE_A:
1396 case LEX_CHAR_UPPERCASE_B:
1397 case LEX_CHAR_UPPERCASE_C:
1398 case LEX_CHAR_UPPERCASE_D:
1399 case LEX_CHAR_UPPERCASE_E:
1400 case LEX_CHAR_UPPERCASE_F:
1401 case LEX_CHAR_UPPERCASE_G:
1402 case LEX_CHAR_UPPERCASE_H:
1403 case LEX_CHAR_UPPERCASE_I:
1404 case LEX_CHAR_UPPERCASE_J:
1405 case LEX_CHAR_UPPERCASE_K:
1406 case LEX_CHAR_UPPERCASE_L:
1407 case LEX_CHAR_UPPERCASE_M:
1408 case LEX_CHAR_UPPERCASE_N:
1409 case LEX_CHAR_UPPERCASE_O:
1410 case LEX_CHAR_UPPERCASE_P:
1411 case LEX_CHAR_UPPERCASE_Q:
1412 case LEX_CHAR_UPPERCASE_R:
1413 case LEX_CHAR_UPPERCASE_S:
1414 case LEX_CHAR_UPPERCASE_T:
1415 case LEX_CHAR_UPPERCASE_U:
1416 case LEX_CHAR_UPPERCASE_V:
1417 case LEX_CHAR_UPPERCASE_W:
1418 case LEX_CHAR_UPPERCASE_X:
1419 case LEX_CHAR_UPPERCASE_Y:
1420 case LEX_CHAR_UPPERCASE_Z:
1421 case LEX_CHAR_UNDERSCORE: {
1422 kwu.ScanIdContinue();
1423 break;
1424 }
1425 case LEX_CHAR_LEFT_SQUARE: {
1426 GetToken().type_ = TokenType::PUNCTUATOR_LEFT_SQUARE_BRACKET;
1427 break;
1428 }
1429 case LEX_CHAR_BACKSLASH: {
1430 GetToken().flags_ |= TokenFlags::HAS_ESCAPE;
1431
1432 if (Iterator().Peek() != LEX_CHAR_LOWERCASE_U) {
1433 ThrowError("Invalid character");
1434 }
1435
1436 cp = ScanUnicodeEscapeSequence();
1437
1438 kwu.ScanIdentifierStart(kws, cp);
1439 break;
1440 }
1441 case LEX_CHAR_RIGHT_SQUARE: {
1442 GetToken().type_ = TokenType::PUNCTUATOR_RIGHT_SQUARE_BRACKET;
1443 break;
1444 }
1445 case LEX_CHAR_CIRCUMFLEX: {
1446 ScanCircumflexPunctuator();
1447 break;
1448 }
1449 case LEX_CHAR_BACK_TICK: {
1450 ScanBackTick();
1451 return;
1452 }
1453 case LEX_CHAR_LOWERCASE_C: {
1454 if (ScanCharLiteral()) {
1455 break;
1456 }
1457 }
1458 [[fallthrough]];
1459 case LEX_CHAR_LOWERCASE_A:
1460 case LEX_CHAR_LOWERCASE_B:
1461 case LEX_CHAR_LOWERCASE_D:
1462 case LEX_CHAR_LOWERCASE_E:
1463 case LEX_CHAR_LOWERCASE_F:
1464 case LEX_CHAR_LOWERCASE_G:
1465 case LEX_CHAR_LOWERCASE_H:
1466 case LEX_CHAR_LOWERCASE_I:
1467 case LEX_CHAR_LOWERCASE_J:
1468 case LEX_CHAR_LOWERCASE_K:
1469 case LEX_CHAR_LOWERCASE_L:
1470 case LEX_CHAR_LOWERCASE_M:
1471 case LEX_CHAR_LOWERCASE_N:
1472 case LEX_CHAR_LOWERCASE_O:
1473 case LEX_CHAR_LOWERCASE_P:
1474 case LEX_CHAR_LOWERCASE_Q:
1475 case LEX_CHAR_LOWERCASE_R:
1476 case LEX_CHAR_LOWERCASE_S:
1477 case LEX_CHAR_LOWERCASE_T:
1478 case LEX_CHAR_LOWERCASE_U:
1479 case LEX_CHAR_LOWERCASE_V:
1480 case LEX_CHAR_LOWERCASE_W:
1481 case LEX_CHAR_LOWERCASE_X:
1482 case LEX_CHAR_LOWERCASE_Y:
1483 case LEX_CHAR_LOWERCASE_Z: {
1484 kws->ScanKeyword(cp);
1485 break;
1486 }
1487 case LEX_CHAR_LEFT_BRACE: {
1488 GetToken().type_ = TokenType::PUNCTUATOR_LEFT_BRACE;
1489
1490 if (tlCtx_ != nullptr) {
1491 tlCtx_->ConsumeLeftBrace();
1492 }
1493
1494 break;
1495 }
1496 case LEX_CHAR_VLINE: {
1497 ScanVLinePunctuator();
1498 break;
1499 }
1500 case LEX_CHAR_RIGHT_BRACE: {
1501 GetToken().type_ = TokenType::PUNCTUATOR_RIGHT_BRACE;
1502
1503 if (tlCtx_ != nullptr && tlCtx_->ConsumeRightBrace()) {
1504 SetTokenEnd();
1505 return;
1506 }
1507
1508 break;
1509 }
1510 case LEX_CHAR_TILDE: {
1511 GetToken().type_ = TokenType::PUNCTUATOR_TILDE;
1512 break;
1513 }
1514 default: {
1515 Iterator().Backward(1);
1516
1517 if (cp == util::StringView::Iterator::INVALID_CP) {
1518 GetToken().type_ = TokenType::EOS;
1519 break;
1520 }
1521
1522 cp = Iterator().Next();
1523 kwu.ScanIdentifierStart(kws, cp);
1524 break;
1525 }
1526 }
1527
1528 SetTokenEnd();
1529 SkipWhiteSpaces();
1530 }
1531
ScanNumberLeadingZeroImplNonAllowedCases()1532 void Lexer::ScanNumberLeadingZeroImplNonAllowedCases()
1533 {
1534 switch (Iterator().Peek()) {
1535 case LEX_CHAR_0:
1536 case LEX_CHAR_1:
1537 case LEX_CHAR_2:
1538 case LEX_CHAR_3:
1539 case LEX_CHAR_4:
1540 case LEX_CHAR_5:
1541 case LEX_CHAR_6:
1542 case LEX_CHAR_7: {
1543 ThrowError("Implicit octal literal not allowed");
1544 }
1545 case LEX_CHAR_8:
1546 case LEX_CHAR_9: {
1547 ThrowError("NonOctalDecimalIntegerLiteral is not enabled in strict mode code");
1548 }
1549 case LEX_CHAR_UNDERSCORE: {
1550 ThrowError("Numeric separator '_' is not allowed in numbers that start with '0'.");
1551 }
1552 default: {
1553 break;
1554 }
1555 }
1556 }
1557
Pos()1558 LexerPosition &Lexer::Pos()
1559 {
1560 return pos_;
1561 }
1562
Pos() const1563 const LexerPosition &Lexer::Pos() const
1564 {
1565 return pos_;
1566 }
1567 } // namespace ark::es2panda::lexer
1568