1 /**
2 * Copyright (c) 2021 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "lexer.h"
17
18 #include "es2panda.h"
19 #include "generated/keywords.h"
20 #include "lexer/token/letters.h"
21 #include "lexer/token/tokenType.h"
22 #include "parser/context/parserContext.h"
23 #include "parser/program/program.h"
24
25 #include <array>
26
27 namespace panda::es2panda::lexer {
LexerPosition(const util::StringView & source)28 LexerPosition::LexerPosition(const util::StringView &source) : iterator_(source) {}
29
Lexer(const parser::ParserContext * parserContext,bool startLexer)30 Lexer::Lexer(const parser::ParserContext *parserContext, bool startLexer)
31 : allocator_(parserContext->GetProgram()->Allocator()),
32 parserContext_(parserContext),
33 source_(parserContext->GetProgram()->SourceCode()),
34 pos_(source_)
35 {
36 if (startLexer) {
37 SkipWhiteSpaces();
38 }
39 }
40
ScanUnicodeEscapeSequence()41 char32_t Lexer::ScanUnicodeEscapeSequence()
42 {
43 ASSERT(Iterator().Peek() == LEX_CHAR_LOWERCASE_U);
44 auto constexpr UNICODE_ESCAPE_SEQUENCE_LENGTH = 4;
45
46 Iterator().Forward(1);
47
48 if (Iterator().Peek() == LEX_CHAR_LEFT_BRACE) {
49 Iterator().Forward(1);
50 return ScanUnicodeCodePointEscape();
51 }
52
53 if (parserContext_->GetProgram()->Extension() == ScriptExtension::AS) {
54 return ScanHexEscape<UNICODE_ESCAPE_SEQUENCE_LENGTH, true>();
55 }
56
57 return ScanHexEscape<UNICODE_ESCAPE_SEQUENCE_LENGTH>();
58 }
59
60 // '/u{...}' escape sequence should have at least one hex digit inside brackets!
ScanUnicodeCodePointEscape()61 char32_t Lexer::ScanUnicodeCodePointEscape()
62 {
63 double code = 0;
64 char32_t cp = Iterator().Peek();
65 if (!IsHexDigit(cp)) {
66 ThrowError("Invalid unicode escape sequence");
67 }
68
69 while (true) {
70 Iterator().Forward(1);
71
72 constexpr auto MULTIPLIER = 16;
73 code = code * MULTIPLIER + HexValue(cp);
74 if (code > UNICODE_CODE_POINT_MAX) {
75 ThrowError("Invalid unicode escape sequence");
76 }
77
78 cp = Iterator().Peek();
79 if (!IsHexDigit(cp)) {
80 break;
81 }
82 }
83
84 if (cp != LEX_CHAR_RIGHT_BRACE) {
85 ThrowError("Invalid unicode escape sequence");
86 }
87
88 Iterator().Forward(1);
89 return static_cast<char32_t>(code);
90 }
91
Allocator()92 ArenaAllocator *Lexer::Allocator()
93 {
94 return allocator_;
95 }
96
GetToken()97 Token &Lexer::GetToken()
98 {
99 return pos_.token_;
100 }
101
GetToken() const102 const Token &Lexer::GetToken() const
103 {
104 return pos_.token_;
105 }
106
Line() const107 size_t Lexer::Line() const
108 {
109 return pos_.line_;
110 }
111
Save() const112 LexerPosition Lexer::Save() const
113 {
114 return pos_;
115 }
116
BackwardToken(TokenType type,size_t offset)117 void Lexer::BackwardToken(TokenType type, size_t offset)
118 {
119 pos_.token_.type_ = type;
120 pos_.iterator_.Reset(GetToken().End().index - offset);
121 pos_.nextTokenLine_ = 0;
122 }
123
ForwardToken(TokenType type,size_t offset)124 void Lexer::ForwardToken(TokenType type, size_t offset)
125 {
126 pos_.token_.type_ = type;
127 pos_.iterator_.Forward(offset);
128 SkipWhiteSpaces();
129 }
130
Rewind(const LexerPosition & pos)131 void Lexer::Rewind(const LexerPosition &pos)
132 {
133 pos_ = pos;
134 }
135
Lookahead()136 char32_t Lexer::Lookahead()
137 {
138 return Iterator().Peek();
139 }
140
SourceView(const util::StringView::Iterator & begin,const util::StringView::Iterator & end) const141 util::StringView Lexer::SourceView(const util::StringView::Iterator &begin, const util::StringView::Iterator &end) const
142 {
143 return SourceView(begin.Index(), end.Index());
144 }
145
SourceView(size_t begin,size_t end) const146 util::StringView Lexer::SourceView(size_t begin, size_t end) const
147 {
148 return source_.Substr(begin, end);
149 }
150
SkipMultiLineComment()151 void Lexer::SkipMultiLineComment()
152 {
153 while (true) {
154 switch (Iterator().Next()) {
155 case util::StringView::Iterator::INVALID_CP: {
156 ThrowError("Unterminated multi-line comment");
157 break;
158 }
159 case LEX_CHAR_LF:
160 case LEX_CHAR_CR:
161 case LEX_CHAR_LS:
162 case LEX_CHAR_PS: {
163 pos_.nextTokenLine_++;
164 continue;
165 }
166 case LEX_CHAR_ASTERISK: {
167 if (Iterator().Peek() == LEX_CHAR_SLASH) {
168 Iterator().Forward(1);
169 return;
170 }
171
172 break;
173 }
174 default: {
175 break;
176 }
177 }
178 }
179 }
180
181 /* New line character is not processed */
SkipSingleLineComment()182 void Lexer::SkipSingleLineComment()
183 {
184 while (true) {
185 switch (Iterator().Next()) {
186 case util::StringView::Iterator::INVALID_CP:
187 case LEX_CHAR_CR: {
188 if (Iterator().Peek() == LEX_CHAR_LF) {
189 Iterator().Forward(1);
190 }
191
192 [[fallthrough]];
193 }
194 case LEX_CHAR_LF:
195 case LEX_CHAR_LS:
196 case LEX_CHAR_PS: {
197 pos_.nextTokenLine_++;
198 return;
199 }
200 default: {
201 break;
202 }
203 }
204 }
205 }
206
ThrowUnexpectedToken(lexer::TokenType tokenType) const207 [[noreturn]] void Lexer::ThrowUnexpectedToken(lexer::TokenType tokenType) const
208 {
209 std::stringstream ss;
210 ss << "Unexpected token: '" << TokenToString(tokenType) << "'.";
211 ThrowError(ss.str());
212 }
213
ThrowError(std::string_view message) const214 [[noreturn]] void Lexer::ThrowError(std::string_view message) const
215 {
216 lexer::LineIndex index(source_);
217 lexer::SourceLocation loc = index.GetLocation(SourcePosition(Iterator().Index(), pos_.line_));
218
219 throw Error(es2panda::ErrorType::SYNTAX, parserContext_->GetProgram()->SourceFilePath().Utf8(), message, loc.line,
220 loc.col);
221 }
222
CheckNumberLiteralEnd()223 void Lexer::CheckNumberLiteralEnd()
224 {
225 if (Iterator().Peek() == LEX_CHAR_LOWERCASE_N) {
226 GetToken().flags_ |= TokenFlags::NUMBER_BIGINT;
227 GetToken().src_ = SourceView(GetToken().Start().index, Iterator().Index());
228 Iterator().Forward(1);
229 } else {
230 GetToken().src_ = SourceView(GetToken().Start().index, Iterator().Index());
231 }
232
233 const auto nextCp = Iterator().PeekCp();
234 if (KeywordsUtil::IsIdentifierStart(nextCp) || IsDecimalDigit(nextCp)) {
235 ThrowError("Invalid numeric literal");
236 }
237 }
238
ScanDecimalNumbers()239 void Lexer::ScanDecimalNumbers()
240 {
241 bool allowNumericOnNext = true;
242
243 while (true) {
244 switch (Iterator().Peek()) {
245 case LEX_CHAR_0:
246 case LEX_CHAR_1:
247 case LEX_CHAR_2:
248 case LEX_CHAR_3:
249 case LEX_CHAR_4:
250 case LEX_CHAR_5:
251 case LEX_CHAR_6:
252 case LEX_CHAR_7:
253 case LEX_CHAR_8:
254 case LEX_CHAR_9: {
255 Iterator().Forward(1);
256 allowNumericOnNext = true;
257 break;
258 }
259 case LEX_CHAR_UNDERSCORE: {
260 Iterator().Backward(1);
261
262 if (Iterator().Peek() == LEX_CHAR_DOT || !allowNumericOnNext) {
263 Iterator().Forward(1);
264 ThrowError("Invalid numeric separator");
265 }
266
267 GetToken().flags_ |= TokenFlags::NUMBER_HAS_UNDERSCORE;
268 Iterator().Forward(2U);
269 allowNumericOnNext = false;
270 break;
271 }
272 default: {
273 if (!allowNumericOnNext) {
274 ThrowError("Numeric separators are not allowed at the end of numeric literals");
275 }
276 return;
277 }
278 }
279 }
280 }
281
ConvertNumber(const std::string & utf8,NumberFlags flags)282 void Lexer::ConvertNumber(const std::string &utf8, [[maybe_unused]] NumberFlags flags)
283 {
284 ConversionResult res;
285 const long double temp = StrToNumeric(&std::strtold, utf8.c_str(), res);
286 if (res == ConversionResult::SUCCESS) {
287 GetToken().number_ = Number(GetToken().src_, static_cast<double>(temp));
288 } else if (res == ConversionResult::INVALID_ARGUMENT) {
289 ThrowError("Invalid number");
290 } else if (res == ConversionResult::OUT_OF_RANGE) {
291 GetToken().number_ = Number(GetToken().src_, std::numeric_limits<double>::infinity());
292 }
293 }
294
ScanNumber(bool allowBigInt)295 void Lexer::ScanNumber(bool allowBigInt)
296 {
297 const bool isPeriod = GetToken().type_ == TokenType::PUNCTUATOR_PERIOD;
298 GetToken().type_ = TokenType::LITERAL_NUMBER;
299 GetToken().keywordType_ = TokenType::LITERAL_NUMBER;
300
301 if (!isPeriod) {
302 ScanDecimalNumbers();
303 }
304
305 size_t exponentSignPos = std::numeric_limits<size_t>::max();
306 bool parseExponent = true;
307 auto flags = NumberFlags::NONE;
308
309 if (Iterator().Peek() == LEX_CHAR_DOT || isPeriod) {
310 flags |= NumberFlags::DECIMAL_POINT;
311 allowBigInt = false;
312 if (!isPeriod) {
313 Iterator().Forward(1);
314 }
315
316 auto cp = Iterator().Peek();
317 if (IsDecimalDigit(cp) || cp == LEX_CHAR_LOWERCASE_E || cp == LEX_CHAR_UPPERCASE_E) {
318 ScanDecimalNumbers();
319 } else {
320 parseExponent = false;
321 }
322 }
323
324 switch (Iterator().Peek()) {
325 case LEX_CHAR_LOWERCASE_E:
326 case LEX_CHAR_UPPERCASE_E: {
327 allowBigInt = false;
328
329 if (!parseExponent) {
330 break;
331 }
332
333 flags |= NumberFlags::EXPONENT;
334
335 Iterator().Forward(1);
336
337 switch (Iterator().Peek()) {
338 case LEX_CHAR_UNDERSCORE: {
339 break;
340 }
341 case LEX_CHAR_PLUS:
342 case LEX_CHAR_MINUS: {
343 Iterator().Forward(1);
344 break;
345 }
346 default: {
347 exponentSignPos = Iterator().Index() - GetToken().Start().index;
348 break;
349 }
350 }
351
352 if (!IsDecimalDigit(Iterator().Peek())) {
353 ThrowError("Invalid numeric literal");
354 }
355 ScanDecimalNumbers();
356 break;
357 }
358 default: {
359 break;
360 }
361 }
362
363 CheckNumberLiteralEnd();
364
365 if ((GetToken().flags_ & TokenFlags::NUMBER_BIGINT) != 0) {
366 if (!allowBigInt) {
367 ThrowError("Invalid BigInt number");
368 }
369
370 return;
371 }
372
373 util::StringView sv = SourceView(GetToken().Start().index, Iterator().Index());
374 std::string utf8 = std::string {sv.Utf8()};
375 bool needConversion = false;
376
377 if (exponentSignPos != std::numeric_limits<size_t>::max()) {
378 utf8.insert(exponentSignPos, 1, '+');
379 needConversion = true;
380 }
381
382 if ((GetToken().flags_ & TokenFlags::NUMBER_HAS_UNDERSCORE) != 0U) {
383 utf8.erase(std::remove(utf8.begin(), utf8.end(), LEX_CHAR_UNDERSCORE), utf8.end());
384 needConversion = true;
385 }
386
387 if (needConversion) {
388 util::UString converted(utf8, Allocator());
389 GetToken().src_ = converted.View();
390 } else {
391 GetToken().src_ = sv;
392 }
393
394 ConvertNumber(utf8, flags);
395 }
396
PushTemplateContext(TemplateLiteralParserContext * ctx)397 void Lexer::PushTemplateContext(TemplateLiteralParserContext *ctx)
398 {
399 tlCtx_ = ctx;
400 }
401
ScanTemplateStringEnd()402 void Lexer::ScanTemplateStringEnd()
403 {
404 ASSERT(Iterator().Peek() == LEX_CHAR_BACK_TICK);
405 Iterator().Forward(1);
406 SetTokenEnd();
407 SkipWhiteSpaces();
408 }
409
ScanTemplateString()410 LexerTemplateString Lexer::ScanTemplateString()
411 {
412 LexerTemplateString templateStr(Allocator());
413 size_t cpSize = 0U;
414
415 auto const checkOctalDigit = [this](char32_t const nextCp) -> void {
416 if (IsOctalDigit(nextCp)) {
417 Iterator().Forward(1);
418
419 if (Iterator().Peek() != LEX_CHAR_BACK_TICK) {
420 ThrowError("Octal escape sequences are not allowed in template strings");
421 }
422
423 Iterator().Backward(1);
424 }
425 };
426
427 while (true) {
428 char32_t cp = Iterator().PeekCp(&cpSize);
429
430 switch (cp) {
431 case util::StringView::Iterator::INVALID_CP: {
432 ThrowError("Unexpected token, expected '${' or '`'");
433 break;
434 }
435 case LEX_CHAR_BACK_TICK: {
436 templateStr.end = Iterator().Index();
437 return templateStr;
438 }
439 case LEX_CHAR_CR: {
440 Iterator().Forward(1);
441
442 if (Iterator().Peek() != LEX_CHAR_LF) {
443 Iterator().Backward(1);
444 }
445
446 [[fallthrough]];
447 }
448 case LEX_CHAR_LF: {
449 pos_.line_++;
450 templateStr.str.Append(LEX_CHAR_LF);
451 Iterator().Forward(1);
452 continue;
453 }
454 case LEX_CHAR_BACKSLASH: {
455 Iterator().Forward(1);
456
457 char32_t nextCp = Iterator().Peek();
458 checkOctalDigit(nextCp);
459
460 if (nextCp == LEX_CHAR_BACK_TICK || nextCp == LEX_CHAR_BACKSLASH) {
461 templateStr.str.Append(cp);
462 templateStr.str.Append(nextCp);
463 Iterator().Forward(1);
464 continue;
465 }
466
467 Iterator().Backward(1);
468 break;
469 }
470 case LEX_CHAR_DOLLAR_SIGN: {
471 templateStr.end = Iterator().Index();
472 Iterator().Forward(1);
473
474 if (Iterator().Peek() == LEX_CHAR_LEFT_BRACE) {
475 Iterator().Forward(1);
476 templateStr.scanExpression = true;
477 SkipWhiteSpaces();
478 return templateStr;
479 }
480
481 templateStr.str.Append(cp);
482 continue;
483 }
484 default: {
485 break;
486 }
487 }
488
489 templateStr.str.Append(cp);
490 Iterator().Forward(cpSize);
491 }
492
493 UNREACHABLE();
494 return templateStr;
495 }
496
ResetTokenEnd()497 void Lexer::ResetTokenEnd()
498 {
499 SetTokenStart();
500 pos_.iterator_.Reset(GetToken().End().index);
501 pos_.line_ = GetToken().End().line;
502 pos_.nextTokenLine_ = 0;
503 }
504
ScanStringUnicodePart(util::UString * str)505 void Lexer::ScanStringUnicodePart(util::UString *str)
506 {
507 if (auto cp = ScanUnicodeCharacter(); cp != util::StringView::Iterator::INVALID_CP) {
508 str->Append(cp);
509 }
510 }
511
ScanUnicodeCharacter()512 char32_t Lexer::ScanUnicodeCharacter()
513 {
514 size_t cpSize {};
515 char32_t cp = Iterator().PeekCp(&cpSize);
516
517 switch (cp) {
518 case util::StringView::Iterator::INVALID_CP: {
519 ThrowError("Unterminated string");
520 break;
521 }
522 case LEX_CHAR_CR: {
523 Iterator().Forward(1);
524 if (Iterator().Peek() != LEX_CHAR_LF) {
525 Iterator().Backward(1);
526 }
527
528 [[fallthrough]];
529 }
530 case LEX_CHAR_LS:
531 case LEX_CHAR_PS:
532 case LEX_CHAR_LF: {
533 pos_.line_++;
534 Iterator().Forward(cpSize);
535 return util::StringView::Iterator::INVALID_CP;
536 }
537 case LEX_CHAR_LOWERCASE_B: {
538 cp = LEX_CHAR_BS;
539 break;
540 }
541 case LEX_CHAR_LOWERCASE_T: {
542 cp = LEX_CHAR_TAB;
543 break;
544 }
545 case LEX_CHAR_LOWERCASE_N: {
546 cp = LEX_CHAR_LF;
547 break;
548 }
549 case LEX_CHAR_LOWERCASE_V: {
550 cp = LEX_CHAR_VT;
551 break;
552 }
553 case LEX_CHAR_LOWERCASE_F: {
554 cp = LEX_CHAR_FF;
555 break;
556 }
557 case LEX_CHAR_LOWERCASE_R: {
558 cp = LEX_CHAR_CR;
559 break;
560 }
561 case LEX_CHAR_LOWERCASE_X: {
562 Iterator().Forward(1);
563 cp = ScanHexEscape<2U>();
564 return cp;
565 }
566 case LEX_CHAR_LOWERCASE_U: {
567 cp = ScanUnicodeEscapeSequence();
568 return cp;
569 }
570 case LEX_CHAR_0: {
571 Iterator().Forward(1);
572 bool isDecimal = IsDecimalDigit(Iterator().Peek());
573 Iterator().Backward(1);
574
575 if (!isDecimal) {
576 cp = LEX_CHAR_NULL;
577 break;
578 }
579
580 [[fallthrough]];
581 }
582 default: {
583 if (IsDecimalDigit(Iterator().Peek())) {
584 ThrowError("Invalid character escape sequence in strict mode");
585 }
586
587 break;
588 }
589 }
590
591 Iterator().Forward(cpSize);
592 return cp;
593 }
594
ScanQuestionPunctuator()595 void Lexer::ScanQuestionPunctuator()
596 {
597 GetToken().type_ = TokenType::PUNCTUATOR_QUESTION_MARK;
598
599 switch (Iterator().Peek()) {
600 case LEX_CHAR_QUESTION: {
601 GetToken().type_ = TokenType::PUNCTUATOR_NULLISH_COALESCING;
602 Iterator().Forward(1);
603
604 switch (Iterator().Peek()) {
605 case LEX_CHAR_EQUALS: {
606 GetToken().type_ = TokenType::PUNCTUATOR_LOGICAL_NULLISH_EQUAL;
607 Iterator().Forward(1);
608 break;
609 }
610 default: {
611 break;
612 }
613 }
614
615 break;
616 }
617 case LEX_CHAR_DOT: {
618 Iterator().Forward(1);
619
620 if (!IsDecimalDigit(Iterator().Peek())) {
621 GetToken().type_ = TokenType::PUNCTUATOR_QUESTION_DOT;
622 return;
623 }
624
625 Iterator().Backward(1);
626 break;
627 }
628 default: {
629 break;
630 }
631 }
632 }
633
ScanLessThanPunctuator()634 void Lexer::ScanLessThanPunctuator()
635 {
636 GetToken().type_ = TokenType::PUNCTUATOR_LESS_THAN;
637
638 switch (Iterator().Peek()) {
639 case LEX_CHAR_LESS_THAN: {
640 GetToken().type_ = TokenType::PUNCTUATOR_LEFT_SHIFT;
641 Iterator().Forward(1);
642
643 if (Iterator().Peek() == LEX_CHAR_EQUALS) {
644 GetToken().type_ = TokenType::PUNCTUATOR_LEFT_SHIFT_EQUAL;
645 Iterator().Forward(1);
646 }
647 break;
648 }
649 case LEX_CHAR_EQUALS: {
650 GetToken().type_ = TokenType::PUNCTUATOR_LESS_THAN_EQUAL;
651 Iterator().Forward(1);
652 break;
653 }
654 default: {
655 break;
656 }
657 }
658 }
659
ScanGreaterThanPunctuator()660 void Lexer::ScanGreaterThanPunctuator()
661 {
662 GetToken().type_ = TokenType::PUNCTUATOR_GREATER_THAN;
663
664 switch (Iterator().Peek()) {
665 case LEX_CHAR_GREATER_THAN: {
666 GetToken().type_ = TokenType::PUNCTUATOR_RIGHT_SHIFT;
667 Iterator().Forward(1);
668
669 switch (Iterator().Peek()) {
670 case LEX_CHAR_GREATER_THAN: {
671 GetToken().type_ = TokenType::PUNCTUATOR_UNSIGNED_RIGHT_SHIFT;
672 Iterator().Forward(1);
673
674 if (Iterator().Peek() == LEX_CHAR_EQUALS) {
675 GetToken().type_ = TokenType::PUNCTUATOR_UNSIGNED_RIGHT_SHIFT_EQUAL;
676 Iterator().Forward(1);
677 }
678 break;
679 }
680 case LEX_CHAR_EQUALS: {
681 GetToken().type_ = TokenType::PUNCTUATOR_RIGHT_SHIFT_EQUAL;
682 Iterator().Forward(1);
683 break;
684 }
685 default: {
686 break;
687 }
688 }
689 break;
690 }
691 case LEX_CHAR_EQUALS: {
692 GetToken().type_ = TokenType::PUNCTUATOR_GREATER_THAN_EQUAL;
693 Iterator().Forward(1);
694 break;
695 }
696 default: {
697 break;
698 }
699 }
700 }
701
ScanEqualsPunctuator()702 void Lexer::ScanEqualsPunctuator()
703 {
704 GetToken().type_ = TokenType::PUNCTUATOR_SUBSTITUTION;
705
706 switch (Iterator().Peek()) {
707 case LEX_CHAR_EQUALS: {
708 GetToken().type_ = TokenType::PUNCTUATOR_EQUAL;
709 Iterator().Forward(1);
710
711 if (Iterator().Peek() == LEX_CHAR_EQUALS) {
712 GetToken().type_ = TokenType::PUNCTUATOR_STRICT_EQUAL;
713 Iterator().Forward(1);
714 }
715 break;
716 }
717 case LEX_CHAR_GREATER_THAN: {
718 GetToken().type_ = TokenType::PUNCTUATOR_ARROW;
719 Iterator().Forward(1);
720 break;
721 }
722 default: {
723 break;
724 }
725 }
726 }
727
ScanExclamationPunctuator()728 void Lexer::ScanExclamationPunctuator()
729 {
730 GetToken().type_ = TokenType::PUNCTUATOR_EXCLAMATION_MARK;
731
732 switch (Iterator().Peek()) {
733 case LEX_CHAR_EQUALS: {
734 GetToken().type_ = TokenType::PUNCTUATOR_NOT_EQUAL;
735 Iterator().Forward(1);
736
737 if (Iterator().Peek() == LEX_CHAR_EQUALS) {
738 GetToken().type_ = TokenType::PUNCTUATOR_NOT_STRICT_EQUAL;
739 Iterator().Forward(1);
740 }
741 break;
742 }
743 default: {
744 break;
745 }
746 }
747 }
748
ScanAmpersandPunctuator()749 void Lexer::ScanAmpersandPunctuator()
750 {
751 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_AND;
752
753 switch (Iterator().Peek()) {
754 case LEX_CHAR_AMPERSAND: {
755 GetToken().type_ = TokenType::PUNCTUATOR_LOGICAL_AND;
756 Iterator().Forward(1);
757
758 switch (Iterator().Peek()) {
759 case LEX_CHAR_EQUALS: {
760 GetToken().type_ = TokenType::PUNCTUATOR_LOGICAL_AND_EQUAL;
761 Iterator().Forward(1);
762 break;
763 }
764 default: {
765 break;
766 }
767 }
768
769 break;
770 }
771 case LEX_CHAR_EQUALS: {
772 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_AND_EQUAL;
773 Iterator().Forward(1);
774 break;
775 }
776 default: {
777 break;
778 }
779 }
780 }
781
ScanAtPunctuator()782 void Lexer::ScanAtPunctuator()
783 {
784 GetToken().type_ = TokenType::PUNCTUATOR_AT;
785
786 if (Iterator().Peek() == LEX_CHAR_AT) {
787 GetToken().type_ = TokenType::PUNCTUATOR_FORMAT;
788 Iterator().Forward(1U);
789 }
790 }
791
ScanVLinePunctuator()792 void Lexer::ScanVLinePunctuator()
793 {
794 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_OR;
795
796 switch (Iterator().Peek()) {
797 case LEX_CHAR_VLINE: {
798 GetToken().type_ = TokenType::PUNCTUATOR_LOGICAL_OR;
799 Iterator().Forward(1);
800
801 switch (Iterator().Peek()) {
802 case LEX_CHAR_EQUALS: {
803 GetToken().type_ = TokenType::PUNCTUATOR_LOGICAL_OR_EQUAL;
804 Iterator().Forward(1);
805 break;
806 }
807 default: {
808 break;
809 }
810 }
811
812 break;
813 }
814 case LEX_CHAR_EQUALS: {
815 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_OR_EQUAL;
816 Iterator().Forward(1);
817 break;
818 }
819 default: {
820 break;
821 }
822 }
823 }
824
ScanCircumflexPunctuator()825 void Lexer::ScanCircumflexPunctuator()
826 {
827 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_XOR;
828
829 switch (Iterator().Peek()) {
830 case LEX_CHAR_EQUALS: {
831 GetToken().type_ = TokenType::PUNCTUATOR_BITWISE_XOR_EQUAL;
832 Iterator().Forward(1);
833 break;
834 }
835 default: {
836 break;
837 }
838 }
839 }
840
ScanPlusPunctuator()841 void Lexer::ScanPlusPunctuator()
842 {
843 GetToken().type_ = TokenType::PUNCTUATOR_PLUS;
844
845 switch (Iterator().Peek()) {
846 case LEX_CHAR_PLUS: {
847 GetToken().type_ = TokenType::PUNCTUATOR_PLUS_PLUS;
848 Iterator().Forward(1);
849 break;
850 }
851 case LEX_CHAR_EQUALS: {
852 GetToken().type_ = TokenType::PUNCTUATOR_PLUS_EQUAL;
853 Iterator().Forward(1);
854 break;
855 }
856 default: {
857 break;
858 }
859 }
860 }
861
ScanMinusPunctuator()862 void Lexer::ScanMinusPunctuator()
863 {
864 GetToken().type_ = TokenType::PUNCTUATOR_MINUS;
865
866 switch (Iterator().Peek()) {
867 case LEX_CHAR_MINUS: {
868 GetToken().type_ = TokenType::PUNCTUATOR_MINUS_MINUS;
869 Iterator().Forward(1);
870 break;
871 }
872 case LEX_CHAR_EQUALS: {
873 GetToken().type_ = TokenType::PUNCTUATOR_MINUS_EQUAL;
874 Iterator().Forward(1);
875 break;
876 }
877 default: {
878 break;
879 }
880 }
881 }
882
ScanSlashPunctuator()883 void Lexer::ScanSlashPunctuator()
884 {
885 GetToken().type_ = TokenType::PUNCTUATOR_DIVIDE;
886
887 switch (Iterator().Peek()) {
888 case LEX_CHAR_EQUALS: {
889 GetToken().type_ = TokenType::PUNCTUATOR_DIVIDE_EQUAL;
890 Iterator().Forward(1);
891 break;
892 }
893 default: {
894 break;
895 }
896 }
897 }
898
ScanDotPunctuator()899 void Lexer::ScanDotPunctuator()
900 {
901 GetToken().type_ = TokenType::PUNCTUATOR_PERIOD;
902
903 switch (Iterator().Peek()) {
904 case LEX_CHAR_0:
905 case LEX_CHAR_1:
906 case LEX_CHAR_2:
907 case LEX_CHAR_3:
908 case LEX_CHAR_4:
909 case LEX_CHAR_5:
910 case LEX_CHAR_6:
911 case LEX_CHAR_7:
912 case LEX_CHAR_8:
913 case LEX_CHAR_9: {
914 ScanNumber();
915 break;
916 }
917 case LEX_CHAR_QUESTION: {
918 GetToken().type_ = TokenType::PUNCTUATOR_PERIOD_QUESTION;
919 Iterator().Forward(1);
920 break;
921 }
922 case LEX_CHAR_DOT: {
923 Iterator().Forward(1);
924
925 if (Iterator().Peek() == LEX_CHAR_DOT) {
926 GetToken().type_ = TokenType::PUNCTUATOR_PERIOD_PERIOD_PERIOD;
927 Iterator().Forward(1);
928 break;
929 }
930
931 Iterator().Backward(1);
932 break;
933 }
934 default: {
935 break;
936 }
937 }
938 }
939
ScanAsteriskPunctuator()940 void Lexer::ScanAsteriskPunctuator()
941 {
942 GetToken().type_ = TokenType::PUNCTUATOR_MULTIPLY;
943
944 switch (Iterator().Peek()) {
945 case LEX_CHAR_ASTERISK: {
946 GetToken().type_ = TokenType::PUNCTUATOR_EXPONENTIATION;
947 Iterator().Forward(1);
948
949 if (Iterator().Peek() == LEX_CHAR_EQUALS) {
950 GetToken().type_ = TokenType::PUNCTUATOR_EXPONENTIATION_EQUAL;
951 Iterator().Forward(1);
952 }
953 break;
954 }
955 case LEX_CHAR_EQUALS: {
956 GetToken().type_ = TokenType::PUNCTUATOR_MULTIPLY_EQUAL;
957 Iterator().Forward(1);
958 break;
959 }
960 default: {
961 break;
962 }
963 }
964 }
965
ScanPercentPunctuator()966 void Lexer::ScanPercentPunctuator()
967 {
968 GetToken().type_ = TokenType::PUNCTUATOR_MOD;
969
970 switch (Iterator().Peek()) {
971 case LEX_CHAR_EQUALS: {
972 GetToken().type_ = TokenType::PUNCTUATOR_MOD_EQUAL;
973 Iterator().Forward(1);
974 break;
975 }
976 default: {
977 break;
978 }
979 }
980 }
981
IsLineTerminatorOrEos() const982 bool Lexer::IsLineTerminatorOrEos() const
983 {
984 switch (Iterator().PeekCp()) {
985 case util::StringView::Iterator::INVALID_CP:
986 case LEX_CHAR_LF:
987 case LEX_CHAR_CR:
988 case LEX_CHAR_LS:
989 case LEX_CHAR_PS: {
990 return true;
991 }
992 default: {
993 break;
994 }
995 }
996
997 return false;
998 }
999
ScanRegExpPattern()1000 void Lexer::ScanRegExpPattern()
1001 {
1002 bool isCharClass = false;
1003 size_t cpSize {};
1004
1005 while (true) {
1006 switch (Iterator().PeekCp(&cpSize)) {
1007 case util::StringView::Iterator::INVALID_CP:
1008 case LEX_CHAR_LF:
1009 case LEX_CHAR_CR:
1010 case LEX_CHAR_LS:
1011 case LEX_CHAR_PS: {
1012 ThrowError("Unterminated RegExp");
1013 break;
1014 }
1015 case LEX_CHAR_SLASH: {
1016 if (!isCharClass) {
1017 return;
1018 }
1019
1020 break;
1021 }
1022 case LEX_CHAR_LEFT_SQUARE: {
1023 isCharClass = true;
1024 break;
1025 }
1026 case LEX_CHAR_RIGHT_SQUARE: {
1027 isCharClass = false;
1028 break;
1029 }
1030 case LEX_CHAR_BACKSLASH: {
1031 Iterator().Forward(1);
1032
1033 if (IsLineTerminatorOrEos()) {
1034 continue;
1035 }
1036
1037 break;
1038 }
1039 default: {
1040 break;
1041 }
1042 }
1043
1044 Iterator().Forward(cpSize);
1045 }
1046 }
1047
ScanRegExpFlags()1048 RegExpFlags Lexer::ScanRegExpFlags()
1049 {
1050 RegExpFlags resultFlags = RegExpFlags::EMPTY;
1051
1052 while (true) {
1053 size_t cpSize {};
1054 auto cp = Iterator().PeekCp(&cpSize);
1055 if (!KeywordsUtil::IsIdentifierPart(cp)) {
1056 break;
1057 }
1058
1059 Iterator().Forward(cpSize);
1060
1061 RegExpFlags flag = RegExpFlags::EMPTY;
1062
1063 switch (cp) {
1064 case LEX_CHAR_LOWERCASE_G: {
1065 flag = RegExpFlags::GLOBAL;
1066 break;
1067 }
1068 case LEX_CHAR_LOWERCASE_I: {
1069 flag = RegExpFlags::IGNORE_CASE;
1070 break;
1071 }
1072 case LEX_CHAR_LOWERCASE_M: {
1073 flag = RegExpFlags::MULTILINE;
1074 break;
1075 }
1076 case LEX_CHAR_LOWERCASE_S: {
1077 flag = RegExpFlags::DOTALL;
1078 break;
1079 }
1080 case LEX_CHAR_LOWERCASE_U: {
1081 flag = RegExpFlags::UNICODE;
1082 break;
1083 }
1084 case LEX_CHAR_LOWERCASE_Y: {
1085 flag = RegExpFlags::STICKY;
1086 break;
1087 }
1088 case LEX_CHAR_SP: {
1089 return resultFlags;
1090 }
1091 default: {
1092 ThrowError("Invalid RegExp flag");
1093 }
1094 }
1095
1096 if (flag == RegExpFlags::EMPTY || (resultFlags & flag) != 0) {
1097 ThrowError("Invalid RegExp flag");
1098 }
1099
1100 resultFlags = resultFlags | flag;
1101 }
1102
1103 return resultFlags;
1104 }
1105
ScanRegExp()1106 RegExp Lexer::ScanRegExp()
1107 {
1108 // for proper handling such regexps as /=/
1109 if (GetToken().Type() == lexer::TokenType::PUNCTUATOR_DIVIDE_EQUAL) {
1110 Iterator().Backward(1);
1111 }
1112
1113 GetToken().type_ = TokenType::LITERAL_REGEXP;
1114 GetToken().keywordType_ = TokenType::LITERAL_REGEXP;
1115
1116 const auto patternStart = Iterator().Index();
1117 ScanRegExpPattern();
1118 const auto pattern = SourceView(patternStart, Iterator().Index());
1119
1120 ASSERT(Iterator().Peek() == LEX_CHAR_SLASH);
1121 Iterator().Forward(1);
1122
1123 const auto flagsStart = Iterator().Index();
1124 RegExpFlags resultFlags = ScanRegExpFlags();
1125 const auto flags = SourceView(flagsStart, Iterator().Index());
1126
1127 SkipWhiteSpaces();
1128 SetTokenEnd();
1129
1130 return {pattern, flags, resultFlags};
1131 }
1132
CheckArrow()1133 bool Lexer::CheckArrow()
1134 {
1135 if (Iterator().Peek() != LEX_CHAR_EQUALS) {
1136 return false;
1137 }
1138 Iterator().Forward(1);
1139
1140 bool res = Iterator().Peek() == LEX_CHAR_GREATER_THAN;
1141 Iterator().Backward(1);
1142
1143 return res;
1144 }
1145
SetTokenStart()1146 void Lexer::SetTokenStart()
1147 {
1148 if (pos_.nextTokenLine_ != 0) {
1149 pos_.line_ += pos_.nextTokenLine_;
1150 pos_.nextTokenLine_ = 0;
1151 GetToken().flags_ = TokenFlags::NEW_LINE;
1152 } else {
1153 GetToken().flags_ = TokenFlags::NONE;
1154 }
1155
1156 pos_.token_.loc_.start = SourcePosition {Iterator().Index(), pos_.line_};
1157 GetToken().keywordType_ = TokenType::EOS;
1158 }
1159
SetTokenEnd()1160 void Lexer::SetTokenEnd()
1161 {
1162 pos_.token_.loc_.end = SourcePosition {Iterator().Index(), pos_.line_};
1163 }
1164
SkipWhiteSpaces()1165 void Lexer::SkipWhiteSpaces()
1166 {
1167 while (true) {
1168 auto cp = Iterator().Peek();
1169
1170 switch (cp) {
1171 case LEX_CHAR_CR: {
1172 Iterator().Forward(1);
1173
1174 if (Iterator().Peek() != LEX_CHAR_LF) {
1175 Iterator().Backward(1);
1176 }
1177
1178 [[fallthrough]];
1179 }
1180 case LEX_CHAR_LF: {
1181 Iterator().Forward(1);
1182 pos_.nextTokenLine_++;
1183 continue;
1184 }
1185 case LEX_CHAR_VT:
1186 case LEX_CHAR_FF:
1187 case LEX_CHAR_SP:
1188 case LEX_CHAR_TAB: {
1189 Iterator().Forward(1);
1190 continue;
1191 }
1192 case LEX_CHAR_SLASH: {
1193 Iterator().Forward(1);
1194 cp = Iterator().Peek();
1195 if (cp == LEX_CHAR_SLASH) {
1196 Iterator().Forward(1);
1197 SkipSingleLineComment();
1198 continue;
1199 }
1200 if (cp == LEX_CHAR_ASTERISK) {
1201 Iterator().Forward(1);
1202 SkipMultiLineComment();
1203 continue;
1204 }
1205
1206 Iterator().Backward(1);
1207 return;
1208 }
1209 default: {
1210 if (cp < LEX_ASCII_MAX_BITS) {
1211 return;
1212 }
1213
1214 size_t cpSize {};
1215 cp = Iterator().PeekCp(&cpSize);
1216
1217 switch (cp) {
1218 case LEX_CHAR_LS:
1219 case LEX_CHAR_PS: {
1220 pos_.nextTokenLine_++;
1221 [[fallthrough]];
1222 }
1223 case LEX_CHAR_NBSP:
1224 case LEX_CHAR_ZWNBSP: {
1225 Iterator().Forward(cpSize);
1226 continue;
1227 }
1228 default: {
1229 return;
1230 }
1231 }
1232 }
1233 }
1234 }
1235 }
1236
ScanHashMark()1237 void Lexer::ScanHashMark()
1238 {
1239 GetToken().type_ = TokenType::PUNCTUATOR_HASH_MARK;
1240 }
1241
ScanBackTick()1242 void Lexer::ScanBackTick()
1243 {
1244 GetToken().type_ = TokenType::PUNCTUATOR_BACK_TICK;
1245 SetTokenEnd();
1246 }
1247
1248 // NOLINTNEXTLINE(google-default-arguments)
NextToken(NextTokenFlags flags)1249 void Lexer::NextToken(NextTokenFlags flags)
1250 {
1251 JSKeywords kws(this, flags);
1252 NextToken(&kws);
1253 }
1254
ScanColonPunctuator()1255 void Lexer::ScanColonPunctuator()
1256 {
1257 GetToken().type_ = TokenType::PUNCTUATOR_COLON;
1258 }
1259
ScanDollarPunctuator()1260 bool Lexer::ScanDollarPunctuator()
1261 {
1262 return false;
1263 }
1264
1265 // NOLINTNEXTLINE(readability-function-size)
NextToken(Keywords * kws)1266 void Lexer::NextToken(Keywords *kws)
1267 {
1268 KeywordsUtil &kwu = kws->Util();
1269
1270 SetTokenStart();
1271
1272 auto cp = Iterator().Peek();
1273 Iterator().Forward(1);
1274
1275 switch (cp) {
1276 case LEX_CHAR_EXCLAMATION: {
1277 ScanExclamationPunctuator();
1278 break;
1279 }
1280 case LEX_CHAR_SINGLE_QUOTE: {
1281 ScanString<LEX_CHAR_SINGLE_QUOTE>();
1282 break;
1283 }
1284 case LEX_CHAR_DOUBLE_QUOTE: {
1285 ScanString<LEX_CHAR_DOUBLE_QUOTE>();
1286 break;
1287 }
1288 case LEX_CHAR_HASH_MARK: {
1289 ScanHashMark();
1290 break;
1291 }
1292 case LEX_CHAR_PERCENT: {
1293 ScanPercentPunctuator();
1294 break;
1295 }
1296 case LEX_CHAR_AMPERSAND: {
1297 ScanAmpersandPunctuator();
1298 break;
1299 }
1300 case LEX_CHAR_LEFT_PAREN: {
1301 GetToken().type_ = TokenType::PUNCTUATOR_LEFT_PARENTHESIS;
1302 break;
1303 }
1304 case LEX_CHAR_RIGHT_PAREN: {
1305 GetToken().type_ = TokenType::PUNCTUATOR_RIGHT_PARENTHESIS;
1306 break;
1307 }
1308 case LEX_CHAR_ASTERISK: {
1309 ScanAsteriskPunctuator();
1310 break;
1311 }
1312 case LEX_CHAR_PLUS: {
1313 ScanPlusPunctuator();
1314 break;
1315 }
1316 case LEX_CHAR_COMMA: {
1317 GetToken().type_ = TokenType::PUNCTUATOR_COMMA;
1318 break;
1319 }
1320 case LEX_CHAR_MINUS: {
1321 ScanMinusPunctuator();
1322 break;
1323 }
1324 case LEX_CHAR_DOT: {
1325 ScanDotPunctuator();
1326 break;
1327 }
1328 case LEX_CHAR_SLASH: {
1329 ScanSlashPunctuator();
1330 break;
1331 }
1332 case LEX_CHAR_0: {
1333 ScanNumberLeadingZero();
1334 break;
1335 }
1336 case LEX_CHAR_1:
1337 case LEX_CHAR_2:
1338 case LEX_CHAR_3:
1339 case LEX_CHAR_4:
1340 case LEX_CHAR_5:
1341 case LEX_CHAR_6:
1342 case LEX_CHAR_7:
1343 case LEX_CHAR_8:
1344 case LEX_CHAR_9: {
1345 ScanNumber();
1346 break;
1347 }
1348 case LEX_CHAR_COLON: {
1349 ScanColonPunctuator();
1350 break;
1351 }
1352 case LEX_CHAR_SEMICOLON: {
1353 GetToken().type_ = TokenType::PUNCTUATOR_SEMI_COLON;
1354 break;
1355 }
1356 case LEX_CHAR_LESS_THAN: {
1357 ScanLessThanPunctuator();
1358 break;
1359 }
1360 case LEX_CHAR_EQUALS: {
1361 ScanEqualsPunctuator();
1362 break;
1363 }
1364 case LEX_CHAR_GREATER_THAN: {
1365 ScanGreaterThanPunctuator();
1366 break;
1367 }
1368 case LEX_CHAR_QUESTION: {
1369 ScanQuestionPunctuator();
1370 break;
1371 }
1372 case LEX_CHAR_AT: {
1373 ScanAtPunctuator();
1374 break;
1375 }
1376 case LEX_CHAR_DOLLAR_SIGN: {
1377 if (ScanDollarPunctuator()) {
1378 break;
1379 }
1380
1381 [[fallthrough]];
1382 }
1383 case LEX_CHAR_UPPERCASE_A:
1384 case LEX_CHAR_UPPERCASE_B:
1385 case LEX_CHAR_UPPERCASE_C:
1386 case LEX_CHAR_UPPERCASE_D:
1387 case LEX_CHAR_UPPERCASE_E:
1388 case LEX_CHAR_UPPERCASE_F:
1389 case LEX_CHAR_UPPERCASE_G:
1390 case LEX_CHAR_UPPERCASE_H:
1391 case LEX_CHAR_UPPERCASE_I:
1392 case LEX_CHAR_UPPERCASE_J:
1393 case LEX_CHAR_UPPERCASE_K:
1394 case LEX_CHAR_UPPERCASE_L:
1395 case LEX_CHAR_UPPERCASE_M:
1396 case LEX_CHAR_UPPERCASE_N:
1397 case LEX_CHAR_UPPERCASE_O:
1398 case LEX_CHAR_UPPERCASE_P:
1399 case LEX_CHAR_UPPERCASE_Q:
1400 case LEX_CHAR_UPPERCASE_R:
1401 case LEX_CHAR_UPPERCASE_S:
1402 case LEX_CHAR_UPPERCASE_T:
1403 case LEX_CHAR_UPPERCASE_U:
1404 case LEX_CHAR_UPPERCASE_V:
1405 case LEX_CHAR_UPPERCASE_W:
1406 case LEX_CHAR_UPPERCASE_X:
1407 case LEX_CHAR_UPPERCASE_Y:
1408 case LEX_CHAR_UPPERCASE_Z:
1409 case LEX_CHAR_UNDERSCORE: {
1410 kwu.ScanIdContinue();
1411 break;
1412 }
1413 case LEX_CHAR_LEFT_SQUARE: {
1414 GetToken().type_ = TokenType::PUNCTUATOR_LEFT_SQUARE_BRACKET;
1415 break;
1416 }
1417 case LEX_CHAR_BACKSLASH: {
1418 GetToken().flags_ |= TokenFlags::HAS_ESCAPE;
1419
1420 if (Iterator().Peek() != LEX_CHAR_LOWERCASE_U) {
1421 ThrowError("Invalid character");
1422 }
1423
1424 cp = ScanUnicodeEscapeSequence();
1425
1426 kwu.ScanIdentifierStart(kws, cp);
1427 break;
1428 }
1429 case LEX_CHAR_RIGHT_SQUARE: {
1430 GetToken().type_ = TokenType::PUNCTUATOR_RIGHT_SQUARE_BRACKET;
1431 break;
1432 }
1433 case LEX_CHAR_CIRCUMFLEX: {
1434 ScanCircumflexPunctuator();
1435 break;
1436 }
1437 case LEX_CHAR_BACK_TICK: {
1438 ScanBackTick();
1439 return;
1440 }
1441 case LEX_CHAR_LOWERCASE_C: {
1442 if (ScanCharLiteral()) {
1443 break;
1444 }
1445 }
1446 [[fallthrough]];
1447 case LEX_CHAR_LOWERCASE_A:
1448 case LEX_CHAR_LOWERCASE_B:
1449 case LEX_CHAR_LOWERCASE_D:
1450 case LEX_CHAR_LOWERCASE_E:
1451 case LEX_CHAR_LOWERCASE_F:
1452 case LEX_CHAR_LOWERCASE_G:
1453 case LEX_CHAR_LOWERCASE_H:
1454 case LEX_CHAR_LOWERCASE_I:
1455 case LEX_CHAR_LOWERCASE_J:
1456 case LEX_CHAR_LOWERCASE_K:
1457 case LEX_CHAR_LOWERCASE_L:
1458 case LEX_CHAR_LOWERCASE_M:
1459 case LEX_CHAR_LOWERCASE_N:
1460 case LEX_CHAR_LOWERCASE_O:
1461 case LEX_CHAR_LOWERCASE_P:
1462 case LEX_CHAR_LOWERCASE_Q:
1463 case LEX_CHAR_LOWERCASE_R:
1464 case LEX_CHAR_LOWERCASE_S:
1465 case LEX_CHAR_LOWERCASE_T:
1466 case LEX_CHAR_LOWERCASE_U:
1467 case LEX_CHAR_LOWERCASE_V:
1468 case LEX_CHAR_LOWERCASE_W:
1469 case LEX_CHAR_LOWERCASE_X:
1470 case LEX_CHAR_LOWERCASE_Y:
1471 case LEX_CHAR_LOWERCASE_Z: {
1472 kws->ScanKeyword(cp);
1473 break;
1474 }
1475 case LEX_CHAR_LEFT_BRACE: {
1476 GetToken().type_ = TokenType::PUNCTUATOR_LEFT_BRACE;
1477
1478 if (tlCtx_ != nullptr) {
1479 tlCtx_->ConsumeLeftBrace();
1480 }
1481
1482 break;
1483 }
1484 case LEX_CHAR_VLINE: {
1485 ScanVLinePunctuator();
1486 break;
1487 }
1488 case LEX_CHAR_RIGHT_BRACE: {
1489 GetToken().type_ = TokenType::PUNCTUATOR_RIGHT_BRACE;
1490
1491 if (tlCtx_ != nullptr && tlCtx_->ConsumeRightBrace()) {
1492 SetTokenEnd();
1493 return;
1494 }
1495
1496 break;
1497 }
1498 case LEX_CHAR_TILDE: {
1499 GetToken().type_ = TokenType::PUNCTUATOR_TILDE;
1500 break;
1501 }
1502 default: {
1503 Iterator().Backward(1);
1504
1505 if (cp == util::StringView::Iterator::INVALID_CP) {
1506 GetToken().type_ = TokenType::EOS;
1507 break;
1508 }
1509
1510 cp = Iterator().Next();
1511 kwu.ScanIdentifierStart(kws, cp);
1512 break;
1513 }
1514 }
1515
1516 SetTokenEnd();
1517 SkipWhiteSpaces();
1518 }
1519
ScanNumberLeadingZeroImplNonAllowedCases()1520 void Lexer::ScanNumberLeadingZeroImplNonAllowedCases()
1521 {
1522 switch (Iterator().Peek()) {
1523 case LEX_CHAR_0:
1524 case LEX_CHAR_1:
1525 case LEX_CHAR_2:
1526 case LEX_CHAR_3:
1527 case LEX_CHAR_4:
1528 case LEX_CHAR_5:
1529 case LEX_CHAR_6:
1530 case LEX_CHAR_7: {
1531 ThrowError("Implicit octal literal not allowed");
1532 }
1533 case LEX_CHAR_8:
1534 case LEX_CHAR_9: {
1535 ThrowError("NonOctalDecimalIntegerLiteral is not enabled in strict mode code");
1536 }
1537 case LEX_CHAR_UNDERSCORE: {
1538 ThrowError("Numeric separator '_' is not allowed in numbers that start with '0'.");
1539 }
1540 default: {
1541 break;
1542 }
1543 }
1544 }
1545
Pos()1546 LexerPosition &Lexer::Pos()
1547 {
1548 return pos_;
1549 }
1550
Pos() const1551 const LexerPosition &Lexer::Pos() const
1552 {
1553 return pos_;
1554 }
1555 } // namespace panda::es2panda::lexer
1556