• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2012 the V8 project authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #ifndef V8_PARSING_TOKEN_H_
6 #define V8_PARSING_TOKEN_H_
7 
8 #include "src/base/bit-field.h"
9 #include "src/base/bounds.h"
10 #include "src/base/logging.h"
11 #include "src/common/globals.h"
12 
13 namespace v8 {
14 namespace internal {
15 
16 // TOKEN_LIST takes a list of 3 macros M, all of which satisfy the
17 // same signature M(name, string, precedence), where name is the
18 // symbolic token name, string is the corresponding syntactic symbol
19 // (or nullptr, for literals), and precedence is the precedence (or 0).
20 // The parameters are invoked for token categories as follows:
21 //
22 //   T: Non-keyword tokens
23 //   K: Keyword tokens
24 
25 // IGNORE_TOKEN is a convenience macro that can be supplied as
26 // an argument (at any position) for a TOKEN_LIST call. It does
27 // nothing with tokens belonging to the respective category.
28 
29 #define IGNORE_TOKEN(name, string, precedence)
30 
31 /* Binary operators */
32 /* ADD and SUB are at the end since they are UnaryOp */
33 #define BINARY_OP_TOKEN_LIST(T, E) \
34   E(T, NULLISH, "??", 3)           \
35   E(T, OR, "||", 4)                \
36   E(T, AND, "&&", 5)               \
37   E(T, BIT_OR, "|", 6)             \
38   E(T, BIT_XOR, "^", 7)            \
39   E(T, BIT_AND, "&", 8)            \
40   E(T, SHL, "<<", 11)              \
41   E(T, SAR, ">>", 11)              \
42   E(T, SHR, ">>>", 11)             \
43   E(T, MUL, "*", 13)               \
44   E(T, DIV, "/", 13)               \
45   E(T, MOD, "%", 13)               \
46   E(T, EXP, "**", 14)              \
47   E(T, ADD, "+", 12)               \
48   E(T, SUB, "-", 12)
49 
50 #define EXPAND_BINOP_ASSIGN_TOKEN(T, name, string, precedence) \
51   T(ASSIGN_##name, string "=", 2)
52 
53 #define EXPAND_BINOP_TOKEN(T, name, string, precedence) \
54   T(name, string, precedence)
55 
56 #define TOKEN_LIST(T, K)                                           \
57                                                                    \
58   /* BEGIN PropertyOrCall */                                       \
59   /* BEGIN Member */                                               \
60   /* BEGIN Template */                                             \
61   /* ES6 Template Literals */                                      \
62   T(TEMPLATE_SPAN, nullptr, 0)                                     \
63   T(TEMPLATE_TAIL, nullptr, 0)                                     \
64   /* END Template */                                               \
65                                                                    \
66   /* Punctuators (ECMA-262, section 7.7, page 15). */              \
67   /* BEGIN Property */                                             \
68   T(PERIOD, ".", 0)                                                \
69   T(LBRACK, "[", 0)                                                \
70   /* END Property */                                               \
71   /* END Member */                                                 \
72   T(QUESTION_PERIOD, "?.", 0)                                      \
73   T(LPAREN, "(", 0)                                                \
74   /* END PropertyOrCall */                                         \
75   T(RPAREN, ")", 0)                                                \
76   T(RBRACK, "]", 0)                                                \
77   T(LBRACE, "{", 0)                                                \
78   T(COLON, ":", 0)                                                 \
79   T(ELLIPSIS, "...", 0)                                            \
80   T(CONDITIONAL, "?", 3)                                           \
81   /* BEGIN AutoSemicolon */                                        \
82   T(SEMICOLON, ";", 0)                                             \
83   T(RBRACE, "}", 0)                                                \
84   /* End of source indicator. */                                   \
85   T(EOS, "EOS", 0)                                                 \
86   /* END AutoSemicolon */                                          \
87                                                                    \
88   /* BEGIN ArrowOrAssignmentOp */                                  \
89   T(ARROW, "=>", 0)                                                \
90   /* BEGIN AssignmentOp */                                         \
91   /* IsAssignmentOp() relies on this block of enum values being */ \
92   /* contiguous and sorted in the same order! */                   \
93   T(INIT, "=init", 2) /* AST-use only. */                          \
94   T(ASSIGN, "=", 2)                                                \
95   BINARY_OP_TOKEN_LIST(T, EXPAND_BINOP_ASSIGN_TOKEN)               \
96   /* END AssignmentOp */                                           \
97   /* END ArrowOrAssignmentOp */                                    \
98                                                                    \
99   /* Binary operators sorted by precedence. */                     \
100   /* IsBinaryOp() relies on this block of enum values */           \
101   /* being contiguous and sorted in the same order! */             \
102   T(COMMA, ",", 1)                                                 \
103                                                                    \
104   /* Unary operators, starting at ADD in BINARY_OP_TOKEN_LIST  */  \
105   /* IsUnaryOp() relies on this block of enum values */            \
106   /* being contiguous and sorted in the same order! */             \
107   BINARY_OP_TOKEN_LIST(T, EXPAND_BINOP_TOKEN)                      \
108                                                                    \
109   T(NOT, "!", 0)                                                   \
110   T(BIT_NOT, "~", 0)                                               \
111   K(DELETE, "delete", 0)                                           \
112   K(TYPEOF, "typeof", 0)                                           \
113   K(VOID, "void", 0)                                               \
114                                                                    \
115   /* BEGIN IsCountOp */                                            \
116   T(INC, "++", 0)                                                  \
117   T(DEC, "--", 0)                                                  \
118   /* END IsCountOp */                                              \
119   /* END IsUnaryOrCountOp */                                       \
120                                                                    \
121   /* Compare operators sorted by precedence. */                    \
122   /* IsCompareOp() relies on this block of enum values */          \
123   /* being contiguous and sorted in the same order! */             \
124   T(EQ, "==", 9)                                                   \
125   T(EQ_STRICT, "===", 9)                                           \
126   T(NE, "!=", 9)                                                   \
127   T(NE_STRICT, "!==", 9)                                           \
128   T(LT, "<", 10)                                                   \
129   T(GT, ">", 10)                                                   \
130   T(LTE, "<=", 10)                                                 \
131   T(GTE, ">=", 10)                                                 \
132   K(INSTANCEOF, "instanceof", 10)                                  \
133   K(IN, "in", 10)                                                  \
134                                                                    \
135   /* Keywords (ECMA-262, section 7.5.2, page 13). */               \
136   K(BREAK, "break", 0)                                             \
137   K(CASE, "case", 0)                                               \
138   K(CATCH, "catch", 0)                                             \
139   K(CONTINUE, "continue", 0)                                       \
140   K(DEBUGGER, "debugger", 0)                                       \
141   K(DEFAULT, "default", 0)                                         \
142   /* DELETE */                                                     \
143   K(DO, "do", 0)                                                   \
144   K(ELSE, "else", 0)                                               \
145   K(FINALLY, "finally", 0)                                         \
146   K(FOR, "for", 0)                                                 \
147   K(FUNCTION, "function", 0)                                       \
148   K(IF, "if", 0)                                                   \
149   /* IN */                                                         \
150   /* INSTANCEOF */                                                 \
151   K(NEW, "new", 0)                                                 \
152   K(RETURN, "return", 0)                                           \
153   K(SWITCH, "switch", 0)                                           \
154   K(THROW, "throw", 0)                                             \
155   K(TRY, "try", 0)                                                 \
156   /* TYPEOF */                                                     \
157   K(VAR, "var", 0)                                                 \
158   /* VOID */                                                       \
159   K(WHILE, "while", 0)                                             \
160   K(WITH, "with", 0)                                               \
161   K(THIS, "this", 0)                                               \
162                                                                    \
163   /* Literals (ECMA-262, section 7.8, page 16). */                 \
164   K(NULL_LITERAL, "null", 0)                                       \
165   K(TRUE_LITERAL, "true", 0)                                       \
166   K(FALSE_LITERAL, "false", 0)                                     \
167   T(NUMBER, nullptr, 0)                                            \
168   T(SMI, nullptr, 0)                                               \
169   T(BIGINT, nullptr, 0)                                            \
170   T(STRING, nullptr, 0)                                            \
171                                                                    \
172   /* BEGIN Callable */                                             \
173   K(SUPER, "super", 0)                                             \
174   /* BEGIN AnyIdentifier */                                        \
175   /* Identifiers (not keywords or future reserved words). */       \
176   T(IDENTIFIER, nullptr, 0)                                        \
177   K(GET, "get", 0)                                                 \
178   K(SET, "set", 0)                                                 \
179   K(ASYNC, "async", 0)                                             \
180   /* `await` is a reserved word in module code only */             \
181   K(AWAIT, "await", 0)                                             \
182   K(YIELD, "yield", 0)                                             \
183   K(LET, "let", 0)                                                 \
184   K(STATIC, "static", 0)                                           \
185   /* Future reserved words (ECMA-262, section 7.6.1.2). */         \
186   T(FUTURE_STRICT_RESERVED_WORD, nullptr, 0)                       \
187   T(ESCAPED_STRICT_RESERVED_WORD, nullptr, 0)                      \
188   /* END AnyIdentifier */                                          \
189   /* END Callable */                                               \
190   K(ENUM, "enum", 0)                                               \
191   K(CLASS, "class", 0)                                             \
192   K(CONST, "const", 0)                                             \
193   K(EXPORT, "export", 0)                                           \
194   K(EXTENDS, "extends", 0)                                         \
195   K(IMPORT, "import", 0)                                           \
196   T(PRIVATE_NAME, nullptr, 0)                                      \
197                                                                    \
198   /* Illegal token - not able to scan. */                          \
199   T(ILLEGAL, "ILLEGAL", 0)                                         \
200   T(ESCAPED_KEYWORD, nullptr, 0)                                   \
201                                                                    \
202   /* Scanner-internal use only. */                                 \
203   T(WHITESPACE, nullptr, 0)                                        \
204   T(UNINITIALIZED, nullptr, 0)                                     \
205   T(REGEXP_LITERAL, nullptr, 0)
206 
207 class V8_EXPORT_PRIVATE Token {
208  public:
209   // All token values.
210 #define T(name, string, precedence) name,
211   enum Value : uint8_t { TOKEN_LIST(T, T) NUM_TOKENS };
212 #undef T
213 
214   // Returns a string corresponding to the C++ token name
215   // (e.g. "LT" for the token LT).
Name(Value token)216   static const char* Name(Value token) {
217     DCHECK_GT(NUM_TOKENS, token);  // token is unsigned
218     return name_[token];
219   }
220 
221   using IsKeywordBits = base::BitField8<bool, 0, 1>;
222   using IsPropertyNameBits = IsKeywordBits::Next<bool, 1>;
223 
224   // Predicates
IsKeyword(Value token)225   static bool IsKeyword(Value token) {
226     return IsKeywordBits::decode(token_flags[token]);
227   }
228 
IsPropertyName(Value token)229   static bool IsPropertyName(Value token) {
230     return IsPropertyNameBits::decode(token_flags[token]);
231   }
232 
IsValidIdentifier(Value token,LanguageMode language_mode,bool is_generator,bool disallow_await)233   V8_INLINE static bool IsValidIdentifier(Value token,
234                                           LanguageMode language_mode,
235                                           bool is_generator,
236                                           bool disallow_await) {
237     if (V8_LIKELY(base::IsInRange(token, IDENTIFIER, ASYNC))) return true;
238     if (token == AWAIT) return !disallow_await;
239     if (token == YIELD) return !is_generator && is_sloppy(language_mode);
240     return IsStrictReservedWord(token) && is_sloppy(language_mode);
241   }
242 
IsCallable(Value token)243   static bool IsCallable(Value token) {
244     return base::IsInRange(token, SUPER, ESCAPED_STRICT_RESERVED_WORD);
245   }
246 
IsAutoSemicolon(Value token)247   static bool IsAutoSemicolon(Value token) {
248     return base::IsInRange(token, SEMICOLON, EOS);
249   }
250 
IsAnyIdentifier(Value token)251   static bool IsAnyIdentifier(Value token) {
252     return base::IsInRange(token, IDENTIFIER, ESCAPED_STRICT_RESERVED_WORD);
253   }
254 
IsStrictReservedWord(Value token)255   static bool IsStrictReservedWord(Value token) {
256     return base::IsInRange(token, YIELD, ESCAPED_STRICT_RESERVED_WORD);
257   }
258 
IsLiteral(Value token)259   static bool IsLiteral(Value token) {
260     return base::IsInRange(token, NULL_LITERAL, STRING);
261   }
262 
IsTemplate(Value token)263   static bool IsTemplate(Value token) {
264     return base::IsInRange(token, TEMPLATE_SPAN, TEMPLATE_TAIL);
265   }
266 
IsMember(Value token)267   static bool IsMember(Value token) {
268     return base::IsInRange(token, TEMPLATE_SPAN, LBRACK);
269   }
270 
IsProperty(Value token)271   static bool IsProperty(Value token) {
272     return base::IsInRange(token, PERIOD, LBRACK);
273   }
274 
IsPropertyOrCall(Value token)275   static bool IsPropertyOrCall(Value token) {
276     return base::IsInRange(token, TEMPLATE_SPAN, LPAREN);
277   }
278 
IsArrowOrAssignmentOp(Value token)279   static bool IsArrowOrAssignmentOp(Value token) {
280     return base::IsInRange(token, ARROW, ASSIGN_SUB);
281   }
282 
IsAssignmentOp(Value token)283   static bool IsAssignmentOp(Value token) {
284     return base::IsInRange(token, INIT, ASSIGN_SUB);
285   }
286 
IsLogicalAssignmentOp(Value token)287   static bool IsLogicalAssignmentOp(Value token) {
288     return base::IsInRange(token, ASSIGN_NULLISH, ASSIGN_AND);
289   }
290 
IsBinaryOp(Value op)291   static bool IsBinaryOp(Value op) { return base::IsInRange(op, COMMA, SUB); }
292 
IsCompareOp(Value op)293   static bool IsCompareOp(Value op) { return base::IsInRange(op, EQ, IN); }
294 
IsOrderedRelationalCompareOp(Value op)295   static bool IsOrderedRelationalCompareOp(Value op) {
296     return base::IsInRange(op, LT, GTE);
297   }
298 
IsEqualityOp(Value op)299   static bool IsEqualityOp(Value op) {
300     return base::IsInRange(op, EQ, EQ_STRICT);
301   }
302 
BinaryOpForAssignment(Value op)303   static Value BinaryOpForAssignment(Value op) {
304     DCHECK(base::IsInRange(op, ASSIGN_NULLISH, ASSIGN_SUB));
305     Value result = static_cast<Value>(op - ASSIGN_NULLISH + NULLISH);
306     DCHECK(IsBinaryOp(result));
307     return result;
308   }
309 
IsBitOp(Value op)310   static bool IsBitOp(Value op) {
311     return base::IsInRange(op, BIT_OR, SHR) || op == BIT_NOT;
312   }
313 
IsUnaryOp(Value op)314   static bool IsUnaryOp(Value op) { return base::IsInRange(op, ADD, VOID); }
IsCountOp(Value op)315   static bool IsCountOp(Value op) { return base::IsInRange(op, INC, DEC); }
IsUnaryOrCountOp(Value op)316   static bool IsUnaryOrCountOp(Value op) {
317     return base::IsInRange(op, ADD, DEC);
318   }
IsShiftOp(Value op)319   static bool IsShiftOp(Value op) { return base::IsInRange(op, SHL, SHR); }
320 
321   // Returns a string corresponding to the JS token string
322   // (.e., "<" for the token LT) or nullptr if the token doesn't
323   // have a (unique) string (e.g. an IDENTIFIER).
String(Value token)324   static const char* String(Value token) {
325     DCHECK_GT(NUM_TOKENS, token);  // token is unsigned
326     return string_[token];
327   }
328 
StringLength(Value token)329   static uint8_t StringLength(Value token) {
330     DCHECK_GT(NUM_TOKENS, token);  // token is unsigned
331     return string_length_[token];
332   }
333 
334   // Returns the precedence > 0 for binary and compare
335   // operators; returns 0 otherwise.
Precedence(Value token,bool accept_IN)336   static int Precedence(Value token, bool accept_IN) {
337     DCHECK_GT(NUM_TOKENS, token);  // token is unsigned
338     return precedence_[accept_IN][token];
339   }
340 
341  private:
342   static const char* const name_[NUM_TOKENS];
343   static const char* const string_[NUM_TOKENS];
344   static const uint8_t string_length_[NUM_TOKENS];
345   static const int8_t precedence_[2][NUM_TOKENS];
346   static const uint8_t token_flags[NUM_TOKENS];
347 };
348 
349 }  // namespace internal
350 }  // namespace v8
351 
352 #endif  // V8_PARSING_TOKEN_H_
353