• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1"""Token constants."""
2# Auto-generated by Tools/scripts/generate_token.py
3
4__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF']
5
6ENDMARKER = 0
7NAME = 1
8NUMBER = 2
9STRING = 3
10NEWLINE = 4
11INDENT = 5
12DEDENT = 6
13LPAR = 7
14RPAR = 8
15LSQB = 9
16RSQB = 10
17COLON = 11
18COMMA = 12
19SEMI = 13
20PLUS = 14
21MINUS = 15
22STAR = 16
23SLASH = 17
24VBAR = 18
25AMPER = 19
26LESS = 20
27GREATER = 21
28EQUAL = 22
29DOT = 23
30PERCENT = 24
31LBRACE = 25
32RBRACE = 26
33EQEQUAL = 27
34NOTEQUAL = 28
35LESSEQUAL = 29
36GREATEREQUAL = 30
37TILDE = 31
38CIRCUMFLEX = 32
39LEFTSHIFT = 33
40RIGHTSHIFT = 34
41DOUBLESTAR = 35
42PLUSEQUAL = 36
43MINEQUAL = 37
44STAREQUAL = 38
45SLASHEQUAL = 39
46PERCENTEQUAL = 40
47AMPEREQUAL = 41
48VBAREQUAL = 42
49CIRCUMFLEXEQUAL = 43
50LEFTSHIFTEQUAL = 44
51RIGHTSHIFTEQUAL = 45
52DOUBLESTAREQUAL = 46
53DOUBLESLASH = 47
54DOUBLESLASHEQUAL = 48
55AT = 49
56ATEQUAL = 50
57RARROW = 51
58ELLIPSIS = 52
59COLONEQUAL = 53
60OP = 54
61AWAIT = 55
62ASYNC = 56
63TYPE_IGNORE = 57
64TYPE_COMMENT = 58
65SOFT_KEYWORD = 59
66# These aren't used by the C tokenizer but are needed for tokenize.py
67ERRORTOKEN = 60
68COMMENT = 61
69NL = 62
70ENCODING = 63
71N_TOKENS = 64
72# Special definitions for cooperation with parser
73NT_OFFSET = 256
74
75tok_name = {value: name
76            for name, value in globals().items()
77            if isinstance(value, int) and not name.startswith('_')}
78__all__.extend(tok_name.values())
79
80EXACT_TOKEN_TYPES = {
81    '!=': NOTEQUAL,
82    '%': PERCENT,
83    '%=': PERCENTEQUAL,
84    '&': AMPER,
85    '&=': AMPEREQUAL,
86    '(': LPAR,
87    ')': RPAR,
88    '*': STAR,
89    '**': DOUBLESTAR,
90    '**=': DOUBLESTAREQUAL,
91    '*=': STAREQUAL,
92    '+': PLUS,
93    '+=': PLUSEQUAL,
94    ',': COMMA,
95    '-': MINUS,
96    '-=': MINEQUAL,
97    '->': RARROW,
98    '.': DOT,
99    '...': ELLIPSIS,
100    '/': SLASH,
101    '//': DOUBLESLASH,
102    '//=': DOUBLESLASHEQUAL,
103    '/=': SLASHEQUAL,
104    ':': COLON,
105    ':=': COLONEQUAL,
106    ';': SEMI,
107    '<': LESS,
108    '<<': LEFTSHIFT,
109    '<<=': LEFTSHIFTEQUAL,
110    '<=': LESSEQUAL,
111    '=': EQUAL,
112    '==': EQEQUAL,
113    '>': GREATER,
114    '>=': GREATEREQUAL,
115    '>>': RIGHTSHIFT,
116    '>>=': RIGHTSHIFTEQUAL,
117    '@': AT,
118    '@=': ATEQUAL,
119    '[': LSQB,
120    ']': RSQB,
121    '^': CIRCUMFLEX,
122    '^=': CIRCUMFLEXEQUAL,
123    '{': LBRACE,
124    '|': VBAR,
125    '|=': VBAREQUAL,
126    '}': RBRACE,
127    '~': TILDE,
128}
129
130def ISTERMINAL(x):
131    return x < NT_OFFSET
132
133def ISNONTERMINAL(x):
134    return x >= NT_OFFSET
135
136def ISEOF(x):
137    return x == ENDMARKER
138