1"""Token constants.""" 2# Auto-generated by Tools/scripts/generate_token.py 3 4__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] 5 6ENDMARKER = 0 7NAME = 1 8NUMBER = 2 9STRING = 3 10NEWLINE = 4 11INDENT = 5 12DEDENT = 6 13LPAR = 7 14RPAR = 8 15LSQB = 9 16RSQB = 10 17COLON = 11 18COMMA = 12 19SEMI = 13 20PLUS = 14 21MINUS = 15 22STAR = 16 23SLASH = 17 24VBAR = 18 25AMPER = 19 26LESS = 20 27GREATER = 21 28EQUAL = 22 29DOT = 23 30PERCENT = 24 31LBRACE = 25 32RBRACE = 26 33EQEQUAL = 27 34NOTEQUAL = 28 35LESSEQUAL = 29 36GREATEREQUAL = 30 37TILDE = 31 38CIRCUMFLEX = 32 39LEFTSHIFT = 33 40RIGHTSHIFT = 34 41DOUBLESTAR = 35 42PLUSEQUAL = 36 43MINEQUAL = 37 44STAREQUAL = 38 45SLASHEQUAL = 39 46PERCENTEQUAL = 40 47AMPEREQUAL = 41 48VBAREQUAL = 42 49CIRCUMFLEXEQUAL = 43 50LEFTSHIFTEQUAL = 44 51RIGHTSHIFTEQUAL = 45 52DOUBLESTAREQUAL = 46 53DOUBLESLASH = 47 54DOUBLESLASHEQUAL = 48 55AT = 49 56ATEQUAL = 50 57RARROW = 51 58ELLIPSIS = 52 59COLONEQUAL = 53 60OP = 54 61AWAIT = 55 62ASYNC = 56 63TYPE_IGNORE = 57 64TYPE_COMMENT = 58 65# These aren't used by the C tokenizer but are needed for tokenize.py 66ERRORTOKEN = 59 67COMMENT = 60 68NL = 61 69ENCODING = 62 70N_TOKENS = 63 71# Special definitions for cooperation with parser 72NT_OFFSET = 256 73 74tok_name = {value: name 75 for name, value in globals().items() 76 if isinstance(value, int) and not name.startswith('_')} 77__all__.extend(tok_name.values()) 78 79EXACT_TOKEN_TYPES = { 80 '!=': NOTEQUAL, 81 '%': PERCENT, 82 '%=': PERCENTEQUAL, 83 '&': AMPER, 84 '&=': AMPEREQUAL, 85 '(': LPAR, 86 ')': RPAR, 87 '*': STAR, 88 '**': DOUBLESTAR, 89 '**=': DOUBLESTAREQUAL, 90 '*=': STAREQUAL, 91 '+': PLUS, 92 '+=': PLUSEQUAL, 93 ',': COMMA, 94 '-': MINUS, 95 '-=': MINEQUAL, 96 '->': RARROW, 97 '.': DOT, 98 '...': ELLIPSIS, 99 '/': SLASH, 100 '//': DOUBLESLASH, 101 '//=': DOUBLESLASHEQUAL, 102 '/=': SLASHEQUAL, 103 ':': COLON, 104 ':=': COLONEQUAL, 105 ';': SEMI, 106 '<': LESS, 107 '<<': LEFTSHIFT, 108 '<<=': LEFTSHIFTEQUAL, 109 '<=': LESSEQUAL, 110 '=': EQUAL, 111 '==': EQEQUAL, 112 '>': GREATER, 113 '>=': GREATEREQUAL, 114 '>>': RIGHTSHIFT, 115 '>>=': RIGHTSHIFTEQUAL, 116 '@': AT, 117 '@=': ATEQUAL, 118 '[': LSQB, 119 ']': RSQB, 120 '^': CIRCUMFLEX, 121 '^=': CIRCUMFLEXEQUAL, 122 '{': LBRACE, 123 '|': VBAR, 124 '|=': VBAREQUAL, 125 '}': RBRACE, 126 '~': TILDE, 127} 128 129def ISTERMINAL(x): 130 return x < NT_OFFSET 131 132def ISNONTERMINAL(x): 133 return x >= NT_OFFSET 134 135def ISEOF(x): 136 return x == ENDMARKER 137