1"""Token constants.""" 2# Auto-generated by Tools/scripts/generate_token.py 3 4__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] 5 6ENDMARKER = 0 7NAME = 1 8NUMBER = 2 9STRING = 3 10NEWLINE = 4 11INDENT = 5 12DEDENT = 6 13LPAR = 7 14RPAR = 8 15LSQB = 9 16RSQB = 10 17COLON = 11 18COMMA = 12 19SEMI = 13 20PLUS = 14 21MINUS = 15 22STAR = 16 23SLASH = 17 24VBAR = 18 25AMPER = 19 26LESS = 20 27GREATER = 21 28EQUAL = 22 29DOT = 23 30PERCENT = 24 31LBRACE = 25 32RBRACE = 26 33EQEQUAL = 27 34NOTEQUAL = 28 35LESSEQUAL = 29 36GREATEREQUAL = 30 37TILDE = 31 38CIRCUMFLEX = 32 39LEFTSHIFT = 33 40RIGHTSHIFT = 34 41DOUBLESTAR = 35 42PLUSEQUAL = 36 43MINEQUAL = 37 44STAREQUAL = 38 45SLASHEQUAL = 39 46PERCENTEQUAL = 40 47AMPEREQUAL = 41 48VBAREQUAL = 42 49CIRCUMFLEXEQUAL = 43 50LEFTSHIFTEQUAL = 44 51RIGHTSHIFTEQUAL = 45 52DOUBLESTAREQUAL = 46 53DOUBLESLASH = 47 54DOUBLESLASHEQUAL = 48 55AT = 49 56ATEQUAL = 50 57RARROW = 51 58ELLIPSIS = 52 59COLONEQUAL = 53 60OP = 54 61AWAIT = 55 62ASYNC = 56 63TYPE_IGNORE = 57 64TYPE_COMMENT = 58 65SOFT_KEYWORD = 59 66# These aren't used by the C tokenizer but are needed for tokenize.py 67ERRORTOKEN = 60 68COMMENT = 61 69NL = 62 70ENCODING = 63 71N_TOKENS = 64 72# Special definitions for cooperation with parser 73NT_OFFSET = 256 74 75tok_name = {value: name 76 for name, value in globals().items() 77 if isinstance(value, int) and not name.startswith('_')} 78__all__.extend(tok_name.values()) 79 80EXACT_TOKEN_TYPES = { 81 '!=': NOTEQUAL, 82 '%': PERCENT, 83 '%=': PERCENTEQUAL, 84 '&': AMPER, 85 '&=': AMPEREQUAL, 86 '(': LPAR, 87 ')': RPAR, 88 '*': STAR, 89 '**': DOUBLESTAR, 90 '**=': DOUBLESTAREQUAL, 91 '*=': STAREQUAL, 92 '+': PLUS, 93 '+=': PLUSEQUAL, 94 ',': COMMA, 95 '-': MINUS, 96 '-=': MINEQUAL, 97 '->': RARROW, 98 '.': DOT, 99 '...': ELLIPSIS, 100 '/': SLASH, 101 '//': DOUBLESLASH, 102 '//=': DOUBLESLASHEQUAL, 103 '/=': SLASHEQUAL, 104 ':': COLON, 105 ':=': COLONEQUAL, 106 ';': SEMI, 107 '<': LESS, 108 '<<': LEFTSHIFT, 109 '<<=': LEFTSHIFTEQUAL, 110 '<=': LESSEQUAL, 111 '=': EQUAL, 112 '==': EQEQUAL, 113 '>': GREATER, 114 '>=': GREATEREQUAL, 115 '>>': RIGHTSHIFT, 116 '>>=': RIGHTSHIFTEQUAL, 117 '@': AT, 118 '@=': ATEQUAL, 119 '[': LSQB, 120 ']': RSQB, 121 '^': CIRCUMFLEX, 122 '^=': CIRCUMFLEXEQUAL, 123 '{': LBRACE, 124 '|': VBAR, 125 '|=': VBAREQUAL, 126 '}': RBRACE, 127 '~': TILDE, 128} 129 130def ISTERMINAL(x): 131 return x < NT_OFFSET 132 133def ISNONTERMINAL(x): 134 return x >= NT_OFFSET 135 136def ISEOF(x): 137 return x == ENDMARKER 138