1#! /usr/bin/env python 2 3"""Token constants (from "token.h").""" 4 5# Taken from Python (r53757) and modified to include some tokens 6# originally monkeypatched in by pgen2.tokenize 7 8#--start constants-- 9ENDMARKER = 0 10NAME = 1 11NUMBER = 2 12STRING = 3 13NEWLINE = 4 14INDENT = 5 15DEDENT = 6 16LPAR = 7 17RPAR = 8 18LSQB = 9 19RSQB = 10 20COLON = 11 21COMMA = 12 22SEMI = 13 23PLUS = 14 24MINUS = 15 25STAR = 16 26SLASH = 17 27VBAR = 18 28AMPER = 19 29LESS = 20 30GREATER = 21 31EQUAL = 22 32DOT = 23 33PERCENT = 24 34BACKQUOTE = 25 35LBRACE = 26 36RBRACE = 27 37EQEQUAL = 28 38NOTEQUAL = 29 39LESSEQUAL = 30 40GREATEREQUAL = 31 41TILDE = 32 42CIRCUMFLEX = 33 43LEFTSHIFT = 34 44RIGHTSHIFT = 35 45DOUBLESTAR = 36 46PLUSEQUAL = 37 47MINEQUAL = 38 48STAREQUAL = 39 49SLASHEQUAL = 40 50PERCENTEQUAL = 41 51AMPEREQUAL = 42 52VBAREQUAL = 43 53CIRCUMFLEXEQUAL = 44 54LEFTSHIFTEQUAL = 45 55RIGHTSHIFTEQUAL = 46 56DOUBLESTAREQUAL = 47 57DOUBLESLASH = 48 58DOUBLESLASHEQUAL = 49 59AT = 50 60ATEQUAL = 51 61OP = 52 62COMMENT = 53 63NL = 54 64RARROW = 55 65ERRORTOKEN = 56 66N_TOKENS = 57 67NT_OFFSET = 256 68#--end constants-- 69 70tok_name = {} 71for _name, _value in globals().items(): 72 if type(_value) is type(0): 73 tok_name[_value] = _name 74 75 76def ISTERMINAL(x): 77 return x < NT_OFFSET 78 79def ISNONTERMINAL(x): 80 return x >= NT_OFFSET 81 82def ISEOF(x): 83 return x == ENDMARKER 84