1 // Auto-generated by Tools/build/generate_token.py 2 3 /* Token types */ 4 #ifndef Py_INTERNAL_TOKEN_H 5 #define Py_INTERNAL_TOKEN_H 6 #ifdef __cplusplus 7 extern "C" { 8 #endif 9 10 #ifndef Py_BUILD_CORE 11 # error "this header requires Py_BUILD_CORE define" 12 #endif 13 14 #undef TILDE /* Prevent clash of our definition with system macro. Ex AIX, ioctl.h */ 15 16 #define ENDMARKER 0 17 #define NAME 1 18 #define NUMBER 2 19 #define STRING 3 20 #define NEWLINE 4 21 #define INDENT 5 22 #define DEDENT 6 23 #define LPAR 7 24 #define RPAR 8 25 #define LSQB 9 26 #define RSQB 10 27 #define COLON 11 28 #define COMMA 12 29 #define SEMI 13 30 #define PLUS 14 31 #define MINUS 15 32 #define STAR 16 33 #define SLASH 17 34 #define VBAR 18 35 #define AMPER 19 36 #define LESS 20 37 #define GREATER 21 38 #define EQUAL 22 39 #define DOT 23 40 #define PERCENT 24 41 #define LBRACE 25 42 #define RBRACE 26 43 #define EQEQUAL 27 44 #define NOTEQUAL 28 45 #define LESSEQUAL 29 46 #define GREATEREQUAL 30 47 #define TILDE 31 48 #define CIRCUMFLEX 32 49 #define LEFTSHIFT 33 50 #define RIGHTSHIFT 34 51 #define DOUBLESTAR 35 52 #define PLUSEQUAL 36 53 #define MINEQUAL 37 54 #define STAREQUAL 38 55 #define SLASHEQUAL 39 56 #define PERCENTEQUAL 40 57 #define AMPEREQUAL 41 58 #define VBAREQUAL 42 59 #define CIRCUMFLEXEQUAL 43 60 #define LEFTSHIFTEQUAL 44 61 #define RIGHTSHIFTEQUAL 45 62 #define DOUBLESTAREQUAL 46 63 #define DOUBLESLASH 47 64 #define DOUBLESLASHEQUAL 48 65 #define AT 49 66 #define ATEQUAL 50 67 #define RARROW 51 68 #define ELLIPSIS 52 69 #define COLONEQUAL 53 70 #define EXCLAMATION 54 71 #define OP 55 72 #define TYPE_IGNORE 56 73 #define TYPE_COMMENT 57 74 #define SOFT_KEYWORD 58 75 #define FSTRING_START 59 76 #define FSTRING_MIDDLE 60 77 #define FSTRING_END 61 78 #define COMMENT 62 79 #define NL 63 80 #define ERRORTOKEN 64 81 #define N_TOKENS 66 82 #define NT_OFFSET 256 83 84 /* Special definitions for cooperation with parser */ 85 86 #define ISTERMINAL(x) ((x) < NT_OFFSET) 87 #define ISNONTERMINAL(x) ((x) >= NT_OFFSET) 88 #define ISEOF(x) ((x) == ENDMARKER) 89 #define ISWHITESPACE(x) ((x) == ENDMARKER || \ 90 (x) == NEWLINE || \ 91 (x) == INDENT || \ 92 (x) == DEDENT) 93 #define ISSTRINGLIT(x) ((x) == STRING || \ 94 (x) == FSTRING_MIDDLE) 95 96 97 // Export these 4 symbols for 'test_peg_generator' 98 PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ 99 PyAPI_FUNC(int) _PyToken_OneChar(int); 100 PyAPI_FUNC(int) _PyToken_TwoChars(int, int); 101 PyAPI_FUNC(int) _PyToken_ThreeChars(int, int, int); 102 103 #ifdef __cplusplus 104 } 105 #endif 106 #endif // !Py_INTERNAL_TOKEN_H 107