1 #ifndef _PY_TOKENIZER_HELPERS_H_ 2 #define _PY_TOKENIZER_HELPERS_H_ 3 4 #include "Python.h" 5 6 #include "../lexer/state.h" 7 8 #define ADVANCE_LINENO() \ 9 tok->lineno++; \ 10 tok->col_offset = 0; 11 12 int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...); 13 int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); 14 int _PyTokenizer_indenterror(struct tok_state *tok); 15 int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); 16 int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); 17 char *_PyTokenizer_error_ret(struct tok_state *tok); 18 19 char *_PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok); 20 char *_PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf, struct tok_state *tok); 21 PyObject *_PyTokenizer_translate_into_utf8(const char* str, const char* enc); 22 23 int _PyTokenizer_check_bom(int get_char(struct tok_state *), 24 void unget_char(int, struct tok_state *), 25 int set_readline(struct tok_state *, const char *), 26 struct tok_state *tok); 27 int _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, 28 int set_readline(struct tok_state *, const char *)); 29 int _PyTokenizer_ensure_utf8(char *line, struct tok_state *tok); 30 31 #ifdef Py_DEBUG 32 void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size); 33 void _PyTokenizer_tok_dump(int type, char *start, char *end); 34 #endif 35 36 37 #endif 38