40 static struct tok_state *tok_new(void);
41 static int tok_nextc(struct tok_state *tok);
42 static void tok_backup(struct tok_state *tok, int c);
51 static struct tok_state *
54     struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(  in tok_new()
55                                             sizeof(struct tok_state));  in tok_new()
94 new_string(const char *s, Py_ssize_t len, struct tok_state *tok)  in new_string()
107 error_ret(struct tok_state *tok) /* XXX */  in error_ret()
152 get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok)  in get_coding_spec()
206 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,  in check_coding_spec()
207                   int set_readline(struct tok_state *, const char *))  in check_coding_spec()  argument
265 check_bom(int get_char(struct tok_state *),  in check_bom()  argument
266           void unget_char(int, struct tok_state *),  in check_bom()
267           int set_readline(struct tok_state *, const char *),  in check_bom()  argument
268           struct tok_state *tok)  in check_bom()
342 fp_readl(char *s, int size, struct tok_state *tok)  in fp_readl()
413 fp_setreadl(struct tok_state *tok, const char* enc)  in fp_setreadl()
462 static int fp_getc(struct tok_state *tok) {  in fp_getc()
468 static void fp_ungetc(int c, struct tok_state *tok) {  in fp_ungetc()
504 decoding_fgets(char *s, int size, struct tok_state *tok)  in decoding_fgets()
559 decoding_feof(struct tok_state *tok)  in decoding_feof()
581 buf_getc(struct tok_state *tok) {  in buf_getc()
588 buf_ungetc(int c, struct tok_state *tok) {  in buf_ungetc()
597 buf_setreadl(struct tok_state *tok, const char* enc) {  in buf_setreadl()
618 translate_newlines(const char *s, int exec_input, struct tok_state *tok) {  in translate_newlines()
668 decode_str(const char *input, int single, struct tok_state *tok)  in decode_str()
725 struct tok_state *
728     struct tok_state *tok = tok_new();  in PyTokenizer_FromString()
744 struct tok_state *
747     struct tok_state *tok = tok_new();  in PyTokenizer_FromUTF8()
774 struct tok_state *
778     struct tok_state *tok = tok_new();  in PyTokenizer_FromFile()
808 PyTokenizer_Free(struct tok_state *tok)  in PyTokenizer_Free()
825 tok_nextc(struct tok_state *tok)  in tok_nextc()
1027 tok_backup(struct tok_state *tok, int c)  in tok_backup()
1041 syntaxerror(struct tok_state *tok, const char *format, ...)  in syntaxerror()
1086 indenterror(struct tok_state *tok)  in indenterror()
1097 verify_identifier(struct tok_state *tok)  in verify_identifier()
1150 tok_decimal_tail(struct tok_state *tok)  in tok_decimal_tail()
1174 tok_get(struct tok_state *tok, const char **p_start, const char **p_end)  in tok_get()
1447                 struct tok_state ahead_tok;  in tok_get()
1837 PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end)  in PyTokenizer_Get()
1860     struct tok_state *tok;  in PyTokenizer_FindEncodingFilename()