31 static struct tok_state *tok_new(void);
32 static int tok_nextc(struct tok_state *tok);
33 static void tok_backup(struct tok_state *tok, int c);
97 static struct tok_state *
100     struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(  in tok_new()
101                                             sizeof(struct tok_state));  in tok_new()
147 decoding_fgets(char *s, int size, struct tok_state *tok)  in decoding_fgets()
153 decoding_feof(struct tok_state *tok)  in decoding_feof()
159 decode_str(const char *str, int exec_input, struct tok_state *tok)  in decode_str()
167 error_ret(struct tok_state *tok) /* XXX */  in error_ret()
260 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,  in check_coding_spec()
261                   int set_readline(struct tok_state *, const char *))  in check_coding_spec()  argument
327 check_bom(int get_char(struct tok_state *),  in check_bom()  argument
328           void unget_char(int, struct tok_state *),  in check_bom()
329           int set_readline(struct tok_state *, const char *),  in check_bom()  argument
330           struct tok_state *tok)  in check_bom()
401 fp_readl(char *s, int size, struct tok_state *tok)  in fp_readl()
468 fp_setreadl(struct tok_state *tok, const char* enc)  in fp_setreadl()
493 static int fp_getc(struct tok_state *tok) {  in fp_getc()
499 static void fp_ungetc(int c, struct tok_state *tok) {  in fp_ungetc()
507 decoding_fgets(char *s, int size, struct tok_state *tok)  in decoding_fgets()
565 decoding_feof(struct tok_state *tok)  in decoding_feof()
587 buf_getc(struct tok_state *tok) {  in buf_getc()
594 buf_ungetc(int c, struct tok_state *tok) {  in buf_ungetc()
603 buf_setreadl(struct tok_state *tok, const char* enc) {  in buf_setreadl()
626 translate_newlines(const char *s, int exec_input, struct tok_state *tok) {  in translate_newlines()
670 decode_str(const char *input, int single, struct tok_state *tok)  in decode_str()
733 struct tok_state *
736     struct tok_state *tok = tok_new();  in PyTokenizer_FromString()
753 struct tok_state *
756     struct tok_state *tok = tok_new();  in PyTokenizer_FromFile()
775 PyTokenizer_Free(struct tok_state *tok)  in PyTokenizer_Free()
792 tok_stdin_decode(struct tok_state *tok, char **inp)  in tok_stdin_decode()
857 tok_nextc(register struct tok_state *tok)  in tok_nextc()
1030 tok_backup(register struct tok_state *tok, register int c)  in tok_backup()
1198 indenterror(struct tok_state *tok)  in indenterror()
1216 tok_get(register struct tok_state *tok, char **p_start, char **p_end)  in tok_get()
1681 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end)  in PyTokenizer_Get()
1702 PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset)  in PyTokenizer_RestoreEncoding()
1722 PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset)  in PyTokenizer_RestoreEncoding()