50     struct tok_state *tok;  in PyParser_ParseStringFlagsFilenameEx()  local
54     if ((tok = PyTokenizer_FromString(s, start == file_input)) == NULL) {  in PyParser_ParseStringFlagsFilenameEx()
59     tok->filename = filename ? filename : "<string>";  in PyParser_ParseStringFlagsFilenameEx()
61         tok->altwarning = (tok->filename != NULL);  in PyParser_ParseStringFlagsFilenameEx()
63             tok->alterror++;  in PyParser_ParseStringFlagsFilenameEx()
66     return parsetok(tok, g, start, err_ret, flags);  in PyParser_ParseStringFlagsFilenameEx()
91     struct tok_state *tok;  in PyParser_ParseFileFlagsEx()  local
95     if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {  in PyParser_ParseFileFlagsEx()
99     tok->filename = filename;  in PyParser_ParseFileFlagsEx()
101         tok->altwarning = (filename != NULL);  in PyParser_ParseFileFlagsEx()
103             tok->alterror++;  in PyParser_ParseFileFlagsEx()
106     return parsetok(tok, g, start, err_ret, flags);  in PyParser_ParseFileFlagsEx()
129 parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,  in parsetok()  argument
139         PyTokenizer_Free(tok);  in parsetok()
159         type = PyTokenizer_Get(tok, &a, &b);  in parsetok()
161             err_ret->error = tok->done;  in parsetok()
170             if (tok->indent &&  in parsetok()
173                 tok->pendin = -tok->indent;  in parsetok()
174                 tok->indent = 0;  in parsetok()
192         if (a != NULL && a >= tok->line_start) {  in parsetok()
193             col_offset = a - tok->line_start;  in parsetok()
200              PyParser_AddToken(ps, (int)type, str, tok->lineno, col_offset,  in parsetok()
223         if (tok->lineno <= 1 && tok->done == E_EOF)  in parsetok()
225         err_ret->lineno = tok->lineno;  in parsetok()
226         if (tok->buf != NULL) {  in parsetok()
229             assert(tok->cur - tok->buf < INT_MAX);  in parsetok()
230             err_ret->offset = (int)(tok->cur - tok->buf);  in parsetok()
231             len = tok->inp - tok->buf;  in parsetok()
233             text = PyTokenizer_RestoreEncoding(tok, len, &err_ret->offset);  in parsetok()
240                         strncpy(text, tok->buf, len);  in parsetok()
246     } else if (tok->encoding != NULL) {  in parsetok()
252             r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);  in parsetok()
260         strcpy(r->n_str, tok->encoding);  in parsetok()
261         PyMem_FREE(tok->encoding);  in parsetok()
262         tok->encoding = NULL;  in parsetok()
269     PyTokenizer_Free(tok);  in parsetok()