Lines Matching refs:tok
32 static int tok_nextc(struct tok_state *tok);
33 static void tok_backup(struct tok_state *tok, int c);
100 struct tok_state *tok = (struct tok_state *)PyMem_MALLOC( in tok_new() local
102 if (tok == NULL) in tok_new()
104 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL; in tok_new()
105 tok->done = E_OK; in tok_new()
106 tok->fp = NULL; in tok_new()
107 tok->input = NULL; in tok_new()
108 tok->tabsize = TABSIZE; in tok_new()
109 tok->indent = 0; in tok_new()
110 tok->indstack[0] = 0; in tok_new()
111 tok->atbol = 1; in tok_new()
112 tok->pendin = 0; in tok_new()
113 tok->prompt = tok->nextprompt = NULL; in tok_new()
114 tok->lineno = 0; in tok_new()
115 tok->level = 0; in tok_new()
116 tok->filename = NULL; in tok_new()
117 tok->altwarning = 0; in tok_new()
118 tok->alterror = 0; in tok_new()
119 tok->alttabsize = 1; in tok_new()
120 tok->altindstack[0] = 0; in tok_new()
121 tok->decoding_state = 0; in tok_new()
122 tok->decoding_erred = 0; in tok_new()
123 tok->read_coding_spec = 0; in tok_new()
124 tok->encoding = NULL; in tok_new()
125 tok->cont_line = 0; in tok_new()
127 tok->decoding_readline = NULL; in tok_new()
128 tok->decoding_buffer = NULL; in tok_new()
130 return tok; in tok_new()
147 decoding_fgets(char *s, int size, struct tok_state *tok) in decoding_fgets() argument
149 return fgets(s, size, tok->fp); in decoding_fgets()
153 decoding_feof(struct tok_state *tok) in decoding_feof() argument
155 return feof(tok->fp); in decoding_feof()
159 decode_str(const char *str, int exec_input, struct tok_state *tok) in decode_str() argument
167 error_ret(struct tok_state *tok) /* XXX */ in error_ret() argument
169 tok->decoding_erred = 1; in error_ret()
170 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */ in error_ret()
171 PyMem_FREE(tok->buf); in error_ret()
172 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL; in error_ret()
173 tok->done = E_DECODE; in error_ret()
260 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, in check_coding_spec() argument
266 if (tok->cont_line) { in check_coding_spec()
268 tok->read_coding_spec = 1; in check_coding_spec()
280 tok->read_coding_spec = 1; in check_coding_spec()
285 tok->read_coding_spec = 1; in check_coding_spec()
286 if (tok->encoding == NULL) { in check_coding_spec()
287 assert(tok->decoding_state == 1); /* raw */ in check_coding_spec()
290 tok->encoding = cs; in check_coding_spec()
293 r = set_readline(tok, cs); in check_coding_spec()
295 tok->encoding = cs; in check_coding_spec()
296 tok->decoding_state = -1; in check_coding_spec()
312 r = (strcmp(tok->encoding, cs) == 0); in check_coding_spec()
330 struct tok_state *tok) in check_bom() argument
333 ch1 = get_char(tok); in check_bom()
334 tok->decoding_state = 1; in check_bom()
338 ch2 = get_char(tok); in check_bom()
340 unget_char(ch2, tok); in check_bom()
341 unget_char(ch1, tok); in check_bom()
344 ch3 = get_char(tok); in check_bom()
346 unget_char(ch3, tok); in check_bom()
347 unget_char(ch2, tok); in check_bom()
348 unget_char(ch1, tok); in check_bom()
355 ch2 = get_char(tok); in check_bom()
357 unget_char(ch2, tok); in check_bom()
358 unget_char(ch1, tok); in check_bom()
361 if (!set_readline(tok, "utf-16-be")) in check_bom()
363 tok->decoding_state = -1; in check_bom()
365 ch2 = get_char(tok); in check_bom()
367 unget_char(ch2, tok); in check_bom()
368 unget_char(ch1, tok); in check_bom()
371 if (!set_readline(tok, "utf-16-le")) in check_bom()
373 tok->decoding_state = -1; in check_bom()
376 unget_char(ch1, tok); in check_bom()
379 if (tok->encoding != NULL) in check_bom()
380 PyMem_FREE(tok->encoding); in check_bom()
381 tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */ in check_bom()
401 fp_readl(char *s, int size, struct tok_state *tok) in fp_readl() argument
409 PyObject* buf = tok->decoding_buffer; in fp_readl()
418 buf = PyObject_CallObject(tok->decoding_readline, NULL); in fp_readl()
420 return error_ret(tok); in fp_readl()
425 return error_ret(tok); in fp_readl()
428 tok->decoding_buffer = NULL; in fp_readl()
436 return error_ret(tok); in fp_readl()
441 tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size); in fp_readl()
442 if (tok->decoding_buffer == NULL) { in fp_readl()
444 return error_ret(tok); in fp_readl()
468 fp_setreadl(struct tok_state *tok, const char* enc) in fp_setreadl() argument
473 stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL); in fp_setreadl()
487 tok->decoding_readline = readline; in fp_setreadl()
493 static int fp_getc(struct tok_state *tok) { in fp_getc() argument
494 return getc(tok->fp); in fp_getc()
499 static void fp_ungetc(int c, struct tok_state *tok) { in fp_ungetc() argument
500 ungetc(c, tok->fp); in fp_ungetc()
507 decoding_fgets(char *s, int size, struct tok_state *tok) in decoding_fgets() argument
512 if (tok->decoding_state < 0) { in decoding_fgets()
515 line = fp_readl(s, size, tok); in decoding_fgets()
517 } else if (tok->decoding_state > 0) { in decoding_fgets()
520 tok->fp, NULL); in decoding_fgets()
526 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok)) in decoding_fgets()
527 return error_ret(tok); in decoding_fgets()
528 assert(tok->decoding_state != 0); in decoding_fgets()
531 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) { in decoding_fgets()
532 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) { in decoding_fgets()
533 return error_ret(tok); in decoding_fgets()
539 if (line && !tok->encoding) { in decoding_fgets()
556 badchar, tok->filename, tok->lineno + 1); in decoding_fgets()
558 return error_ret(tok); in decoding_fgets()
565 decoding_feof(struct tok_state *tok) in decoding_feof() argument
567 if (tok->decoding_state >= 0) { in decoding_feof()
568 return feof(tok->fp); in decoding_feof()
570 PyObject* buf = tok->decoding_buffer; in decoding_feof()
572 buf = PyObject_CallObject(tok->decoding_readline, NULL); in decoding_feof()
574 error_ret(tok); in decoding_feof()
577 tok->decoding_buffer = buf; in decoding_feof()
587 buf_getc(struct tok_state *tok) { in buf_getc() argument
588 return Py_CHARMASK(*tok->str++); in buf_getc()
594 buf_ungetc(int c, struct tok_state *tok) { in buf_ungetc() argument
595 tok->str--; in buf_ungetc()
596 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */ in buf_ungetc()
603 buf_setreadl(struct tok_state *tok, const char* enc) { in buf_setreadl() argument
604 tok->enc = enc; in buf_setreadl()
626 translate_newlines(const char *s, int exec_input, struct tok_state *tok) { in translate_newlines() argument
632 tok->done = E_NOMEM; in translate_newlines()
670 decode_str(const char *input, int single, struct tok_state *tok) in decode_str() argument
677 tok->input = str = translate_newlines(input, single, tok); in decode_str()
680 tok->enc = NULL; in decode_str()
681 tok->str = str; in decode_str()
682 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok)) in decode_str()
683 return error_ret(tok); in decode_str()
684 str = tok->str; /* string after BOM if any */ in decode_str()
687 if (tok->enc != NULL) { in decode_str()
688 utf8 = translate_into_utf8(str, tok->enc); in decode_str()
690 return error_ret(tok); in decode_str()
703 tok->enc = NULL; in decode_str()
707 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) in decode_str()
708 return error_ret(tok); in decode_str()
709 if (tok->enc == NULL && !tok->read_coding_spec && newl[1]) { in decode_str()
711 tok, buf_setreadl)) in decode_str()
712 return error_ret(tok); in decode_str()
716 if (tok->enc != NULL) { in decode_str()
718 utf8 = translate_into_utf8(str, tok->enc); in decode_str()
720 return error_ret(tok); in decode_str()
724 assert(tok->decoding_buffer == NULL); in decode_str()
725 tok->decoding_buffer = utf8; /* CAUTION */ in decode_str()
736 struct tok_state *tok = tok_new(); in PyTokenizer_FromString() local
737 if (tok == NULL) in PyTokenizer_FromString()
739 str = (char *)decode_str(str, exec_input, tok); in PyTokenizer_FromString()
741 PyTokenizer_Free(tok); in PyTokenizer_FromString()
746 tok->buf = tok->cur = tok->end = tok->inp = (char*)str; in PyTokenizer_FromString()
747 return tok; in PyTokenizer_FromString()
756 struct tok_state *tok = tok_new(); in PyTokenizer_FromFile() local
757 if (tok == NULL) in PyTokenizer_FromFile()
759 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) { in PyTokenizer_FromFile()
760 PyTokenizer_Free(tok); in PyTokenizer_FromFile()
763 tok->cur = tok->inp = tok->buf; in PyTokenizer_FromFile()
764 tok->end = tok->buf + BUFSIZ; in PyTokenizer_FromFile()
765 tok->fp = fp; in PyTokenizer_FromFile()
766 tok->prompt = ps1; in PyTokenizer_FromFile()
767 tok->nextprompt = ps2; in PyTokenizer_FromFile()
768 return tok; in PyTokenizer_FromFile()
775 PyTokenizer_Free(struct tok_state *tok) in PyTokenizer_Free() argument
777 if (tok->encoding != NULL) in PyTokenizer_Free()
778 PyMem_FREE(tok->encoding); in PyTokenizer_Free()
780 Py_XDECREF(tok->decoding_readline); in PyTokenizer_Free()
781 Py_XDECREF(tok->decoding_buffer); in PyTokenizer_Free()
783 if (tok->fp != NULL && tok->buf != NULL) in PyTokenizer_Free()
784 PyMem_FREE(tok->buf); in PyTokenizer_Free()
785 if (tok->input) in PyTokenizer_Free()
786 PyMem_FREE((char *)tok->input); in PyTokenizer_Free()
787 PyMem_FREE(tok); in PyTokenizer_Free()
792 tok_stdin_decode(struct tok_state *tok, char **inp) in tok_stdin_decode() argument
828 if (tok->encoding != NULL) in tok_stdin_decode()
829 PyMem_FREE(tok->encoding); in tok_stdin_decode()
830 tok->encoding = new_string(encoding, strlen(encoding)); in tok_stdin_decode()
831 if (tok->encoding == NULL) in tok_stdin_decode()
839 tok->done = E_NOMEM; in tok_stdin_decode()
845 tok->done = E_ERROR; in tok_stdin_decode()
857 tok_nextc(register struct tok_state *tok) in tok_nextc() argument
860 if (tok->cur != tok->inp) { in tok_nextc()
861 return Py_CHARMASK(*tok->cur++); /* Fast path */ in tok_nextc()
863 if (tok->done != E_OK) in tok_nextc()
865 if (tok->fp == NULL) { in tok_nextc()
866 char *end = strchr(tok->inp, '\n'); in tok_nextc()
870 end = strchr(tok->inp, '\0'); in tok_nextc()
871 if (end == tok->inp) { in tok_nextc()
872 tok->done = E_EOF; in tok_nextc()
876 if (tok->start == NULL) in tok_nextc()
877 tok->buf = tok->cur; in tok_nextc()
878 tok->line_start = tok->cur; in tok_nextc()
879 tok->lineno++; in tok_nextc()
880 tok->inp = end; in tok_nextc()
881 return Py_CHARMASK(*tok->cur++); in tok_nextc()
883 if (tok->prompt != NULL) { in tok_nextc()
884 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt); in tok_nextc()
885 if (tok->nextprompt != NULL) in tok_nextc()
886 tok->prompt = tok->nextprompt; in tok_nextc()
888 tok->done = E_INTR; in tok_nextc()
891 tok->done = E_EOF; in tok_nextc()
894 else if (tok_stdin_decode(tok, &newtok) != 0) in tok_nextc()
897 else if (tok->start != NULL) { in tok_nextc()
898 size_t start = tok->start - tok->buf; in tok_nextc()
899 size_t oldlen = tok->cur - tok->buf; in tok_nextc()
901 char *buf = tok->buf; in tok_nextc()
903 tok->lineno++; in tok_nextc()
905 PyMem_FREE(tok->buf); in tok_nextc()
906 tok->buf = NULL; in tok_nextc()
908 tok->done = E_NOMEM; in tok_nextc()
911 tok->buf = buf; in tok_nextc()
912 tok->cur = tok->buf + oldlen; in tok_nextc()
913 tok->line_start = tok->cur; in tok_nextc()
914 strcpy(tok->buf + oldlen, newtok); in tok_nextc()
916 tok->inp = tok->buf + newlen; in tok_nextc()
917 tok->end = tok->inp + 1; in tok_nextc()
918 tok->start = tok->buf + start; in tok_nextc()
921 tok->lineno++; in tok_nextc()
922 if (tok->buf != NULL) in tok_nextc()
923 PyMem_FREE(tok->buf); in tok_nextc()
924 tok->buf = newtok; in tok_nextc()
925 tok->cur = tok->buf; in tok_nextc()
926 tok->line_start = tok->buf; in tok_nextc()
927 tok->inp = strchr(tok->buf, '\0'); in tok_nextc()
928 tok->end = tok->inp + 1; in tok_nextc()
935 if (tok->start == NULL) { in tok_nextc()
936 if (tok->buf == NULL) { in tok_nextc()
937 tok->buf = (char *) in tok_nextc()
939 if (tok->buf == NULL) { in tok_nextc()
940 tok->done = E_NOMEM; in tok_nextc()
943 tok->end = tok->buf + BUFSIZ; in tok_nextc()
945 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf), in tok_nextc()
946 tok) == NULL) { in tok_nextc()
947 if (!tok->decoding_erred) in tok_nextc()
948 tok->done = E_EOF; in tok_nextc()
952 tok->done = E_OK; in tok_nextc()
953 tok->inp = strchr(tok->buf, '\0'); in tok_nextc()
954 done = tok->inp == tok->buf || tok->inp[-1] == '\n'; in tok_nextc()
958 cur = tok->cur - tok->buf; in tok_nextc()
959 if (decoding_feof(tok)) { in tok_nextc()
960 tok->done = E_EOF; in tok_nextc()
964 tok->done = E_OK; in tok_nextc()
966 tok->lineno++; in tok_nextc()
969 Py_ssize_t curstart = tok->start == NULL ? -1 : in tok_nextc()
970 tok->start - tok->buf; in tok_nextc()
971 Py_ssize_t curvalid = tok->inp - tok->buf; in tok_nextc()
973 char *newbuf = tok->buf; in tok_nextc()
977 tok->done = E_NOMEM; in tok_nextc()
978 tok->cur = tok->inp; in tok_nextc()
981 tok->buf = newbuf; in tok_nextc()
982 tok->cur = tok->buf + cur; in tok_nextc()
983 tok->line_start = tok->cur; in tok_nextc()
984 tok->inp = tok->buf + curvalid; in tok_nextc()
985 tok->end = tok->buf + newsize; in tok_nextc()
986 tok->start = curstart < 0 ? NULL : in tok_nextc()
987 tok->buf + curstart; in tok_nextc()
988 if (decoding_fgets(tok->inp, in tok_nextc()
989 (int)(tok->end - tok->inp), in tok_nextc()
990 tok) == NULL) { in tok_nextc()
994 if (tok->decoding_erred) in tok_nextc()
998 strcpy(tok->inp, "\n"); in tok_nextc()
1000 tok->inp = strchr(tok->inp, '\0'); in tok_nextc()
1001 done = tok->inp[-1] == '\n'; in tok_nextc()
1003 if (tok->buf != NULL) { in tok_nextc()
1004 tok->cur = tok->buf + cur; in tok_nextc()
1005 tok->line_start = tok->cur; in tok_nextc()
1008 pt = tok->inp - 2; in tok_nextc()
1009 if (pt >= tok->buf && *pt == '\r') { in tok_nextc()
1012 tok->inp = pt; in tok_nextc()
1016 if (tok->done != E_OK) { in tok_nextc()
1017 if (tok->prompt != NULL) in tok_nextc()
1019 tok->cur = tok->inp; in tok_nextc()
1030 tok_backup(register struct tok_state *tok, register int c) in tok_backup() argument
1033 if (--tok->cur < tok->buf) in tok_backup()
1035 if (*tok->cur != c) in tok_backup()
1036 *tok->cur = c; in tok_backup()
1198 indenterror(struct tok_state *tok) in indenterror() argument
1200 if (tok->alterror) { in indenterror()
1201 tok->done = E_TABSPACE; in indenterror()
1202 tok->cur = tok->inp; in indenterror()
1205 if (tok->altwarning) { in indenterror()
1207 "in indentation\n", tok->filename); in indenterror()
1208 tok->altwarning = 0; in indenterror()
1216 tok_get(register struct tok_state *tok, char **p_start, char **p_end) in tok_get() argument
1223 tok->start = NULL; in tok_get()
1227 if (tok->atbol) { in tok_get()
1230 tok->atbol = 0; in tok_get()
1232 c = tok_nextc(tok); in tok_get()
1236 col = (col/tok->tabsize + 1) * tok->tabsize; in tok_get()
1237 altcol = (altcol/tok->alttabsize + 1) in tok_get()
1238 * tok->alttabsize; in tok_get()
1245 tok_backup(tok, c); in tok_get()
1252 if (col == 0 && c == '\n' && tok->prompt != NULL) in tok_get()
1259 if (!blankline && tok->level == 0) { in tok_get()
1260 if (col == tok->indstack[tok->indent]) { in tok_get()
1262 if (altcol != tok->altindstack[tok->indent]) { in tok_get()
1263 if (indenterror(tok)) in tok_get()
1267 else if (col > tok->indstack[tok->indent]) { in tok_get()
1269 if (tok->indent+1 >= MAXINDENT) { in tok_get()
1270 tok->done = E_TOODEEP; in tok_get()
1271 tok->cur = tok->inp; in tok_get()
1274 if (altcol <= tok->altindstack[tok->indent]) { in tok_get()
1275 if (indenterror(tok)) in tok_get()
1278 tok->pendin++; in tok_get()
1279 tok->indstack[++tok->indent] = col; in tok_get()
1280 tok->altindstack[tok->indent] = altcol; in tok_get()
1284 while (tok->indent > 0 && in tok_get()
1285 col < tok->indstack[tok->indent]) { in tok_get()
1286 tok->pendin--; in tok_get()
1287 tok->indent--; in tok_get()
1289 if (col != tok->indstack[tok->indent]) { in tok_get()
1290 tok->done = E_DEDENT; in tok_get()
1291 tok->cur = tok->inp; in tok_get()
1294 if (altcol != tok->altindstack[tok->indent]) { in tok_get()
1295 if (indenterror(tok)) in tok_get()
1302 tok->start = tok->cur; in tok_get()
1305 if (tok->pendin != 0) { in tok_get()
1306 if (tok->pendin < 0) { in tok_get()
1307 tok->pendin++; in tok_get()
1311 tok->pendin--; in tok_get()
1317 tok->start = NULL; in tok_get()
1320 c = tok_nextc(tok); in tok_get()
1324 tok->start = tok->cur - 1; in tok_get()
1339 *tp++ = c = tok_nextc(tok); in tok_get()
1350 tok->tabsize = newsize; in tok_get()
1359 c = tok_nextc(tok); in tok_get()
1364 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN; in tok_get()
1373 c = tok_nextc(tok); in tok_get()
1375 c = tok_nextc(tok); in tok_get()
1381 c = tok_nextc(tok); in tok_get()
1387 c = tok_nextc(tok); in tok_get()
1389 c = tok_nextc(tok); in tok_get()
1395 c = tok_nextc(tok); in tok_get()
1397 tok_backup(tok, c); in tok_get()
1398 *p_start = tok->start; in tok_get()
1399 *p_end = tok->cur; in tok_get()
1405 tok->atbol = 1; in tok_get()
1406 if (blankline || tok->level > 0) in tok_get()
1408 *p_start = tok->start; in tok_get()
1409 *p_end = tok->cur - 1; /* Leave '\n' out of the string */ in tok_get()
1410 tok->cont_line = 0; in tok_get()
1416 c = tok_nextc(tok); in tok_get()
1421 tok_backup(tok, c); in tok_get()
1422 *p_start = tok->start; in tok_get()
1423 *p_end = tok->cur; in tok_get()
1432 c = tok_nextc(tok); in tok_get()
1442 c = tok_nextc(tok); in tok_get()
1444 tok->done = E_TOKEN; in tok_get()
1445 tok_backup(tok, c); in tok_get()
1449 c = tok_nextc(tok); in tok_get()
1454 c = tok_nextc(tok); in tok_get()
1456 tok->done = E_TOKEN; in tok_get()
1457 tok_backup(tok, c); in tok_get()
1461 c = tok_nextc(tok); in tok_get()
1466 c = tok_nextc(tok); in tok_get()
1468 tok->done = E_TOKEN; in tok_get()
1469 tok_backup(tok, c); in tok_get()
1473 c = tok_nextc(tok); in tok_get()
1481 c = tok_nextc(tok); in tok_get()
1486 c = tok_nextc(tok); in tok_get()
1498 tok->done = E_TOKEN; in tok_get()
1499 tok_backup(tok, c); in tok_get()
1504 c = tok_nextc(tok); in tok_get()
1509 c = tok_nextc(tok); in tok_get()
1512 c = tok_nextc(tok); in tok_get()
1519 c = tok_nextc(tok); in tok_get()
1527 c = tok_nextc(tok); in tok_get()
1529 c = tok_nextc(tok); in tok_get()
1531 tok->done = E_TOKEN; in tok_get()
1532 tok_backup(tok, c); in tok_get()
1536 tok_backup(tok, c); in tok_get()
1537 tok_backup(tok, e); in tok_get()
1538 *p_start = tok->start; in tok_get()
1539 *p_end = tok->cur; in tok_get()
1543 c = tok_nextc(tok); in tok_get()
1550 c = tok_nextc(tok); in tok_get()
1554 tok_backup(tok, c); in tok_get()
1555 *p_start = tok->start; in tok_get()
1556 *p_end = tok->cur; in tok_get()
1563 Py_ssize_t quote2 = tok->cur - tok->start + 1; in tok_get()
1568 c = tok_nextc(tok); in tok_get()
1571 tok->done = E_EOLS; in tok_get()
1572 tok_backup(tok, c); in tok_get()
1576 tok->cont_line = 1; /* multiline string. */ in tok_get()
1580 tok->done = E_EOFS; in tok_get()
1582 tok->done = E_EOLS; in tok_get()
1583 tok->cur = tok->inp; in tok_get()
1588 if (tok->cur - tok->start == quote2) { in tok_get()
1589 c = tok_nextc(tok); in tok_get()
1595 tok_backup(tok, c); in tok_get()
1602 c = tok_nextc(tok); in tok_get()
1604 tok->done = E_EOLS; in tok_get()
1605 tok->cur = tok->inp; in tok_get()
1612 *p_start = tok->start; in tok_get()
1613 *p_end = tok->cur; in tok_get()
1619 c = tok_nextc(tok); in tok_get()
1621 tok->done = E_LINECONT; in tok_get()
1622 tok->cur = tok->inp; in tok_get()
1625 tok->cont_line = 1; in tok_get()
1631 int c2 = tok_nextc(tok); in tok_get()
1637 tok->filename, tok->lineno, in tok_get()
1644 int c3 = tok_nextc(tok); in tok_get()
1649 tok_backup(tok, c3); in tok_get()
1651 *p_start = tok->start; in tok_get()
1652 *p_end = tok->cur; in tok_get()
1655 tok_backup(tok, c2); in tok_get()
1663 tok->level++; in tok_get()
1668 tok->level--; in tok_get()
1673 *p_start = tok->start; in tok_get()
1674 *p_end = tok->cur; in tok_get()
1679 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end) in PyTokenizer_Get() argument
1681 int result = tok_get(tok, p_start, p_end); in PyTokenizer_Get()
1682 if (tok->decoding_erred) { in PyTokenizer_Get()
1684 tok->done = E_DECODE; in PyTokenizer_Get()
1695 PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset) in PyTokenizer_RestoreEncoding() argument
1715 PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset) in PyTokenizer_RestoreEncoding() argument
1718 if (tok->encoding) { in PyTokenizer_RestoreEncoding()
1720 PyObject *lineobj = dec_utf8(tok->encoding, tok->buf, len); in PyTokenizer_RestoreEncoding()
1734 PyObject *offsetobj = dec_utf8(tok->encoding, in PyTokenizer_RestoreEncoding()
1735 tok->buf, *offset-1); in PyTokenizer_RestoreEncoding()