Lines Matching refs:tok
44 static int tok_nextc(struct tok_state *tok);
45 static void tok_backup(struct tok_state *tok, int c);
57 struct tok_state *tok = (struct tok_state *)PyMem_MALLOC( in tok_new() local
59 if (tok == NULL) in tok_new()
61 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL; in tok_new()
62 tok->done = E_OK; in tok_new()
63 tok->fp = NULL; in tok_new()
64 tok->input = NULL; in tok_new()
65 tok->tabsize = TABSIZE; in tok_new()
66 tok->indent = 0; in tok_new()
67 tok->indstack[0] = 0; in tok_new()
69 tok->atbol = 1; in tok_new()
70 tok->pendin = 0; in tok_new()
71 tok->prompt = tok->nextprompt = NULL; in tok_new()
72 tok->lineno = 0; in tok_new()
73 tok->level = 0; in tok_new()
74 tok->altindstack[0] = 0; in tok_new()
75 tok->decoding_state = STATE_INIT; in tok_new()
76 tok->decoding_erred = 0; in tok_new()
77 tok->read_coding_spec = 0; in tok_new()
78 tok->enc = NULL; in tok_new()
79 tok->encoding = NULL; in tok_new()
80 tok->cont_line = 0; in tok_new()
81 tok->filename = NULL; in tok_new()
82 tok->decoding_readline = NULL; in tok_new()
83 tok->decoding_buffer = NULL; in tok_new()
84 tok->type_comments = 0; in tok_new()
86 tok->async_hacks = 0; in tok_new()
87 tok->async_def = 0; in tok_new()
88 tok->async_def_indent = 0; in tok_new()
89 tok->async_def_nl = 0; in tok_new()
91 return tok; in tok_new()
95 new_string(const char *s, Py_ssize_t len, struct tok_state *tok) in new_string() argument
99 tok->done = E_NOMEM; in new_string()
108 error_ret(struct tok_state *tok) /* XXX */ in error_ret() argument
110 tok->decoding_erred = 1; in error_ret()
111 if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */ in error_ret()
112 PyMem_FREE(tok->buf); in error_ret()
113 tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL; in error_ret()
114 tok->done = E_DECODE; in error_ret()
151 get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok) in get_coding_spec() argument
180 char* r = new_string(begin, t - begin, tok); in get_coding_spec()
187 r = new_string(q, strlen(q), tok); in get_coding_spec()
205 check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, in check_coding_spec() argument
211 if (tok->cont_line) { in check_coding_spec()
213 tok->read_coding_spec = 1; in check_coding_spec()
216 if (!get_coding_spec(line, &cs, size, tok)) in check_coding_spec()
226 tok->read_coding_spec = 1; in check_coding_spec()
232 tok->read_coding_spec = 1; in check_coding_spec()
233 if (tok->encoding == NULL) { in check_coding_spec()
234 assert(tok->decoding_state == STATE_RAW); in check_coding_spec()
236 tok->encoding = cs; in check_coding_spec()
238 r = set_readline(tok, cs); in check_coding_spec()
240 tok->encoding = cs; in check_coding_spec()
241 tok->decoding_state = STATE_NORMAL; in check_coding_spec()
250 r = (strcmp(tok->encoding, cs) == 0); in check_coding_spec()
267 struct tok_state *tok) in check_bom() argument
270 ch1 = get_char(tok); in check_bom()
271 tok->decoding_state = STATE_RAW; in check_bom()
275 ch2 = get_char(tok); in check_bom()
277 unget_char(ch2, tok); in check_bom()
278 unget_char(ch1, tok); in check_bom()
281 ch3 = get_char(tok); in check_bom()
283 unget_char(ch3, tok); in check_bom()
284 unget_char(ch2, tok); in check_bom()
285 unget_char(ch1, tok); in check_bom()
292 ch2 = get_char(tok); in check_bom()
294 unget_char(ch2, tok); in check_bom()
295 unget_char(ch1, tok); in check_bom()
298 if (!set_readline(tok, "utf-16-be")) in check_bom()
300 tok->decoding_state = STATE_NORMAL; in check_bom()
302 ch2 = get_char(tok); in check_bom()
304 unget_char(ch2, tok); in check_bom()
305 unget_char(ch1, tok); in check_bom()
308 if (!set_readline(tok, "utf-16-le")) in check_bom()
310 tok->decoding_state = STATE_NORMAL; in check_bom()
313 unget_char(ch1, tok); in check_bom()
316 if (tok->encoding != NULL) in check_bom()
317 PyMem_FREE(tok->encoding); in check_bom()
318 tok->encoding = new_string("utf-8", 5, tok); in check_bom()
319 if (!tok->encoding) in check_bom()
341 fp_readl(char *s, int size, struct tok_state *tok) in fp_readl() argument
351 if (tok->decoding_buffer) { in fp_readl()
352 bufobj = tok->decoding_buffer; in fp_readl()
357 bufobj = _PyObject_CallNoArg(tok->decoding_readline); in fp_readl()
377 Py_XDECREF(tok->decoding_buffer); in fp_readl()
380 tok->decoding_buffer = PyByteArray_FromStringAndSize(buf+size, in fp_readl()
382 if (tok->decoding_buffer == NULL) in fp_readl()
387 tok->decoding_buffer = NULL; in fp_readl()
398 return error_ret(tok); in fp_readl()
412 fp_setreadl(struct tok_state *tok, const char* enc) in fp_setreadl() argument
420 fd = fileno(tok->fp); in fp_setreadl()
426 pos = ftell(tok->fp); in fp_setreadl()
447 Py_XSETREF(tok->decoding_readline, readline); in fp_setreadl()
461 static int fp_getc(struct tok_state *tok) { in fp_getc() argument
462 return getc(tok->fp); in fp_getc()
467 static void fp_ungetc(int c, struct tok_state *tok) { in fp_ungetc() argument
468 ungetc(c, tok->fp); in fp_ungetc()
503 decoding_fgets(char *s, int size, struct tok_state *tok) in decoding_fgets() argument
508 if (tok->decoding_state == STATE_NORMAL) { in decoding_fgets()
511 line = fp_readl(s, size, tok); in decoding_fgets()
513 } else if (tok->decoding_state == STATE_RAW) { in decoding_fgets()
516 tok->fp, NULL); in decoding_fgets()
522 if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok)) in decoding_fgets()
523 return error_ret(tok); in decoding_fgets()
524 assert(tok->decoding_state != STATE_INIT); in decoding_fgets()
527 if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) { in decoding_fgets()
528 if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) { in decoding_fgets()
529 return error_ret(tok); in decoding_fgets()
534 if (line && !tok->encoding) { in decoding_fgets()
551 badchar, tok->filename, tok->lineno + 1); in decoding_fgets()
552 return error_ret(tok); in decoding_fgets()
558 decoding_feof(struct tok_state *tok) in decoding_feof() argument
560 if (tok->decoding_state != STATE_NORMAL) { in decoding_feof()
561 return feof(tok->fp); in decoding_feof()
563 PyObject* buf = tok->decoding_buffer; in decoding_feof()
565 buf = _PyObject_CallNoArg(tok->decoding_readline); in decoding_feof()
567 error_ret(tok); in decoding_feof()
570 tok->decoding_buffer = buf; in decoding_feof()
580 buf_getc(struct tok_state *tok) { in buf_getc() argument
581 return Py_CHARMASK(*tok->str++); in buf_getc()
587 buf_ungetc(int c, struct tok_state *tok) { in buf_ungetc() argument
588 tok->str--; in buf_ungetc()
589 assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */ in buf_ungetc()
596 buf_setreadl(struct tok_state *tok, const char* enc) { in buf_setreadl() argument
597 tok->enc = enc; in buf_setreadl()
617 translate_newlines(const char *s, int exec_input, struct tok_state *tok) { in translate_newlines() argument
624 tok->done = E_NOMEM; in translate_newlines()
667 decode_str(const char *input, int single, struct tok_state *tok) in decode_str() argument
674 tok->input = str = translate_newlines(input, single, tok); in decode_str()
677 tok->enc = NULL; in decode_str()
678 tok->str = str; in decode_str()
679 if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok)) in decode_str()
680 return error_ret(tok); in decode_str()
681 str = tok->str; /* string after BOM if any */ in decode_str()
683 if (tok->enc != NULL) { in decode_str()
684 utf8 = translate_into_utf8(str, tok->enc); in decode_str()
686 return error_ret(tok); in decode_str()
698 tok->enc = NULL; in decode_str()
702 if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) in decode_str()
703 return error_ret(tok); in decode_str()
704 if (tok->enc == NULL && !tok->read_coding_spec && newl[1]) { in decode_str()
706 tok, buf_setreadl)) in decode_str()
707 return error_ret(tok); in decode_str()
710 if (tok->enc != NULL) { in decode_str()
712 utf8 = translate_into_utf8(str, tok->enc); in decode_str()
714 return error_ret(tok); in decode_str()
717 assert(tok->decoding_buffer == NULL); in decode_str()
718 tok->decoding_buffer = utf8; /* CAUTION */ in decode_str()
727 struct tok_state *tok = tok_new(); in PyTokenizer_FromString() local
728 if (tok == NULL) in PyTokenizer_FromString()
730 str = decode_str(str, exec_input, tok); in PyTokenizer_FromString()
732 PyTokenizer_Free(tok); in PyTokenizer_FromString()
737 tok->buf = tok->cur = tok->end = tok->inp = (char*)str; in PyTokenizer_FromString()
738 return tok; in PyTokenizer_FromString()
744 struct tok_state *tok = tok_new(); in PyTokenizer_FromUTF8() local
745 if (tok == NULL) in PyTokenizer_FromUTF8()
747 tok->input = str = translate_newlines(str, exec_input, tok); in PyTokenizer_FromUTF8()
749 PyTokenizer_Free(tok); in PyTokenizer_FromUTF8()
752 tok->decoding_state = STATE_RAW; in PyTokenizer_FromUTF8()
753 tok->read_coding_spec = 1; in PyTokenizer_FromUTF8()
754 tok->enc = NULL; in PyTokenizer_FromUTF8()
755 tok->str = str; in PyTokenizer_FromUTF8()
756 tok->encoding = (char *)PyMem_MALLOC(6); in PyTokenizer_FromUTF8()
757 if (!tok->encoding) { in PyTokenizer_FromUTF8()
758 PyTokenizer_Free(tok); in PyTokenizer_FromUTF8()
761 strcpy(tok->encoding, "utf-8"); in PyTokenizer_FromUTF8()
764 tok->buf = tok->cur = tok->end = tok->inp = (char*)str; in PyTokenizer_FromUTF8()
765 return tok; in PyTokenizer_FromUTF8()
774 struct tok_state *tok = tok_new(); in PyTokenizer_FromFile() local
775 if (tok == NULL) in PyTokenizer_FromFile()
777 if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) { in PyTokenizer_FromFile()
778 PyTokenizer_Free(tok); in PyTokenizer_FromFile()
781 tok->cur = tok->inp = tok->buf; in PyTokenizer_FromFile()
782 tok->end = tok->buf + BUFSIZ; in PyTokenizer_FromFile()
783 tok->fp = fp; in PyTokenizer_FromFile()
784 tok->prompt = ps1; in PyTokenizer_FromFile()
785 tok->nextprompt = ps2; in PyTokenizer_FromFile()
789 tok->encoding = PyMem_MALLOC(strlen(enc)+1); in PyTokenizer_FromFile()
790 if (!tok->encoding) { in PyTokenizer_FromFile()
791 PyTokenizer_Free(tok); in PyTokenizer_FromFile()
794 strcpy(tok->encoding, enc); in PyTokenizer_FromFile()
795 tok->decoding_state = STATE_NORMAL; in PyTokenizer_FromFile()
797 return tok; in PyTokenizer_FromFile()
804 PyTokenizer_Free(struct tok_state *tok) in PyTokenizer_Free() argument
806 if (tok->encoding != NULL) in PyTokenizer_Free()
807 PyMem_FREE(tok->encoding); in PyTokenizer_Free()
808 Py_XDECREF(tok->decoding_readline); in PyTokenizer_Free()
809 Py_XDECREF(tok->decoding_buffer); in PyTokenizer_Free()
810 Py_XDECREF(tok->filename); in PyTokenizer_Free()
811 if (tok->fp != NULL && tok->buf != NULL) in PyTokenizer_Free()
812 PyMem_FREE(tok->buf); in PyTokenizer_Free()
813 if (tok->input) in PyTokenizer_Free()
814 PyMem_FREE((char *)tok->input); in PyTokenizer_Free()
815 PyMem_FREE(tok); in PyTokenizer_Free()
821 tok_nextc(struct tok_state *tok) in tok_nextc() argument
824 if (tok->cur != tok->inp) { in tok_nextc()
825 return Py_CHARMASK(*tok->cur++); /* Fast path */ in tok_nextc()
827 if (tok->done != E_OK) in tok_nextc()
829 if (tok->fp == NULL) { in tok_nextc()
830 char *end = strchr(tok->inp, '\n'); in tok_nextc()
834 end = strchr(tok->inp, '\0'); in tok_nextc()
835 if (end == tok->inp) { in tok_nextc()
836 tok->done = E_EOF; in tok_nextc()
840 if (tok->start == NULL) in tok_nextc()
841 tok->buf = tok->cur; in tok_nextc()
842 tok->line_start = tok->cur; in tok_nextc()
843 tok->lineno++; in tok_nextc()
844 tok->inp = end; in tok_nextc()
845 return Py_CHARMASK(*tok->cur++); in tok_nextc()
847 if (tok->prompt != NULL) { in tok_nextc()
848 char *newtok = PyOS_Readline(stdin, stdout, tok->prompt); in tok_nextc()
850 char *translated = translate_newlines(newtok, 0, tok); in tok_nextc()
856 if (tok->encoding && newtok && *newtok) { in tok_nextc()
860 PyObject *u = translate_into_utf8(newtok, tok->encoding); in tok_nextc()
863 tok->done = E_DECODE; in tok_nextc()
871 tok->done = E_NOMEM; in tok_nextc()
877 if (tok->nextprompt != NULL) in tok_nextc()
878 tok->prompt = tok->nextprompt; in tok_nextc()
880 tok->done = E_INTR; in tok_nextc()
883 tok->done = E_EOF; in tok_nextc()
885 else if (tok->start != NULL) { in tok_nextc()
886 size_t start = tok->start - tok->buf; in tok_nextc()
887 size_t oldlen = tok->cur - tok->buf; in tok_nextc()
889 char *buf = tok->buf; in tok_nextc()
891 tok->lineno++; in tok_nextc()
893 PyMem_FREE(tok->buf); in tok_nextc()
894 tok->buf = NULL; in tok_nextc()
896 tok->done = E_NOMEM; in tok_nextc()
899 tok->buf = buf; in tok_nextc()
900 tok->cur = tok->buf + oldlen; in tok_nextc()
901 tok->line_start = tok->cur; in tok_nextc()
902 strcpy(tok->buf + oldlen, newtok); in tok_nextc()
904 tok->inp = tok->buf + newlen; in tok_nextc()
905 tok->end = tok->inp + 1; in tok_nextc()
906 tok->start = tok->buf + start; in tok_nextc()
909 tok->lineno++; in tok_nextc()
910 if (tok->buf != NULL) in tok_nextc()
911 PyMem_FREE(tok->buf); in tok_nextc()
912 tok->buf = newtok; in tok_nextc()
913 tok->cur = tok->buf; in tok_nextc()
914 tok->line_start = tok->buf; in tok_nextc()
915 tok->inp = strchr(tok->buf, '\0'); in tok_nextc()
916 tok->end = tok->inp + 1; in tok_nextc()
923 if (tok->start == NULL) { in tok_nextc()
924 if (tok->buf == NULL) { in tok_nextc()
925 tok->buf = (char *) in tok_nextc()
927 if (tok->buf == NULL) { in tok_nextc()
928 tok->done = E_NOMEM; in tok_nextc()
931 tok->end = tok->buf + BUFSIZ; in tok_nextc()
933 if (decoding_fgets(tok->buf, (int)(tok->end - tok->buf), in tok_nextc()
934 tok) == NULL) { in tok_nextc()
935 if (!tok->decoding_erred) in tok_nextc()
936 tok->done = E_EOF; in tok_nextc()
940 tok->done = E_OK; in tok_nextc()
941 tok->inp = strchr(tok->buf, '\0'); in tok_nextc()
942 done = tok->inp == tok->buf || tok->inp[-1] == '\n'; in tok_nextc()
946 cur = tok->cur - tok->buf; in tok_nextc()
947 if (decoding_feof(tok)) { in tok_nextc()
948 tok->done = E_EOF; in tok_nextc()
952 tok->done = E_OK; in tok_nextc()
954 tok->lineno++; in tok_nextc()
957 Py_ssize_t curstart = tok->start == NULL ? -1 : in tok_nextc()
958 tok->start - tok->buf; in tok_nextc()
959 Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf; in tok_nextc()
960 Py_ssize_t curvalid = tok->inp - tok->buf; in tok_nextc()
962 char *newbuf = tok->buf; in tok_nextc()
966 tok->done = E_NOMEM; in tok_nextc()
967 tok->cur = tok->inp; in tok_nextc()
970 tok->buf = newbuf; in tok_nextc()
971 tok->cur = tok->buf + cur; in tok_nextc()
972 tok->multi_line_start = tok->buf + cur_multi_line_start; in tok_nextc()
973 tok->line_start = tok->cur; in tok_nextc()
974 tok->inp = tok->buf + curvalid; in tok_nextc()
975 tok->end = tok->buf + newsize; in tok_nextc()
976 tok->start = curstart < 0 ? NULL : in tok_nextc()
977 tok->buf + curstart; in tok_nextc()
978 if (decoding_fgets(tok->inp, in tok_nextc()
979 (int)(tok->end - tok->inp), in tok_nextc()
980 tok) == NULL) { in tok_nextc()
984 if (tok->decoding_erred) in tok_nextc()
988 if (tok->inp[-1] != '\n') in tok_nextc()
989 strcpy(tok->inp, "\n"); in tok_nextc()
991 tok->inp = strchr(tok->inp, '\0'); in tok_nextc()
992 done = tok->inp[-1] == '\n'; in tok_nextc()
994 if (tok->buf != NULL) { in tok_nextc()
995 tok->cur = tok->buf + cur; in tok_nextc()
996 tok->line_start = tok->cur; in tok_nextc()
999 pt = tok->inp - 2; in tok_nextc()
1000 if (pt >= tok->buf && *pt == '\r') { in tok_nextc()
1003 tok->inp = pt; in tok_nextc()
1007 if (tok->done != E_OK) { in tok_nextc()
1008 if (tok->prompt != NULL) in tok_nextc()
1010 tok->cur = tok->inp; in tok_nextc()
1021 tok_backup(struct tok_state *tok, int c) in tok_backup() argument
1024 if (--tok->cur < tok->buf) in tok_backup()
1026 if (*tok->cur != c) in tok_backup()
1027 *tok->cur = c; in tok_backup()
1033 syntaxerror(struct tok_state *tok, const char *format, ...) in syntaxerror() argument
1043 PyErr_SyntaxLocationObject(tok->filename, in syntaxerror()
1044 tok->lineno, in syntaxerror()
1045 (int)(tok->cur - tok->line_start)); in syntaxerror()
1046 tok->done = E_ERROR; in syntaxerror()
1051 indenterror(struct tok_state *tok) in indenterror() argument
1053 tok->done = E_TABSPACE; in indenterror()
1054 tok->cur = tok->inp; in indenterror()
1062 verify_identifier(struct tok_state *tok) in verify_identifier() argument
1066 if (tok->decoding_erred) in verify_identifier()
1068 s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL); in verify_identifier()
1072 tok->done = E_IDENTIFIER; in verify_identifier()
1074 tok->done = E_ERROR; in verify_identifier()
1081 tok->done = E_IDENTIFIER; in verify_identifier()
1086 tok_decimal_tail(struct tok_state *tok) in tok_decimal_tail() argument
1092 c = tok_nextc(tok); in tok_decimal_tail()
1097 c = tok_nextc(tok); in tok_decimal_tail()
1099 tok_backup(tok, c); in tok_decimal_tail()
1100 syntaxerror(tok, "invalid decimal literal"); in tok_decimal_tail()
1110 tok_get(struct tok_state *tok, char **p_start, char **p_end) in tok_get() argument
1117 tok->start = NULL; in tok_get()
1121 if (tok->atbol) { in tok_get()
1124 tok->atbol = 0; in tok_get()
1126 c = tok_nextc(tok); in tok_get()
1131 col = (col / tok->tabsize + 1) * tok->tabsize; in tok_get()
1141 tok_backup(tok, c); in tok_get()
1148 if (col == 0 && c == '\n' && tok->prompt != NULL) { in tok_get()
1151 else if (tok->prompt != NULL && tok->lineno == 1) { in tok_get()
1163 if (!blankline && tok->level == 0) { in tok_get()
1164 if (col == tok->indstack[tok->indent]) { in tok_get()
1166 if (altcol != tok->altindstack[tok->indent]) { in tok_get()
1167 return indenterror(tok); in tok_get()
1170 else if (col > tok->indstack[tok->indent]) { in tok_get()
1172 if (tok->indent+1 >= MAXINDENT) { in tok_get()
1173 tok->done = E_TOODEEP; in tok_get()
1174 tok->cur = tok->inp; in tok_get()
1177 if (altcol <= tok->altindstack[tok->indent]) { in tok_get()
1178 return indenterror(tok); in tok_get()
1180 tok->pendin++; in tok_get()
1181 tok->indstack[++tok->indent] = col; in tok_get()
1182 tok->altindstack[tok->indent] = altcol; in tok_get()
1186 while (tok->indent > 0 && in tok_get()
1187 col < tok->indstack[tok->indent]) { in tok_get()
1188 tok->pendin--; in tok_get()
1189 tok->indent--; in tok_get()
1191 if (col != tok->indstack[tok->indent]) { in tok_get()
1192 tok->done = E_DEDENT; in tok_get()
1193 tok->cur = tok->inp; in tok_get()
1196 if (altcol != tok->altindstack[tok->indent]) { in tok_get()
1197 return indenterror(tok); in tok_get()
1203 tok->start = tok->cur; in tok_get()
1206 if (tok->pendin != 0) { in tok_get()
1207 if (tok->pendin < 0) { in tok_get()
1208 tok->pendin++; in tok_get()
1212 tok->pendin--; in tok_get()
1218 c = tok_nextc(tok); in tok_get()
1219 tok_backup(tok, c); in tok_get()
1221 if (tok->async_def in tok_get()
1229 && tok->level == 0 in tok_get()
1232 && tok->async_def_nl in tok_get()
1235 && tok->async_def_indent >= tok->indent) in tok_get()
1237 tok->async_def = 0; in tok_get()
1238 tok->async_def_indent = 0; in tok_get()
1239 tok->async_def_nl = 0; in tok_get()
1243 tok->start = NULL; in tok_get()
1246 c = tok_nextc(tok); in tok_get()
1250 tok->start = tok->cur - 1; in tok_get()
1257 c = tok_nextc(tok); in tok_get()
1260 if (tok->type_comments) { in tok_get()
1261 p = tok->start; in tok_get()
1263 while (*prefix && p < tok->cur) { in tok_get()
1281 tok_backup(tok, c); /* don't eat the newline or EOF */ in tok_get()
1288 tok->cur >= ignore_end && memcmp(p, "ignore", 6) == 0 in tok_get()
1289 && !(tok->cur > ignore_end in tok_get()
1294 *p_end = tok->cur; in tok_get()
1298 tok_nextc(tok); in tok_get()
1299 tok->atbol = 1; in tok_get()
1304 *p_end = tok->cur; in tok_get()
1313 return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN; in tok_get()
1340 c = tok_nextc(tok); in tok_get()
1349 c = tok_nextc(tok); in tok_get()
1351 tok_backup(tok, c); in tok_get()
1352 if (nonascii && !verify_identifier(tok)) { in tok_get()
1355 *p_start = tok->start; in tok_get()
1356 *p_end = tok->cur; in tok_get()
1359 if (tok->cur - tok->start == 5 && tok->start[0] == 'a') { in tok_get()
1368 if (!tok->async_hacks || tok->async_def) { in tok_get()
1370 if (memcmp(tok->start, "async", 5) == 0) { in tok_get()
1373 if (memcmp(tok->start, "await", 5) == 0) { in tok_get()
1377 else if (memcmp(tok->start, "async", 5) == 0) { in tok_get()
1385 memcpy(&ahead_tok, tok, sizeof(ahead_tok)); in tok_get()
1395 tok->async_def_indent = tok->indent; in tok_get()
1396 tok->async_def = 1; in tok_get()
1407 tok->atbol = 1; in tok_get()
1408 if (blankline || tok->level > 0) { in tok_get()
1411 *p_start = tok->start; in tok_get()
1412 *p_end = tok->cur - 1; /* Leave '\n' out of the string */ in tok_get()
1413 tok->cont_line = 0; in tok_get()
1414 if (tok->async_def) { in tok_get()
1417 tok->async_def_nl = 1; in tok_get()
1424 c = tok_nextc(tok); in tok_get()
1428 c = tok_nextc(tok); in tok_get()
1430 *p_start = tok->start; in tok_get()
1431 *p_end = tok->cur; in tok_get()
1435 tok_backup(tok, c); in tok_get()
1437 tok_backup(tok, '.'); in tok_get()
1440 tok_backup(tok, c); in tok_get()
1442 *p_start = tok->start; in tok_get()
1443 *p_end = tok->cur; in tok_get()
1451 c = tok_nextc(tok); in tok_get()
1454 c = tok_nextc(tok); in tok_get()
1457 c = tok_nextc(tok); in tok_get()
1460 tok_backup(tok, c); in tok_get()
1461 return syntaxerror(tok, "invalid hexadecimal literal"); in tok_get()
1464 c = tok_nextc(tok); in tok_get()
1470 c = tok_nextc(tok); in tok_get()
1473 c = tok_nextc(tok); in tok_get()
1476 tok_backup(tok, c); in tok_get()
1478 return syntaxerror(tok, in tok_get()
1482 return syntaxerror(tok, "invalid octal literal"); in tok_get()
1486 c = tok_nextc(tok); in tok_get()
1490 return syntaxerror(tok, in tok_get()
1496 c = tok_nextc(tok); in tok_get()
1499 c = tok_nextc(tok); in tok_get()
1502 tok_backup(tok, c); in tok_get()
1504 return syntaxerror(tok, in tok_get()
1508 return syntaxerror(tok, "invalid binary literal"); in tok_get()
1512 c = tok_nextc(tok); in tok_get()
1516 return syntaxerror(tok, in tok_get()
1526 c = tok_nextc(tok); in tok_get()
1528 tok_backup(tok, c); in tok_get()
1529 return syntaxerror(tok, "invalid decimal literal"); in tok_get()
1535 c = tok_nextc(tok); in tok_get()
1539 c = tok_decimal_tail(tok); in tok_get()
1545 c = tok_nextc(tok); in tok_get()
1556 tok_backup(tok, c); in tok_get()
1557 return syntaxerror(tok, in tok_get()
1566 c = tok_decimal_tail(tok); in tok_get()
1573 c = tok_nextc(tok); in tok_get()
1577 c = tok_decimal_tail(tok); in tok_get()
1588 c = tok_nextc(tok); in tok_get()
1590 c = tok_nextc(tok); in tok_get()
1592 tok_backup(tok, c); in tok_get()
1593 return syntaxerror(tok, "invalid decimal literal"); in tok_get()
1596 tok_backup(tok, c); in tok_get()
1597 tok_backup(tok, e); in tok_get()
1598 *p_start = tok->start; in tok_get()
1599 *p_end = tok->cur; in tok_get()
1602 c = tok_decimal_tail(tok); in tok_get()
1610 c = tok_nextc(tok); in tok_get()
1614 tok_backup(tok, c); in tok_get()
1615 *p_start = tok->start; in tok_get()
1616 *p_end = tok->cur; in tok_get()
1631 tok->first_lineno = tok->lineno; in tok_get()
1632 tok->multi_line_start = tok->line_start; in tok_get()
1635 c = tok_nextc(tok); in tok_get()
1637 c = tok_nextc(tok); in tok_get()
1646 tok_backup(tok, c); in tok_get()
1651 c = tok_nextc(tok); in tok_get()
1654 tok->done = E_EOFS; in tok_get()
1657 tok->done = E_EOLS; in tok_get()
1659 tok->cur = tok->inp; in tok_get()
1663 tok->done = E_EOLS; in tok_get()
1664 tok->cur = tok->inp; in tok_get()
1673 tok_nextc(tok); /* skip escaped char */ in tok_get()
1678 *p_start = tok->start; in tok_get()
1679 *p_end = tok->cur; in tok_get()
1685 c = tok_nextc(tok); in tok_get()
1687 tok->done = E_LINECONT; in tok_get()
1688 tok->cur = tok->inp; in tok_get()
1691 c = tok_nextc(tok); in tok_get()
1693 tok->done = E_EOF; in tok_get()
1694 tok->cur = tok->inp; in tok_get()
1697 tok_backup(tok, c); in tok_get()
1699 tok->cont_line = 1; in tok_get()
1705 int c2 = tok_nextc(tok); in tok_get()
1708 int c3 = tok_nextc(tok); in tok_get()
1714 tok_backup(tok, c3); in tok_get()
1716 *p_start = tok->start; in tok_get()
1717 *p_end = tok->cur; in tok_get()
1720 tok_backup(tok, c2); in tok_get()
1728 if (tok->level >= MAXLEVEL) { in tok_get()
1729 return syntaxerror(tok, "too many nested parentheses"); in tok_get()
1731 tok->parenstack[tok->level] = c; in tok_get()
1732 tok->parenlinenostack[tok->level] = tok->lineno; in tok_get()
1733 tok->level++; in tok_get()
1738 if (!tok->level) { in tok_get()
1739 return syntaxerror(tok, "unmatched '%c'", c); in tok_get()
1741 tok->level--; in tok_get()
1742 int opening = tok->parenstack[tok->level]; in tok_get()
1747 if (tok->parenlinenostack[tok->level] != tok->lineno) { in tok_get()
1748 return syntaxerror(tok, in tok_get()
1751 c, opening, tok->parenlinenostack[tok->level]); in tok_get()
1754 return syntaxerror(tok, in tok_get()
1764 *p_start = tok->start; in tok_get()
1765 *p_end = tok->cur; in tok_get()
1770 PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end) in PyTokenizer_Get() argument
1772 int result = tok_get(tok, p_start, p_end); in PyTokenizer_Get()
1773 if (tok->decoding_erred) { in PyTokenizer_Get()
1775 tok->done = E_DECODE; in PyTokenizer_Get()
1793 struct tok_state *tok; in PyTokenizer_FindEncodingFilename() local
1806 tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL); in PyTokenizer_FindEncodingFilename()
1807 if (tok == NULL) { in PyTokenizer_FindEncodingFilename()
1813 tok->filename = filename; in PyTokenizer_FindEncodingFilename()
1816 tok->filename = PyUnicode_FromString("<string>"); in PyTokenizer_FindEncodingFilename()
1817 if (tok->filename == NULL) { in PyTokenizer_FindEncodingFilename()
1819 PyTokenizer_Free(tok); in PyTokenizer_FindEncodingFilename()
1823 while (tok->lineno < 2 && tok->done == E_OK) { in PyTokenizer_FindEncodingFilename()
1824 PyTokenizer_Get(tok, &p_start, &p_end); in PyTokenizer_FindEncodingFilename()
1827 if (tok->encoding) { in PyTokenizer_FindEncodingFilename()
1828 encoding = (char *)PyMem_MALLOC(strlen(tok->encoding) + 1); in PyTokenizer_FindEncodingFilename()
1830 strcpy(encoding, tok->encoding); in PyTokenizer_FindEncodingFilename()
1832 PyTokenizer_Free(tok); in PyTokenizer_FindEncodingFilename()