1 #include "Python.h"
2 #include "errcode.h"
3
4 #include "helpers.h"
5 #include "../lexer/state.h"
6
7 static int
tok_underflow_string(struct tok_state * tok)8 tok_underflow_string(struct tok_state *tok) {
9 char *end = strchr(tok->inp, '\n');
10 if (end != NULL) {
11 end++;
12 }
13 else {
14 end = strchr(tok->inp, '\0');
15 if (end == tok->inp) {
16 tok->done = E_EOF;
17 return 0;
18 }
19 }
20 if (tok->start == NULL) {
21 tok->buf = tok->cur;
22 }
23 tok->line_start = tok->cur;
24 ADVANCE_LINENO();
25 tok->inp = end;
26 return 1;
27 }
28
29 /* Set up tokenizer for UTF-8 string */
30 struct tok_state *
_PyTokenizer_FromUTF8(const char * str,int exec_input,int preserve_crlf)31 _PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf)
32 {
33 struct tok_state *tok = _PyTokenizer_tok_new();
34 char *translated;
35 if (tok == NULL)
36 return NULL;
37 tok->input = translated = _PyTokenizer_translate_newlines(str, exec_input, preserve_crlf, tok);
38 if (translated == NULL) {
39 _PyTokenizer_Free(tok);
40 return NULL;
41 }
42 tok->decoding_state = STATE_NORMAL;
43 tok->enc = NULL;
44 tok->str = translated;
45 tok->encoding = _PyTokenizer_new_string("utf-8", 5, tok);
46 if (!tok->encoding) {
47 _PyTokenizer_Free(tok);
48 return NULL;
49 }
50
51 tok->buf = tok->cur = tok->inp = translated;
52 tok->end = translated;
53 tok->underflow = &tok_underflow_string;
54 return tok;
55 }
56