1 #include "Python.h" 2 #include "errcode.h" 3 4 #include "state.h" 5 6 /* Traverse and remember all f-string buffers, in order to be able to restore 7 them after reallocating tok->buf */ 8 void _PyLexer_remember_fstring_buffers(struct tok_state * tok)9_PyLexer_remember_fstring_buffers(struct tok_state *tok) 10 { 11 int index; 12 tokenizer_mode *mode; 13 14 for (index = tok->tok_mode_stack_index; index >= 0; --index) { 15 mode = &(tok->tok_mode_stack[index]); 16 mode->f_string_start_offset = mode->f_string_start - tok->buf; 17 mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf; 18 } 19 } 20 21 /* Traverse and restore all f-string buffers after reallocating tok->buf */ 22 void _PyLexer_restore_fstring_buffers(struct tok_state * tok)23_PyLexer_restore_fstring_buffers(struct tok_state *tok) 24 { 25 int index; 26 tokenizer_mode *mode; 27 28 for (index = tok->tok_mode_stack_index; index >= 0; --index) { 29 mode = &(tok->tok_mode_stack[index]); 30 mode->f_string_start = tok->buf + mode->f_string_start_offset; 31 mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset; 32 } 33 } 34 35 /* Read a line of text from TOK into S, using the stream in TOK. 36 Return NULL on failure, else S. 37 38 On entry, tok->decoding_buffer will be one of: 39 1) NULL: need to call tok->decoding_readline to get a new line 40 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and 41 stored the result in tok->decoding_buffer 42 3) PyByteArrayObject *: previous call to tok_readline_recode did not have enough room 43 (in the s buffer) to copy entire contents of the line read 44 by tok->decoding_readline. tok->decoding_buffer has the overflow. 45 In this case, tok_readline_recode is called in a loop (with an expanded buffer) 46 until the buffer ends with a '\n' (or until the end of the file is 47 reached): see tok_nextc and its calls to tok_reserve_buf. 48 */ 49 int _PyLexer_tok_reserve_buf(struct tok_state * tok,Py_ssize_t size)50_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) 51 { 52 Py_ssize_t cur = tok->cur - tok->buf; 53 Py_ssize_t oldsize = tok->inp - tok->buf; 54 Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1); 55 if (newsize > tok->end - tok->buf) { 56 char *newbuf = tok->buf; 57 Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf; 58 Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf; 59 Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf; 60 _PyLexer_remember_fstring_buffers(tok); 61 newbuf = (char *)PyMem_Realloc(newbuf, newsize); 62 if (newbuf == NULL) { 63 tok->done = E_NOMEM; 64 return 0; 65 } 66 tok->buf = newbuf; 67 tok->cur = tok->buf + cur; 68 tok->inp = tok->buf + oldsize; 69 tok->end = tok->buf + newsize; 70 tok->start = start < 0 ? NULL : tok->buf + start; 71 tok->line_start = line_start < 0 ? NULL : tok->buf + line_start; 72 tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start; 73 _PyLexer_restore_fstring_buffers(tok); 74 } 75 return 1; 76 } 77