1 #include "Python.h"
2 #include "pycore_pystate.h"
3 #include "pycore_token.h"
4 #include "errcode.h"
5
6 #include "state.h"
7
8 /* Never change this */
9 #define TABSIZE 8
10
11 /* Create and initialize a new tok_state structure */
12 struct tok_state *
_PyTokenizer_tok_new(void)13 _PyTokenizer_tok_new(void)
14 {
15 struct tok_state *tok = (struct tok_state *)PyMem_Calloc(
16 1,
17 sizeof(struct tok_state));
18 if (tok == NULL)
19 return NULL;
20 tok->buf = tok->cur = tok->inp = NULL;
21 tok->fp_interactive = 0;
22 tok->interactive_src_start = NULL;
23 tok->interactive_src_end = NULL;
24 tok->start = NULL;
25 tok->end = NULL;
26 tok->done = E_OK;
27 tok->fp = NULL;
28 tok->input = NULL;
29 tok->tabsize = TABSIZE;
30 tok->indent = 0;
31 tok->indstack[0] = 0;
32 tok->atbol = 1;
33 tok->pendin = 0;
34 tok->prompt = tok->nextprompt = NULL;
35 tok->lineno = 0;
36 tok->starting_col_offset = -1;
37 tok->col_offset = -1;
38 tok->level = 0;
39 tok->altindstack[0] = 0;
40 tok->decoding_state = STATE_INIT;
41 tok->decoding_erred = 0;
42 tok->enc = NULL;
43 tok->encoding = NULL;
44 tok->cont_line = 0;
45 tok->filename = NULL;
46 tok->decoding_readline = NULL;
47 tok->decoding_buffer = NULL;
48 tok->readline = NULL;
49 tok->type_comments = 0;
50 tok->interactive_underflow = IUNDERFLOW_NORMAL;
51 tok->underflow = NULL;
52 tok->str = NULL;
53 tok->report_warnings = 1;
54 tok->tok_extra_tokens = 0;
55 tok->comment_newline = 0;
56 tok->implicit_newline = 0;
57 tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0};
58 tok->tok_mode_stack_index = 0;
59 #ifdef Py_DEBUG
60 tok->debug = _Py_GetConfig()->parser_debug;
61 #endif
62 return tok;
63 }
64
65 static void
free_fstring_expressions(struct tok_state * tok)66 free_fstring_expressions(struct tok_state *tok)
67 {
68 int index;
69 tokenizer_mode *mode;
70
71 for (index = tok->tok_mode_stack_index; index >= 0; --index) {
72 mode = &(tok->tok_mode_stack[index]);
73 if (mode->last_expr_buffer != NULL) {
74 PyMem_Free(mode->last_expr_buffer);
75 mode->last_expr_buffer = NULL;
76 mode->last_expr_size = 0;
77 mode->last_expr_end = -1;
78 mode->in_format_spec = 0;
79 }
80 }
81 }
82
83 /* Free a tok_state structure */
84 void
_PyTokenizer_Free(struct tok_state * tok)85 _PyTokenizer_Free(struct tok_state *tok)
86 {
87 if (tok->encoding != NULL) {
88 PyMem_Free(tok->encoding);
89 }
90 Py_XDECREF(tok->decoding_readline);
91 Py_XDECREF(tok->decoding_buffer);
92 Py_XDECREF(tok->readline);
93 Py_XDECREF(tok->filename);
94 if ((tok->readline != NULL || tok->fp != NULL ) && tok->buf != NULL) {
95 PyMem_Free(tok->buf);
96 }
97 if (tok->input) {
98 PyMem_Free(tok->input);
99 }
100 if (tok->interactive_src_start != NULL) {
101 PyMem_Free(tok->interactive_src_start);
102 }
103 free_fstring_expressions(tok);
104 PyMem_Free(tok);
105 }
106
107 void
_PyToken_Free(struct token * token)108 _PyToken_Free(struct token *token) {
109 Py_XDECREF(token->metadata);
110 }
111
112 void
_PyToken_Init(struct token * token)113 _PyToken_Init(struct token *token) {
114 token->metadata = NULL;
115 }
116
117 int
_PyLexer_type_comment_token_setup(struct tok_state * tok,struct token * token,int type,int col_offset,int end_col_offset,const char * start,const char * end)118 _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset,
119 int end_col_offset, const char *start, const char *end)
120 {
121 token->level = tok->level;
122 token->lineno = token->end_lineno = tok->lineno;
123 token->col_offset = col_offset;
124 token->end_col_offset = end_col_offset;
125 token->start = start;
126 token->end = end;
127 return type;
128 }
129
130 int
_PyLexer_token_setup(struct tok_state * tok,struct token * token,int type,const char * start,const char * end)131 _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end)
132 {
133 assert((start == NULL && end == NULL) || (start != NULL && end != NULL));
134 token->level = tok->level;
135 if (ISSTRINGLIT(type)) {
136 token->lineno = tok->first_lineno;
137 }
138 else {
139 token->lineno = tok->lineno;
140 }
141 token->end_lineno = tok->lineno;
142 token->col_offset = token->end_col_offset = -1;
143 token->start = start;
144 token->end = end;
145
146 if (start != NULL && end != NULL) {
147 token->col_offset = tok->starting_col_offset;
148 token->end_col_offset = tok->col_offset;
149 }
150 return type;
151 }
152