1 #ifndef PEGEN_H
2 #define PEGEN_H
3
4 #define PY_SSIZE_T_CLEAN
5 #include <Python.h>
6 #include <token.h>
7 #include <Python-ast.h>
8 #include <pyarena.h>
9
10 #if 0
11 #define PyPARSE_YIELD_IS_KEYWORD 0x0001
12 #endif
13
14 #define PyPARSE_DONT_IMPLY_DEDENT 0x0002
15
16 #if 0
17 #define PyPARSE_WITH_IS_KEYWORD 0x0003
18 #define PyPARSE_PRINT_IS_FUNCTION 0x0004
19 #define PyPARSE_UNICODE_LITERALS 0x0008
20 #endif
21
22 #define PyPARSE_IGNORE_COOKIE 0x0010
23 #define PyPARSE_BARRY_AS_BDFL 0x0020
24 #define PyPARSE_TYPE_COMMENTS 0x0040
25 #define PyPARSE_ASYNC_HACKS 0x0080
26
27 typedef struct _memo {
28 int type;
29 void *node;
30 int mark;
31 struct _memo *next;
32 } Memo;
33
34 typedef struct {
35 int type;
36 PyObject *bytes;
37 int lineno, col_offset, end_lineno, end_col_offset;
38 Memo *memo;
39 } Token;
40
41 typedef struct {
42 char *str;
43 int type;
44 } KeywordToken;
45
46
47 typedef struct {
48 struct {
49 int lineno;
50 char *comment; // The " <tag>" in "# type: ignore <tag>"
51 } *items;
52 size_t size;
53 size_t num_items;
54 } growable_comment_array;
55
56 typedef struct {
57 struct tok_state *tok;
58 Token **tokens;
59 int mark;
60 int fill, size;
61 PyArena *arena;
62 KeywordToken **keywords;
63 int n_keyword_lists;
64 int start_rule;
65 int *errcode;
66 int parsing_started;
67 PyObject* normalize;
68 int starting_lineno;
69 int starting_col_offset;
70 int error_indicator;
71 int flags;
72 int feature_version;
73 growable_comment_array type_ignore_comments;
74 Token *known_err_token;
75 int level;
76 int call_invalid_rules;
77 } Parser;
78
79 typedef struct {
80 cmpop_ty cmpop;
81 expr_ty expr;
82 } CmpopExprPair;
83
84 typedef struct {
85 expr_ty key;
86 expr_ty value;
87 } KeyValuePair;
88
89 typedef struct {
90 arg_ty arg;
91 expr_ty value;
92 } NameDefaultPair;
93
94 typedef struct {
95 asdl_seq *plain_names;
96 asdl_seq *names_with_defaults; // asdl_seq* of NameDefaultsPair's
97 } SlashWithDefault;
98
99 typedef struct {
100 arg_ty vararg;
101 asdl_seq *kwonlyargs; // asdl_seq* of NameDefaultsPair's
102 arg_ty kwarg;
103 } StarEtc;
104
105 typedef struct {
106 operator_ty kind;
107 } AugOperator;
108
109 typedef struct {
110 void *element;
111 int is_keyword;
112 } KeywordOrStarred;
113
114 void _PyPegen_clear_memo_statistics(void);
115 PyObject *_PyPegen_get_memo_statistics(void);
116
117 int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node);
118 int _PyPegen_update_memo(Parser *p, int mark, int type, void *node);
119 int _PyPegen_is_memoized(Parser *p, int type, void *pres);
120
121 int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *);
122 int _PyPegen_lookahead_with_string(int , expr_ty (func)(Parser *, const char*), Parser *, const char*);
123 int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int);
124 int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *);
125
126 Token *_PyPegen_expect_token(Parser *p, int type);
127 expr_ty _PyPegen_expect_soft_keyword(Parser *p, const char *keyword);
128 Token *_PyPegen_get_last_nonnwhitespace_token(Parser *);
129 int _PyPegen_fill_token(Parser *p);
130 expr_ty _PyPegen_name_token(Parser *p);
131 expr_ty _PyPegen_number_token(Parser *p);
132 void *_PyPegen_string_token(Parser *p);
133 const char *_PyPegen_get_expr_name(expr_ty);
134 void *_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...);
135 void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
136 Py_ssize_t lineno, Py_ssize_t col_offset,
137 const char *errmsg, va_list va);
138 void *_PyPegen_dummy_name(Parser *p, ...);
139
140 Py_LOCAL_INLINE(void *)
RAISE_ERROR_KNOWN_LOCATION(Parser * p,PyObject * errtype,int lineno,int col_offset,const char * errmsg,...)141 RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype, int lineno,
142 int col_offset, const char *errmsg, ...)
143 {
144 va_list va;
145 va_start(va, errmsg);
146 _PyPegen_raise_error_known_location(p, errtype, lineno, col_offset + 1,
147 errmsg, va);
148 va_end(va);
149 return NULL;
150 }
151
152
153 #define UNUSED(expr) do { (void)(expr); } while (0)
154 #define EXTRA_EXPR(head, tail) head->lineno, head->col_offset, tail->end_lineno, tail->end_col_offset, p->arena
155 #define EXTRA _start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena
156 #define RAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, msg, ##__VA_ARGS__)
157 #define RAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, msg, ##__VA_ARGS__)
158 #define RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, msg, ...) \
159 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, (a)->lineno, (a)->col_offset, msg, ##__VA_ARGS__)
160
161 Py_LOCAL_INLINE(void *)
CHECK_CALL(Parser * p,void * result)162 CHECK_CALL(Parser *p, void *result)
163 {
164 if (result == NULL) {
165 assert(PyErr_Occurred());
166 p->error_indicator = 1;
167 }
168 return result;
169 }
170
171 /* This is needed for helper functions that are allowed to
172 return NULL without an error. Example: _PyPegen_seq_extract_starred_exprs */
173 Py_LOCAL_INLINE(void *)
CHECK_CALL_NULL_ALLOWED(Parser * p,void * result)174 CHECK_CALL_NULL_ALLOWED(Parser *p, void *result)
175 {
176 if (result == NULL && PyErr_Occurred()) {
177 p->error_indicator = 1;
178 }
179 return result;
180 }
181
182 #define CHECK(result) CHECK_CALL(p, result)
183 #define CHECK_NULL_ALLOWED(result) CHECK_CALL_NULL_ALLOWED(p, result)
184
185 PyObject *_PyPegen_new_type_comment(Parser *, char *);
186
187 Py_LOCAL_INLINE(PyObject *)
NEW_TYPE_COMMENT(Parser * p,Token * tc)188 NEW_TYPE_COMMENT(Parser *p, Token *tc)
189 {
190 if (tc == NULL) {
191 return NULL;
192 }
193 char *bytes = PyBytes_AsString(tc->bytes);
194 if (bytes == NULL) {
195 goto error;
196 }
197 PyObject *tco = _PyPegen_new_type_comment(p, bytes);
198 if (tco == NULL) {
199 goto error;
200 }
201 return tco;
202 error:
203 p->error_indicator = 1; // Inline CHECK_CALL
204 return NULL;
205 }
206
207 Py_LOCAL_INLINE(void *)
INVALID_VERSION_CHECK(Parser * p,int version,char * msg,void * node)208 INVALID_VERSION_CHECK(Parser *p, int version, char *msg, void *node)
209 {
210 if (node == NULL) {
211 p->error_indicator = 1; // Inline CHECK_CALL
212 return NULL;
213 }
214 if (p->feature_version < version) {
215 p->error_indicator = 1;
216 return RAISE_SYNTAX_ERROR("%s only supported in Python 3.%i and greater",
217 msg, version);
218 }
219 return node;
220 }
221
222 #define CHECK_VERSION(version, msg, node) INVALID_VERSION_CHECK(p, version, msg, node)
223
224 arg_ty _PyPegen_add_type_comment_to_arg(Parser *, arg_ty, Token *);
225 PyObject *_PyPegen_new_identifier(Parser *, char *);
226 Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int, int *, PyArena *);
227 void _PyPegen_Parser_Free(Parser *);
228 mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
229 const char *, const char *, PyCompilerFlags *, int *, PyArena *);
230 void *_PyPegen_run_parser(Parser *);
231 mod_ty _PyPegen_run_parser_from_file(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
232 mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
233 void *_PyPegen_interactive_exit(Parser *);
234 asdl_seq *_PyPegen_singleton_seq(Parser *, void *);
235 asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *);
236 asdl_seq *_PyPegen_seq_append_to_end(Parser *, asdl_seq *, void *);
237 asdl_seq *_PyPegen_seq_flatten(Parser *, asdl_seq *);
238 expr_ty _PyPegen_join_names_with_dot(Parser *, expr_ty, expr_ty);
239 int _PyPegen_seq_count_dots(asdl_seq *);
240 alias_ty _PyPegen_alias_for_star(Parser *);
241 asdl_seq *_PyPegen_map_names_to_ids(Parser *, asdl_seq *);
242 CmpopExprPair *_PyPegen_cmpop_expr_pair(Parser *, cmpop_ty, expr_ty);
243 asdl_int_seq *_PyPegen_get_cmpops(Parser *p, asdl_seq *);
244 asdl_seq *_PyPegen_get_exprs(Parser *, asdl_seq *);
245 expr_ty _PyPegen_set_expr_context(Parser *, expr_ty, expr_context_ty);
246 KeyValuePair *_PyPegen_key_value_pair(Parser *, expr_ty, expr_ty);
247 asdl_seq *_PyPegen_get_keys(Parser *, asdl_seq *);
248 asdl_seq *_PyPegen_get_values(Parser *, asdl_seq *);
249 NameDefaultPair *_PyPegen_name_default_pair(Parser *, arg_ty, expr_ty, Token *);
250 SlashWithDefault *_PyPegen_slash_with_default(Parser *, asdl_seq *, asdl_seq *);
251 StarEtc *_PyPegen_star_etc(Parser *, arg_ty, asdl_seq *, arg_ty);
252 arguments_ty _PyPegen_make_arguments(Parser *, asdl_seq *, SlashWithDefault *,
253 asdl_seq *, asdl_seq *, StarEtc *);
254 arguments_ty _PyPegen_empty_arguments(Parser *);
255 AugOperator *_PyPegen_augoperator(Parser*, operator_ty type);
256 stmt_ty _PyPegen_function_def_decorators(Parser *, asdl_seq *, stmt_ty);
257 stmt_ty _PyPegen_class_def_decorators(Parser *, asdl_seq *, stmt_ty);
258 KeywordOrStarred *_PyPegen_keyword_or_starred(Parser *, void *, int);
259 asdl_seq *_PyPegen_seq_extract_starred_exprs(Parser *, asdl_seq *);
260 asdl_seq *_PyPegen_seq_delete_starred_exprs(Parser *, asdl_seq *);
261 expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_seq *, asdl_seq *,
262 int lineno, int col_offset, int end_lineno,
263 int end_col_offset, PyArena *arena);
264 expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_seq *);
265 asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *);
266 int _PyPegen_check_barry_as_flufl(Parser *, Token *);
267 mod_ty _PyPegen_make_module(Parser *, asdl_seq *);
268
269 // Error reporting helpers
270 typedef enum {
271 STAR_TARGETS,
272 DEL_TARGETS,
273 FOR_TARGETS
274 } TARGETS_TYPE;
275 expr_ty _PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type);
276 #define RAISE_SYNTAX_ERROR_INVALID_TARGET(type, e) _RAISE_SYNTAX_ERROR_INVALID_TARGET(p, type, e)
277
278 Py_LOCAL_INLINE(void *)
_RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser * p,TARGETS_TYPE type,void * e)279 _RAISE_SYNTAX_ERROR_INVALID_TARGET(Parser *p, TARGETS_TYPE type, void *e)
280 {
281 expr_ty invalid_target = CHECK_NULL_ALLOWED(_PyPegen_get_invalid_target(e, type));
282 if (invalid_target != NULL) {
283 const char *msg;
284 if (type == STAR_TARGETS || type == FOR_TARGETS) {
285 msg = "cannot assign to %s";
286 }
287 else {
288 msg = "cannot delete %s";
289 }
290 return RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
291 invalid_target,
292 msg,
293 _PyPegen_get_expr_name(invalid_target)
294 );
295 }
296 return RAISE_SYNTAX_ERROR("invalid syntax");
297 }
298
299 void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);
300 void *_PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args);
301
302
303 // Generated function in parse.c - function definition in python.gram
304 void *_PyPegen_parse(Parser *);
305
306 #endif
307