1 #include <Python.h>
2 #include "pycore_ast.h" // _PyAST_Validate(),
3 #include "pycore_pystate.h" // _PyThreadState_GET()
4 #include <errcode.h>
5 #include "tokenizer.h"
6
7 #include "pegen.h"
8 #include "string_parser.h"
9
10 PyObject *
_PyPegen_new_type_comment(Parser * p,const char * s)11 _PyPegen_new_type_comment(Parser *p, const char *s)
12 {
13 PyObject *res = PyUnicode_DecodeUTF8(s, strlen(s), NULL);
14 if (res == NULL) {
15 return NULL;
16 }
17 if (_PyArena_AddPyObject(p->arena, res) < 0) {
18 Py_DECREF(res);
19 return NULL;
20 }
21 return res;
22 }
23
24 arg_ty
_PyPegen_add_type_comment_to_arg(Parser * p,arg_ty a,Token * tc)25 _PyPegen_add_type_comment_to_arg(Parser *p, arg_ty a, Token *tc)
26 {
27 if (tc == NULL) {
28 return a;
29 }
30 const char *bytes = PyBytes_AsString(tc->bytes);
31 if (bytes == NULL) {
32 return NULL;
33 }
34 PyObject *tco = _PyPegen_new_type_comment(p, bytes);
35 if (tco == NULL) {
36 return NULL;
37 }
38 return _PyAST_arg(a->arg, a->annotation, tco,
39 a->lineno, a->col_offset, a->end_lineno, a->end_col_offset,
40 p->arena);
41 }
42
43 static int
init_normalization(Parser * p)44 init_normalization(Parser *p)
45 {
46 if (p->normalize) {
47 return 1;
48 }
49 PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
50 if (!m)
51 {
52 return 0;
53 }
54 p->normalize = PyObject_GetAttrString(m, "normalize");
55 Py_DECREF(m);
56 if (!p->normalize)
57 {
58 return 0;
59 }
60 return 1;
61 }
62
63 /* Checks if the NOTEQUAL token is valid given the current parser flags
64 0 indicates success and nonzero indicates failure (an exception may be set) */
65 int
_PyPegen_check_barry_as_flufl(Parser * p,Token * t)66 _PyPegen_check_barry_as_flufl(Parser *p, Token* t) {
67 assert(t->bytes != NULL);
68 assert(t->type == NOTEQUAL);
69
70 const char* tok_str = PyBytes_AS_STRING(t->bytes);
71 if (p->flags & PyPARSE_BARRY_AS_BDFL && strcmp(tok_str, "<>") != 0) {
72 RAISE_SYNTAX_ERROR("with Barry as BDFL, use '<>' instead of '!='");
73 return -1;
74 }
75 if (!(p->flags & PyPARSE_BARRY_AS_BDFL)) {
76 return strcmp(tok_str, "!=");
77 }
78 return 0;
79 }
80
81 int
_PyPegen_check_legacy_stmt(Parser * p,expr_ty name)82 _PyPegen_check_legacy_stmt(Parser *p, expr_ty name) {
83 if (name->kind != Name_kind) {
84 return 0;
85 }
86 const char* candidates[2] = {"print", "exec"};
87 for (int i=0; i<2; i++) {
88 if (PyUnicode_CompareWithASCIIString(name->v.Name.id, candidates[i]) == 0) {
89 return 1;
90 }
91 }
92 return 0;
93 }
94
95 PyObject *
_PyPegen_new_identifier(Parser * p,const char * n)96 _PyPegen_new_identifier(Parser *p, const char *n)
97 {
98 PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
99 if (!id) {
100 goto error;
101 }
102 /* PyUnicode_DecodeUTF8 should always return a ready string. */
103 assert(PyUnicode_IS_READY(id));
104 /* Check whether there are non-ASCII characters in the
105 identifier; if so, normalize to NFKC. */
106 if (!PyUnicode_IS_ASCII(id))
107 {
108 PyObject *id2;
109 if (!init_normalization(p))
110 {
111 Py_DECREF(id);
112 goto error;
113 }
114 PyObject *form = PyUnicode_InternFromString("NFKC");
115 if (form == NULL)
116 {
117 Py_DECREF(id);
118 goto error;
119 }
120 PyObject *args[2] = {form, id};
121 id2 = _PyObject_FastCall(p->normalize, args, 2);
122 Py_DECREF(id);
123 Py_DECREF(form);
124 if (!id2) {
125 goto error;
126 }
127 if (!PyUnicode_Check(id2))
128 {
129 PyErr_Format(PyExc_TypeError,
130 "unicodedata.normalize() must return a string, not "
131 "%.200s",
132 _PyType_Name(Py_TYPE(id2)));
133 Py_DECREF(id2);
134 goto error;
135 }
136 id = id2;
137 }
138 PyUnicode_InternInPlace(&id);
139 if (_PyArena_AddPyObject(p->arena, id) < 0)
140 {
141 Py_DECREF(id);
142 goto error;
143 }
144 return id;
145
146 error:
147 p->error_indicator = 1;
148 return NULL;
149 }
150
151 static PyObject *
_create_dummy_identifier(Parser * p)152 _create_dummy_identifier(Parser *p)
153 {
154 return _PyPegen_new_identifier(p, "");
155 }
156
157 static inline Py_ssize_t
byte_offset_to_character_offset(PyObject * line,Py_ssize_t col_offset)158 byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
159 {
160 const char *str = PyUnicode_AsUTF8(line);
161 if (!str) {
162 return 0;
163 }
164 Py_ssize_t len = strlen(str);
165 if (col_offset > len + 1) {
166 col_offset = len + 1;
167 }
168 assert(col_offset >= 0);
169 PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, "replace");
170 if (!text) {
171 return 0;
172 }
173 Py_ssize_t size = PyUnicode_GET_LENGTH(text);
174 Py_DECREF(text);
175 return size;
176 }
177
178 const char *
_PyPegen_get_expr_name(expr_ty e)179 _PyPegen_get_expr_name(expr_ty e)
180 {
181 assert(e != NULL);
182 switch (e->kind) {
183 case Attribute_kind:
184 return "attribute";
185 case Subscript_kind:
186 return "subscript";
187 case Starred_kind:
188 return "starred";
189 case Name_kind:
190 return "name";
191 case List_kind:
192 return "list";
193 case Tuple_kind:
194 return "tuple";
195 case Lambda_kind:
196 return "lambda";
197 case Call_kind:
198 return "function call";
199 case BoolOp_kind:
200 case BinOp_kind:
201 case UnaryOp_kind:
202 return "expression";
203 case GeneratorExp_kind:
204 return "generator expression";
205 case Yield_kind:
206 case YieldFrom_kind:
207 return "yield expression";
208 case Await_kind:
209 return "await expression";
210 case ListComp_kind:
211 return "list comprehension";
212 case SetComp_kind:
213 return "set comprehension";
214 case DictComp_kind:
215 return "dict comprehension";
216 case Dict_kind:
217 return "dict literal";
218 case Set_kind:
219 return "set display";
220 case JoinedStr_kind:
221 case FormattedValue_kind:
222 return "f-string expression";
223 case Constant_kind: {
224 PyObject *value = e->v.Constant.value;
225 if (value == Py_None) {
226 return "None";
227 }
228 if (value == Py_False) {
229 return "False";
230 }
231 if (value == Py_True) {
232 return "True";
233 }
234 if (value == Py_Ellipsis) {
235 return "ellipsis";
236 }
237 return "literal";
238 }
239 case Compare_kind:
240 return "comparison";
241 case IfExp_kind:
242 return "conditional expression";
243 case NamedExpr_kind:
244 return "named expression";
245 default:
246 PyErr_Format(PyExc_SystemError,
247 "unexpected expression in assignment %d (line %d)",
248 e->kind, e->lineno);
249 return NULL;
250 }
251 }
252
253 static int
raise_decode_error(Parser * p)254 raise_decode_error(Parser *p)
255 {
256 assert(PyErr_Occurred());
257 const char *errtype = NULL;
258 if (PyErr_ExceptionMatches(PyExc_UnicodeError)) {
259 errtype = "unicode error";
260 }
261 else if (PyErr_ExceptionMatches(PyExc_ValueError)) {
262 errtype = "value error";
263 }
264 if (errtype) {
265 PyObject *type;
266 PyObject *value;
267 PyObject *tback;
268 PyObject *errstr;
269 PyErr_Fetch(&type, &value, &tback);
270 errstr = PyObject_Str(value);
271 if (errstr) {
272 RAISE_SYNTAX_ERROR("(%s) %U", errtype, errstr);
273 Py_DECREF(errstr);
274 }
275 else {
276 PyErr_Clear();
277 RAISE_SYNTAX_ERROR("(%s) unknown error", errtype);
278 }
279 Py_XDECREF(type);
280 Py_XDECREF(value);
281 Py_XDECREF(tback);
282 }
283
284 return -1;
285 }
286
287 static inline void
raise_unclosed_parentheses_error(Parser * p)288 raise_unclosed_parentheses_error(Parser *p) {
289 int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
290 int error_col = p->tok->parencolstack[p->tok->level-1];
291 RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError,
292 error_lineno, error_col, error_lineno, -1,
293 "'%c' was never closed",
294 p->tok->parenstack[p->tok->level-1]);
295 }
296
297 static void
raise_tokenizer_init_error(PyObject * filename)298 raise_tokenizer_init_error(PyObject *filename)
299 {
300 if (!(PyErr_ExceptionMatches(PyExc_LookupError)
301 || PyErr_ExceptionMatches(PyExc_SyntaxError)
302 || PyErr_ExceptionMatches(PyExc_ValueError)
303 || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) {
304 return;
305 }
306 PyObject *errstr = NULL;
307 PyObject *tuple = NULL;
308 PyObject *type;
309 PyObject *value;
310 PyObject *tback;
311 PyErr_Fetch(&type, &value, &tback);
312 errstr = PyObject_Str(value);
313 if (!errstr) {
314 goto error;
315 }
316
317 PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None);
318 if (!tmp) {
319 goto error;
320 }
321
322 tuple = PyTuple_Pack(2, errstr, tmp);
323 Py_DECREF(tmp);
324 if (!value) {
325 goto error;
326 }
327 PyErr_SetObject(PyExc_SyntaxError, tuple);
328
329 error:
330 Py_XDECREF(type);
331 Py_XDECREF(value);
332 Py_XDECREF(tback);
333 Py_XDECREF(errstr);
334 Py_XDECREF(tuple);
335 }
336
337 static int
tokenizer_error(Parser * p)338 tokenizer_error(Parser *p)
339 {
340 if (PyErr_Occurred()) {
341 return -1;
342 }
343
344 const char *msg = NULL;
345 PyObject* errtype = PyExc_SyntaxError;
346 Py_ssize_t col_offset = -1;
347 switch (p->tok->done) {
348 case E_TOKEN:
349 msg = "invalid token";
350 break;
351 case E_EOF:
352 if (p->tok->level) {
353 raise_unclosed_parentheses_error(p);
354 } else {
355 RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
356 }
357 return -1;
358 case E_DEDENT:
359 RAISE_INDENTATION_ERROR("unindent does not match any outer indentation level");
360 return -1;
361 case E_INTR:
362 if (!PyErr_Occurred()) {
363 PyErr_SetNone(PyExc_KeyboardInterrupt);
364 }
365 return -1;
366 case E_NOMEM:
367 PyErr_NoMemory();
368 return -1;
369 case E_TABSPACE:
370 errtype = PyExc_TabError;
371 msg = "inconsistent use of tabs and spaces in indentation";
372 break;
373 case E_TOODEEP:
374 errtype = PyExc_IndentationError;
375 msg = "too many levels of indentation";
376 break;
377 case E_LINECONT: {
378 col_offset = p->tok->cur - p->tok->buf - 1;
379 msg = "unexpected character after line continuation character";
380 break;
381 }
382 default:
383 msg = "unknown parsing error";
384 }
385
386 RAISE_ERROR_KNOWN_LOCATION(p, errtype, p->tok->lineno,
387 col_offset >= 0 ? col_offset : 0,
388 p->tok->lineno, -1, msg);
389 return -1;
390 }
391
392 void *
_PyPegen_raise_error(Parser * p,PyObject * errtype,const char * errmsg,...)393 _PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...)
394 {
395 if (p->fill == 0) {
396 va_list va;
397 va_start(va, errmsg);
398 _PyPegen_raise_error_known_location(p, errtype, 0, 0, 0, -1, errmsg, va);
399 va_end(va);
400 return NULL;
401 }
402
403 Token *t = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
404 Py_ssize_t col_offset;
405 Py_ssize_t end_col_offset = -1;
406 if (t->col_offset == -1) {
407 if (p->tok->cur == p->tok->buf) {
408 col_offset = 0;
409 } else {
410 const char* start = p->tok->buf ? p->tok->line_start : p->tok->buf;
411 col_offset = Py_SAFE_DOWNCAST(p->tok->cur - start, intptr_t, int);
412 }
413 } else {
414 col_offset = t->col_offset + 1;
415 }
416
417 if (t->end_col_offset != -1) {
418 end_col_offset = t->end_col_offset + 1;
419 }
420
421 va_list va;
422 va_start(va, errmsg);
423 _PyPegen_raise_error_known_location(p, errtype, t->lineno, col_offset, t->end_lineno, end_col_offset, errmsg, va);
424 va_end(va);
425
426 return NULL;
427 }
428
429 static PyObject *
get_error_line(Parser * p,Py_ssize_t lineno)430 get_error_line(Parser *p, Py_ssize_t lineno)
431 {
432 /* If the file descriptor is interactive, the source lines of the current
433 * (multi-line) statement are stored in p->tok->interactive_src_start.
434 * If not, we're parsing from a string, which means that the whole source
435 * is stored in p->tok->str. */
436 assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp == stdin);
437
438 char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str;
439 assert(cur_line != NULL);
440
441 for (int i = 0; i < lineno - 1; i++) {
442 cur_line = strchr(cur_line, '\n') + 1;
443 }
444
445 char *next_newline;
446 if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line
447 next_newline = cur_line + strlen(cur_line);
448 }
449 return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
450 }
451
452 void *
_PyPegen_raise_error_known_location(Parser * p,PyObject * errtype,Py_ssize_t lineno,Py_ssize_t col_offset,Py_ssize_t end_lineno,Py_ssize_t end_col_offset,const char * errmsg,va_list va)453 _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
454 Py_ssize_t lineno, Py_ssize_t col_offset,
455 Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
456 const char *errmsg, va_list va)
457 {
458 PyObject *value = NULL;
459 PyObject *errstr = NULL;
460 PyObject *error_line = NULL;
461 PyObject *tmp = NULL;
462 p->error_indicator = 1;
463
464 if (end_lineno == CURRENT_POS) {
465 end_lineno = p->tok->lineno;
466 }
467 if (end_col_offset == CURRENT_POS) {
468 end_col_offset = p->tok->cur - p->tok->line_start;
469 }
470
471 if (p->start_rule == Py_fstring_input) {
472 const char *fstring_msg = "f-string: ";
473 Py_ssize_t len = strlen(fstring_msg) + strlen(errmsg);
474
475 char *new_errmsg = PyMem_Malloc(len + 1); // Lengths of both strings plus NULL character
476 if (!new_errmsg) {
477 return (void *) PyErr_NoMemory();
478 }
479
480 // Copy both strings into new buffer
481 memcpy(new_errmsg, fstring_msg, strlen(fstring_msg));
482 memcpy(new_errmsg + strlen(fstring_msg), errmsg, strlen(errmsg));
483 new_errmsg[len] = 0;
484 errmsg = new_errmsg;
485 }
486 errstr = PyUnicode_FromFormatV(errmsg, va);
487 if (!errstr) {
488 goto error;
489 }
490
491 if (p->tok->fp_interactive) {
492 error_line = get_error_line(p, lineno);
493 }
494 else if (p->start_rule == Py_file_input) {
495 error_line = _PyErr_ProgramDecodedTextObject(p->tok->filename,
496 (int) lineno, p->tok->encoding);
497 }
498
499 if (!error_line) {
500 /* PyErr_ProgramTextObject was not called or returned NULL. If it was not called,
501 then we need to find the error line from some other source, because
502 p->start_rule != Py_file_input. If it returned NULL, then it either unexpectedly
503 failed or we're parsing from a string or the REPL. There's a third edge case where
504 we're actually parsing from a file, which has an E_EOF SyntaxError and in that case
505 `PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which
506 does not physically exist */
507 assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);
508
509 if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) {
510 Py_ssize_t size = p->tok->inp - p->tok->buf;
511 error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace");
512 }
513 else if (p->tok->fp == NULL || p->tok->fp == stdin) {
514 error_line = get_error_line(p, lineno);
515 }
516 else {
517 error_line = PyUnicode_FromStringAndSize("", 0);
518 }
519 if (!error_line) {
520 goto error;
521 }
522 }
523
524 if (p->start_rule == Py_fstring_input) {
525 col_offset -= p->starting_col_offset;
526 end_col_offset -= p->starting_col_offset;
527 }
528
529 Py_ssize_t col_number = col_offset;
530 Py_ssize_t end_col_number = end_col_offset;
531
532 if (p->tok->encoding != NULL) {
533 col_number = byte_offset_to_character_offset(error_line, col_offset);
534 end_col_number = end_col_number > 0 ?
535 byte_offset_to_character_offset(error_line, end_col_offset) :
536 end_col_number;
537 }
538 tmp = Py_BuildValue("(OiiNii)", p->tok->filename, lineno, col_number, error_line, end_lineno, end_col_number);
539 if (!tmp) {
540 goto error;
541 }
542 value = PyTuple_Pack(2, errstr, tmp);
543 Py_DECREF(tmp);
544 if (!value) {
545 goto error;
546 }
547 PyErr_SetObject(errtype, value);
548
549 Py_DECREF(errstr);
550 Py_DECREF(value);
551 if (p->start_rule == Py_fstring_input) {
552 PyMem_Free((void *)errmsg);
553 }
554 return NULL;
555
556 error:
557 Py_XDECREF(errstr);
558 Py_XDECREF(error_line);
559 if (p->start_rule == Py_fstring_input) {
560 PyMem_Free((void *)errmsg);
561 }
562 return NULL;
563 }
564
565 #if 0
566 static const char *
567 token_name(int type)
568 {
569 if (0 <= type && type <= N_TOKENS) {
570 return _PyParser_TokenNames[type];
571 }
572 return "<Huh?>";
573 }
574 #endif
575
576 // Here, mark is the start of the node, while p->mark is the end.
577 // If node==NULL, they should be the same.
578 int
_PyPegen_insert_memo(Parser * p,int mark,int type,void * node)579 _PyPegen_insert_memo(Parser *p, int mark, int type, void *node)
580 {
581 // Insert in front
582 Memo *m = _PyArena_Malloc(p->arena, sizeof(Memo));
583 if (m == NULL) {
584 return -1;
585 }
586 m->type = type;
587 m->node = node;
588 m->mark = p->mark;
589 m->next = p->tokens[mark]->memo;
590 p->tokens[mark]->memo = m;
591 return 0;
592 }
593
594 // Like _PyPegen_insert_memo(), but updates an existing node if found.
595 int
_PyPegen_update_memo(Parser * p,int mark,int type,void * node)596 _PyPegen_update_memo(Parser *p, int mark, int type, void *node)
597 {
598 for (Memo *m = p->tokens[mark]->memo; m != NULL; m = m->next) {
599 if (m->type == type) {
600 // Update existing node.
601 m->node = node;
602 m->mark = p->mark;
603 return 0;
604 }
605 }
606 // Insert new node.
607 return _PyPegen_insert_memo(p, mark, type, node);
608 }
609
610 // Return dummy NAME.
611 void *
_PyPegen_dummy_name(Parser * p,...)612 _PyPegen_dummy_name(Parser *p, ...)
613 {
614 static void *cache = NULL;
615
616 if (cache != NULL) {
617 return cache;
618 }
619
620 PyObject *id = _create_dummy_identifier(p);
621 if (!id) {
622 return NULL;
623 }
624 cache = _PyAST_Name(id, Load, 1, 0, 1, 0, p->arena);
625 return cache;
626 }
627
628 static int
_get_keyword_or_name_type(Parser * p,const char * name,int name_len)629 _get_keyword_or_name_type(Parser *p, const char *name, int name_len)
630 {
631 assert(name_len > 0);
632 if (name_len >= p->n_keyword_lists ||
633 p->keywords[name_len] == NULL ||
634 p->keywords[name_len]->type == -1) {
635 return NAME;
636 }
637 for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {
638 if (strncmp(k->str, name, name_len) == 0) {
639 return k->type;
640 }
641 }
642 return NAME;
643 }
644
645 static int
growable_comment_array_init(growable_comment_array * arr,size_t initial_size)646 growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
647 assert(initial_size > 0);
648 arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items));
649 arr->size = initial_size;
650 arr->num_items = 0;
651
652 return arr->items != NULL;
653 }
654
655 static int
growable_comment_array_add(growable_comment_array * arr,int lineno,char * comment)656 growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
657 if (arr->num_items >= arr->size) {
658 size_t new_size = arr->size * 2;
659 void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items));
660 if (!new_items_array) {
661 return 0;
662 }
663 arr->items = new_items_array;
664 arr->size = new_size;
665 }
666
667 arr->items[arr->num_items].lineno = lineno;
668 arr->items[arr->num_items].comment = comment; // Take ownership
669 arr->num_items++;
670 return 1;
671 }
672
673 static void
growable_comment_array_deallocate(growable_comment_array * arr)674 growable_comment_array_deallocate(growable_comment_array *arr) {
675 for (unsigned i = 0; i < arr->num_items; i++) {
676 PyMem_Free(arr->items[i].comment);
677 }
678 PyMem_Free(arr->items);
679 }
680
681 static int
initialize_token(Parser * p,Token * token,const char * start,const char * end,int token_type)682 initialize_token(Parser *p, Token *token, const char *start, const char *end, int token_type) {
683 assert(token != NULL);
684
685 token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, start, (int)(end - start)) : token_type;
686 token->bytes = PyBytes_FromStringAndSize(start, end - start);
687 if (token->bytes == NULL) {
688 return -1;
689 }
690
691 if (_PyArena_AddPyObject(p->arena, token->bytes) < 0) {
692 Py_DECREF(token->bytes);
693 return -1;
694 }
695
696 token->level = p->tok->level;
697
698 const char *line_start = token_type == STRING ? p->tok->multi_line_start : p->tok->line_start;
699 int lineno = token_type == STRING ? p->tok->first_lineno : p->tok->lineno;
700 int end_lineno = p->tok->lineno;
701
702 int col_offset = (start != NULL && start >= line_start) ? (int)(start - line_start) : -1;
703 int end_col_offset = (end != NULL && end >= p->tok->line_start) ? (int)(end - p->tok->line_start) : -1;
704
705 token->lineno = lineno;
706 token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + col_offset : col_offset;
707 token->end_lineno = end_lineno;
708 token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + end_col_offset : end_col_offset;
709
710 p->fill += 1;
711
712 if (token_type == ERRORTOKEN && p->tok->done == E_DECODE) {
713 return raise_decode_error(p);
714 }
715
716 return (token_type == ERRORTOKEN ? tokenizer_error(p) : 0);
717 }
718
719 static int
_resize_tokens_array(Parser * p)720 _resize_tokens_array(Parser *p) {
721 int newsize = p->size * 2;
722 Token **new_tokens = PyMem_Realloc(p->tokens, newsize * sizeof(Token *));
723 if (new_tokens == NULL) {
724 PyErr_NoMemory();
725 return -1;
726 }
727 p->tokens = new_tokens;
728
729 for (int i = p->size; i < newsize; i++) {
730 p->tokens[i] = PyMem_Calloc(1, sizeof(Token));
731 if (p->tokens[i] == NULL) {
732 p->size = i; // Needed, in order to cleanup correctly after parser fails
733 PyErr_NoMemory();
734 return -1;
735 }
736 }
737 p->size = newsize;
738 return 0;
739 }
740
741 int
_PyPegen_fill_token(Parser * p)742 _PyPegen_fill_token(Parser *p)
743 {
744 const char *start;
745 const char *end;
746 int type = PyTokenizer_Get(p->tok, &start, &end);
747
748 // Record and skip '# type: ignore' comments
749 while (type == TYPE_IGNORE) {
750 Py_ssize_t len = end - start;
751 char *tag = PyMem_Malloc(len + 1);
752 if (tag == NULL) {
753 PyErr_NoMemory();
754 return -1;
755 }
756 strncpy(tag, start, len);
757 tag[len] = '\0';
758 // Ownership of tag passes to the growable array
759 if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
760 PyErr_NoMemory();
761 return -1;
762 }
763 type = PyTokenizer_Get(p->tok, &start, &end);
764 }
765
766 // If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
767 if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) {
768 type = NEWLINE; /* Add an extra newline */
769 p->parsing_started = 0;
770
771 if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
772 p->tok->pendin = -p->tok->indent;
773 p->tok->indent = 0;
774 }
775 }
776 else {
777 p->parsing_started = 1;
778 }
779
780 // Check if we are at the limit of the token array capacity and resize if needed
781 if ((p->fill == p->size) && (_resize_tokens_array(p) != 0)) {
782 return -1;
783 }
784
785 Token *t = p->tokens[p->fill];
786 return initialize_token(p, t, start, end, type);
787 }
788
789
790 #if defined(Py_DEBUG)
791 // Instrumentation to count the effectiveness of memoization.
792 // The array counts the number of tokens skipped by memoization,
793 // indexed by type.
794
795 #define NSTATISTICS 2000
796 static long memo_statistics[NSTATISTICS];
797
798 void
_PyPegen_clear_memo_statistics()799 _PyPegen_clear_memo_statistics()
800 {
801 for (int i = 0; i < NSTATISTICS; i++) {
802 memo_statistics[i] = 0;
803 }
804 }
805
806 PyObject *
_PyPegen_get_memo_statistics()807 _PyPegen_get_memo_statistics()
808 {
809 PyObject *ret = PyList_New(NSTATISTICS);
810 if (ret == NULL) {
811 return NULL;
812 }
813 for (int i = 0; i < NSTATISTICS; i++) {
814 PyObject *value = PyLong_FromLong(memo_statistics[i]);
815 if (value == NULL) {
816 Py_DECREF(ret);
817 return NULL;
818 }
819 // PyList_SetItem borrows a reference to value.
820 if (PyList_SetItem(ret, i, value) < 0) {
821 Py_DECREF(ret);
822 return NULL;
823 }
824 }
825 return ret;
826 }
827 #endif
828
829 int // bool
_PyPegen_is_memoized(Parser * p,int type,void * pres)830 _PyPegen_is_memoized(Parser *p, int type, void *pres)
831 {
832 if (p->mark == p->fill) {
833 if (_PyPegen_fill_token(p) < 0) {
834 p->error_indicator = 1;
835 return -1;
836 }
837 }
838
839 Token *t = p->tokens[p->mark];
840
841 for (Memo *m = t->memo; m != NULL; m = m->next) {
842 if (m->type == type) {
843 #if defined(PY_DEBUG)
844 if (0 <= type && type < NSTATISTICS) {
845 long count = m->mark - p->mark;
846 // A memoized negative result counts for one.
847 if (count <= 0) {
848 count = 1;
849 }
850 memo_statistics[type] += count;
851 }
852 #endif
853 p->mark = m->mark;
854 *(void **)(pres) = m->node;
855 return 1;
856 }
857 }
858 return 0;
859 }
860
861 int
_PyPegen_lookahead_with_name(int positive,expr_ty (func)(Parser *),Parser * p)862 _PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p)
863 {
864 int mark = p->mark;
865 void *res = func(p);
866 p->mark = mark;
867 return (res != NULL) == positive;
868 }
869
870 int
_PyPegen_lookahead_with_string(int positive,expr_ty (func)(Parser *,const char *),Parser * p,const char * arg)871 _PyPegen_lookahead_with_string(int positive, expr_ty (func)(Parser *, const char*), Parser *p, const char* arg)
872 {
873 int mark = p->mark;
874 void *res = func(p, arg);
875 p->mark = mark;
876 return (res != NULL) == positive;
877 }
878
879 int
_PyPegen_lookahead_with_int(int positive,Token * (func)(Parser *,int),Parser * p,int arg)880 _PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg)
881 {
882 int mark = p->mark;
883 void *res = func(p, arg);
884 p->mark = mark;
885 return (res != NULL) == positive;
886 }
887
888 int
_PyPegen_lookahead(int positive,void * (func)(Parser *),Parser * p)889 _PyPegen_lookahead(int positive, void *(func)(Parser *), Parser *p)
890 {
891 int mark = p->mark;
892 void *res = (void*)func(p);
893 p->mark = mark;
894 return (res != NULL) == positive;
895 }
896
897 Token *
_PyPegen_expect_token(Parser * p,int type)898 _PyPegen_expect_token(Parser *p, int type)
899 {
900 if (p->mark == p->fill) {
901 if (_PyPegen_fill_token(p) < 0) {
902 p->error_indicator = 1;
903 return NULL;
904 }
905 }
906 Token *t = p->tokens[p->mark];
907 if (t->type != type) {
908 return NULL;
909 }
910 p->mark += 1;
911 return t;
912 }
913
914 Token *
_PyPegen_expect_forced_token(Parser * p,int type,const char * expected)915 _PyPegen_expect_forced_token(Parser *p, int type, const char* expected) {
916
917 if (p->error_indicator == 1) {
918 return NULL;
919 }
920
921 if (p->mark == p->fill) {
922 if (_PyPegen_fill_token(p) < 0) {
923 p->error_indicator = 1;
924 return NULL;
925 }
926 }
927 Token *t = p->tokens[p->mark];
928 if (t->type != type) {
929 RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "expected '%s'", expected);
930 return NULL;
931 }
932 p->mark += 1;
933 return t;
934 }
935
936 expr_ty
_PyPegen_expect_soft_keyword(Parser * p,const char * keyword)937 _PyPegen_expect_soft_keyword(Parser *p, const char *keyword)
938 {
939 if (p->mark == p->fill) {
940 if (_PyPegen_fill_token(p) < 0) {
941 p->error_indicator = 1;
942 return NULL;
943 }
944 }
945 Token *t = p->tokens[p->mark];
946 if (t->type != NAME) {
947 return NULL;
948 }
949 const char *s = PyBytes_AsString(t->bytes);
950 if (!s) {
951 p->error_indicator = 1;
952 return NULL;
953 }
954 if (strcmp(s, keyword) != 0) {
955 return NULL;
956 }
957 return _PyPegen_name_token(p);
958 }
959
960 Token *
_PyPegen_get_last_nonnwhitespace_token(Parser * p)961 _PyPegen_get_last_nonnwhitespace_token(Parser *p)
962 {
963 assert(p->mark >= 0);
964 Token *token = NULL;
965 for (int m = p->mark - 1; m >= 0; m--) {
966 token = p->tokens[m];
967 if (token->type != ENDMARKER && (token->type < NEWLINE || token->type > DEDENT)) {
968 break;
969 }
970 }
971 return token;
972 }
973
974 static expr_ty
_PyPegen_name_from_token(Parser * p,Token * t)975 _PyPegen_name_from_token(Parser *p, Token* t)
976 {
977 if (t == NULL) {
978 return NULL;
979 }
980 const char *s = PyBytes_AsString(t->bytes);
981 if (!s) {
982 p->error_indicator = 1;
983 return NULL;
984 }
985 PyObject *id = _PyPegen_new_identifier(p, s);
986 if (id == NULL) {
987 p->error_indicator = 1;
988 return NULL;
989 }
990 return _PyAST_Name(id, Load, t->lineno, t->col_offset, t->end_lineno,
991 t->end_col_offset, p->arena);
992 }
993
994
995 expr_ty
_PyPegen_name_token(Parser * p)996 _PyPegen_name_token(Parser *p)
997 {
998 Token *t = _PyPegen_expect_token(p, NAME);
999 return _PyPegen_name_from_token(p, t);
1000 }
1001
1002 void *
_PyPegen_string_token(Parser * p)1003 _PyPegen_string_token(Parser *p)
1004 {
1005 return _PyPegen_expect_token(p, STRING);
1006 }
1007
1008
_PyPegen_soft_keyword_token(Parser * p)1009 expr_ty _PyPegen_soft_keyword_token(Parser *p) {
1010 Token *t = _PyPegen_expect_token(p, NAME);
1011 if (t == NULL) {
1012 return NULL;
1013 }
1014 char *the_token;
1015 Py_ssize_t size;
1016 PyBytes_AsStringAndSize(t->bytes, &the_token, &size);
1017 for (char **keyword = p->soft_keywords; *keyword != NULL; keyword++) {
1018 if (strncmp(*keyword, the_token, size) == 0) {
1019 return _PyPegen_name_from_token(p, t);
1020 }
1021 }
1022 return NULL;
1023 }
1024
1025 static PyObject *
parsenumber_raw(const char * s)1026 parsenumber_raw(const char *s)
1027 {
1028 const char *end;
1029 long x;
1030 double dx;
1031 Py_complex compl;
1032 int imflag;
1033
1034 assert(s != NULL);
1035 errno = 0;
1036 end = s + strlen(s) - 1;
1037 imflag = *end == 'j' || *end == 'J';
1038 if (s[0] == '0') {
1039 x = (long)PyOS_strtoul(s, (char **)&end, 0);
1040 if (x < 0 && errno == 0) {
1041 return PyLong_FromString(s, (char **)0, 0);
1042 }
1043 }
1044 else {
1045 x = PyOS_strtol(s, (char **)&end, 0);
1046 }
1047 if (*end == '\0') {
1048 if (errno != 0) {
1049 return PyLong_FromString(s, (char **)0, 0);
1050 }
1051 return PyLong_FromLong(x);
1052 }
1053 /* XXX Huge floats may silently fail */
1054 if (imflag) {
1055 compl.real = 0.;
1056 compl.imag = PyOS_string_to_double(s, (char **)&end, NULL);
1057 if (compl.imag == -1.0 && PyErr_Occurred()) {
1058 return NULL;
1059 }
1060 return PyComplex_FromCComplex(compl);
1061 }
1062 dx = PyOS_string_to_double(s, NULL, NULL);
1063 if (dx == -1.0 && PyErr_Occurred()) {
1064 return NULL;
1065 }
1066 return PyFloat_FromDouble(dx);
1067 }
1068
1069 static PyObject *
parsenumber(const char * s)1070 parsenumber(const char *s)
1071 {
1072 char *dup;
1073 char *end;
1074 PyObject *res = NULL;
1075
1076 assert(s != NULL);
1077
1078 if (strchr(s, '_') == NULL) {
1079 return parsenumber_raw(s);
1080 }
1081 /* Create a duplicate without underscores. */
1082 dup = PyMem_Malloc(strlen(s) + 1);
1083 if (dup == NULL) {
1084 return PyErr_NoMemory();
1085 }
1086 end = dup;
1087 for (; *s; s++) {
1088 if (*s != '_') {
1089 *end++ = *s;
1090 }
1091 }
1092 *end = '\0';
1093 res = parsenumber_raw(dup);
1094 PyMem_Free(dup);
1095 return res;
1096 }
1097
1098 expr_ty
_PyPegen_number_token(Parser * p)1099 _PyPegen_number_token(Parser *p)
1100 {
1101 Token *t = _PyPegen_expect_token(p, NUMBER);
1102 if (t == NULL) {
1103 return NULL;
1104 }
1105
1106 const char *num_raw = PyBytes_AsString(t->bytes);
1107 if (num_raw == NULL) {
1108 p->error_indicator = 1;
1109 return NULL;
1110 }
1111
1112 if (p->feature_version < 6 && strchr(num_raw, '_') != NULL) {
1113 p->error_indicator = 1;
1114 return RAISE_SYNTAX_ERROR("Underscores in numeric literals are only supported "
1115 "in Python 3.6 and greater");
1116 }
1117
1118 PyObject *c = parsenumber(num_raw);
1119
1120 if (c == NULL) {
1121 p->error_indicator = 1;
1122 PyThreadState *tstate = _PyThreadState_GET();
1123 // The only way a ValueError should happen in _this_ code is via
1124 // PyLong_FromString hitting a length limit.
1125 if (tstate->curexc_type == PyExc_ValueError &&
1126 tstate->curexc_value != NULL) {
1127 PyObject *type, *value, *tb;
1128 // This acts as PyErr_Clear() as we're replacing curexc.
1129 PyErr_Fetch(&type, &value, &tb);
1130 Py_XDECREF(tb);
1131 Py_DECREF(type);
1132 /* Intentionally omitting columns to avoid a wall of 1000s of '^'s
1133 * on the error message. Nobody is going to overlook their huge
1134 * numeric literal once given the line. */
1135 RAISE_ERROR_KNOWN_LOCATION(
1136 p, PyExc_SyntaxError,
1137 t->lineno, -1 /* col_offset */,
1138 t->end_lineno, -1 /* end_col_offset */,
1139 "%S - Consider hexadecimal for huge integer literals "
1140 "to avoid decimal conversion limits.",
1141 value);
1142 Py_DECREF(value);
1143 }
1144 return NULL;
1145 }
1146
1147 if (_PyArena_AddPyObject(p->arena, c) < 0) {
1148 Py_DECREF(c);
1149 p->error_indicator = 1;
1150 return NULL;
1151 }
1152
1153 return _PyAST_Constant(c, NULL, t->lineno, t->col_offset, t->end_lineno,
1154 t->end_col_offset, p->arena);
1155 }
1156
1157 /* Check that the source for a single input statement really is a single
1158 statement by looking at what is left in the buffer after parsing.
1159 Trailing whitespace and comments are OK. */
1160 static int // bool
bad_single_statement(Parser * p)1161 bad_single_statement(Parser *p)
1162 {
1163 char *cur = p->tok->cur;
1164 char c = *cur;
1165
1166 for (;;) {
1167 while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
1168 c = *++cur;
1169 }
1170
1171 if (!c) {
1172 return 0;
1173 }
1174
1175 if (c != '#') {
1176 return 1;
1177 }
1178
1179 /* Suck up comment. */
1180 while (c && c != '\n') {
1181 c = *++cur;
1182 }
1183 }
1184 }
1185
1186 void
_PyPegen_Parser_Free(Parser * p)1187 _PyPegen_Parser_Free(Parser *p)
1188 {
1189 Py_XDECREF(p->normalize);
1190 for (int i = 0; i < p->size; i++) {
1191 PyMem_Free(p->tokens[i]);
1192 }
1193 PyMem_Free(p->tokens);
1194 growable_comment_array_deallocate(&p->type_ignore_comments);
1195 PyMem_Free(p);
1196 }
1197
1198 static int
compute_parser_flags(PyCompilerFlags * flags)1199 compute_parser_flags(PyCompilerFlags *flags)
1200 {
1201 int parser_flags = 0;
1202 if (!flags) {
1203 return 0;
1204 }
1205 if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT) {
1206 parser_flags |= PyPARSE_DONT_IMPLY_DEDENT;
1207 }
1208 if (flags->cf_flags & PyCF_IGNORE_COOKIE) {
1209 parser_flags |= PyPARSE_IGNORE_COOKIE;
1210 }
1211 if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) {
1212 parser_flags |= PyPARSE_BARRY_AS_BDFL;
1213 }
1214 if (flags->cf_flags & PyCF_TYPE_COMMENTS) {
1215 parser_flags |= PyPARSE_TYPE_COMMENTS;
1216 }
1217 if ((flags->cf_flags & PyCF_ONLY_AST) && flags->cf_feature_version < 7) {
1218 parser_flags |= PyPARSE_ASYNC_HACKS;
1219 }
1220 return parser_flags;
1221 }
1222
1223 Parser *
_PyPegen_Parser_New(struct tok_state * tok,int start_rule,int flags,int feature_version,int * errcode,PyArena * arena)1224 _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
1225 int feature_version, int *errcode, PyArena *arena)
1226 {
1227 Parser *p = PyMem_Malloc(sizeof(Parser));
1228 if (p == NULL) {
1229 return (Parser *) PyErr_NoMemory();
1230 }
1231 assert(tok != NULL);
1232 tok->type_comments = (flags & PyPARSE_TYPE_COMMENTS) > 0;
1233 tok->async_hacks = (flags & PyPARSE_ASYNC_HACKS) > 0;
1234 p->tok = tok;
1235 p->keywords = NULL;
1236 p->n_keyword_lists = -1;
1237 p->soft_keywords = NULL;
1238 p->tokens = PyMem_Malloc(sizeof(Token *));
1239 if (!p->tokens) {
1240 PyMem_Free(p);
1241 return (Parser *) PyErr_NoMemory();
1242 }
1243 p->tokens[0] = PyMem_Calloc(1, sizeof(Token));
1244 if (!p->tokens) {
1245 PyMem_Free(p->tokens);
1246 PyMem_Free(p);
1247 return (Parser *) PyErr_NoMemory();
1248 }
1249 if (!growable_comment_array_init(&p->type_ignore_comments, 10)) {
1250 PyMem_Free(p->tokens[0]);
1251 PyMem_Free(p->tokens);
1252 PyMem_Free(p);
1253 return (Parser *) PyErr_NoMemory();
1254 }
1255
1256 p->mark = 0;
1257 p->fill = 0;
1258 p->size = 1;
1259
1260 p->errcode = errcode;
1261 p->arena = arena;
1262 p->start_rule = start_rule;
1263 p->parsing_started = 0;
1264 p->normalize = NULL;
1265 p->error_indicator = 0;
1266
1267 p->starting_lineno = 0;
1268 p->starting_col_offset = 0;
1269 p->flags = flags;
1270 p->feature_version = feature_version;
1271 p->known_err_token = NULL;
1272 p->level = 0;
1273 p->call_invalid_rules = 0;
1274 p->in_raw_rule = 0;
1275 return p;
1276 }
1277
1278 static void
reset_parser_state(Parser * p)1279 reset_parser_state(Parser *p)
1280 {
1281 for (int i = 0; i < p->fill; i++) {
1282 p->tokens[i]->memo = NULL;
1283 }
1284 p->mark = 0;
1285 p->call_invalid_rules = 1;
1286 // Don't try to get extra tokens in interactive mode when trying to
1287 // raise specialized errors in the second pass.
1288 p->tok->interactive_underflow = IUNDERFLOW_STOP;
1289 }
1290
1291 static int
_PyPegen_check_tokenizer_errors(Parser * p)1292 _PyPegen_check_tokenizer_errors(Parser *p) {
1293 // Tokenize the whole input to see if there are any tokenization
1294 // errors such as mistmatching parentheses. These will get priority
1295 // over generic syntax errors only if the line number of the error is
1296 // before the one that we had for the generic error.
1297
1298 // We don't want to tokenize to the end for interactive input
1299 if (p->tok->prompt != NULL) {
1300 return 0;
1301 }
1302
1303 PyObject *type, *value, *traceback;
1304 PyErr_Fetch(&type, &value, &traceback);
1305
1306 Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
1307 Py_ssize_t current_err_line = current_token->lineno;
1308
1309 int ret = 0;
1310
1311 for (;;) {
1312 const char *start;
1313 const char *end;
1314 switch (PyTokenizer_Get(p->tok, &start, &end)) {
1315 case ERRORTOKEN:
1316 if (p->tok->level != 0) {
1317 int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
1318 if (current_err_line > error_lineno) {
1319 raise_unclosed_parentheses_error(p);
1320 ret = -1;
1321 goto exit;
1322 }
1323 }
1324 break;
1325 case ENDMARKER:
1326 break;
1327 default:
1328 continue;
1329 }
1330 break;
1331 }
1332
1333
1334 exit:
1335 if (PyErr_Occurred()) {
1336 Py_XDECREF(value);
1337 Py_XDECREF(type);
1338 Py_XDECREF(traceback);
1339 } else {
1340 PyErr_Restore(type, value, traceback);
1341 }
1342 return ret;
1343 }
1344
1345 void *
_PyPegen_run_parser(Parser * p)1346 _PyPegen_run_parser(Parser *p)
1347 {
1348 void *res = _PyPegen_parse(p);
1349 assert(p->level == 0);
1350 if (res == NULL) {
1351 if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) {
1352 return NULL;
1353 }
1354 Token *last_token = p->tokens[p->fill - 1];
1355 reset_parser_state(p);
1356 _PyPegen_parse(p);
1357 if (PyErr_Occurred()) {
1358 // Prioritize tokenizer errors to custom syntax errors raised
1359 // on the second phase only if the errors come from the parser.
1360 if (p->tok->done == E_DONE && PyErr_ExceptionMatches(PyExc_SyntaxError)) {
1361 _PyPegen_check_tokenizer_errors(p);
1362 }
1363 return NULL;
1364 }
1365 if (p->fill == 0) {
1366 RAISE_SYNTAX_ERROR("error at start before reading any input");
1367 }
1368 else if (last_token->type == ERRORTOKEN && p->tok->done == E_EOF) {
1369 if (p->tok->level) {
1370 raise_unclosed_parentheses_error(p);
1371 } else {
1372 RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
1373 }
1374 }
1375 else {
1376 if (p->tokens[p->fill-1]->type == INDENT) {
1377 RAISE_INDENTATION_ERROR("unexpected indent");
1378 }
1379 else if (p->tokens[p->fill-1]->type == DEDENT) {
1380 RAISE_INDENTATION_ERROR("unexpected unindent");
1381 }
1382 else {
1383 // Use the last token we found on the first pass to avoid reporting
1384 // incorrect locations for generic syntax errors just because we reached
1385 // further away when trying to find specific syntax errors in the second
1386 // pass.
1387 RAISE_SYNTAX_ERROR_KNOWN_LOCATION(last_token, "invalid syntax");
1388 // _PyPegen_check_tokenizer_errors will override the existing
1389 // generic SyntaxError we just raised if errors are found.
1390 _PyPegen_check_tokenizer_errors(p);
1391 }
1392 }
1393 return NULL;
1394 }
1395
1396 if (p->start_rule == Py_single_input && bad_single_statement(p)) {
1397 p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
1398 return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
1399 }
1400
1401 // test_peg_generator defines _Py_TEST_PEGEN to not call PyAST_Validate()
1402 #if defined(Py_DEBUG) && !defined(_Py_TEST_PEGEN)
1403 if (p->start_rule == Py_single_input ||
1404 p->start_rule == Py_file_input ||
1405 p->start_rule == Py_eval_input)
1406 {
1407 if (!_PyAST_Validate(res)) {
1408 return NULL;
1409 }
1410 }
1411 #endif
1412 return res;
1413 }
1414
1415 mod_ty
_PyPegen_run_parser_from_file_pointer(FILE * fp,int start_rule,PyObject * filename_ob,const char * enc,const char * ps1,const char * ps2,PyCompilerFlags * flags,int * errcode,PyArena * arena)1416 _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob,
1417 const char *enc, const char *ps1, const char *ps2,
1418 PyCompilerFlags *flags, int *errcode, PyArena *arena)
1419 {
1420 struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2);
1421 if (tok == NULL) {
1422 if (PyErr_Occurred()) {
1423 raise_tokenizer_init_error(filename_ob);
1424 return NULL;
1425 }
1426 return NULL;
1427 }
1428 if (!tok->fp || ps1 != NULL || ps2 != NULL ||
1429 PyUnicode_CompareWithASCIIString(filename_ob, "<stdin>") == 0) {
1430 tok->fp_interactive = 1;
1431 }
1432 // This transfers the ownership to the tokenizer
1433 tok->filename = filename_ob;
1434 Py_INCREF(filename_ob);
1435
1436 // From here on we need to clean up even if there's an error
1437 mod_ty result = NULL;
1438
1439 int parser_flags = compute_parser_flags(flags);
1440 Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
1441 errcode, arena);
1442 if (p == NULL) {
1443 goto error;
1444 }
1445
1446 result = _PyPegen_run_parser(p);
1447 _PyPegen_Parser_Free(p);
1448
1449 error:
1450 PyTokenizer_Free(tok);
1451 return result;
1452 }
1453
1454 mod_ty
_PyPegen_run_parser_from_string(const char * str,int start_rule,PyObject * filename_ob,PyCompilerFlags * flags,PyArena * arena)1455 _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob,
1456 PyCompilerFlags *flags, PyArena *arena)
1457 {
1458 int exec_input = start_rule == Py_file_input;
1459
1460 struct tok_state *tok;
1461 if (flags != NULL && flags->cf_flags & PyCF_IGNORE_COOKIE) {
1462 tok = PyTokenizer_FromUTF8(str, exec_input);
1463 } else {
1464 tok = PyTokenizer_FromString(str, exec_input);
1465 }
1466 if (tok == NULL) {
1467 if (PyErr_Occurred()) {
1468 raise_tokenizer_init_error(filename_ob);
1469 }
1470 return NULL;
1471 }
1472 // This transfers the ownership to the tokenizer
1473 tok->filename = filename_ob;
1474 Py_INCREF(filename_ob);
1475
1476 // We need to clear up from here on
1477 mod_ty result = NULL;
1478
1479 int parser_flags = compute_parser_flags(flags);
1480 int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ?
1481 flags->cf_feature_version : PY_MINOR_VERSION;
1482 Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
1483 NULL, arena);
1484 if (p == NULL) {
1485 goto error;
1486 }
1487
1488 result = _PyPegen_run_parser(p);
1489 _PyPegen_Parser_Free(p);
1490
1491 error:
1492 PyTokenizer_Free(tok);
1493 return result;
1494 }
1495
1496 asdl_stmt_seq*
_PyPegen_interactive_exit(Parser * p)1497 _PyPegen_interactive_exit(Parser *p)
1498 {
1499 if (p->errcode) {
1500 *(p->errcode) = E_EOF;
1501 }
1502 return NULL;
1503 }
1504
1505 /* Creates a single-element asdl_seq* that contains a */
1506 asdl_seq *
_PyPegen_singleton_seq(Parser * p,void * a)1507 _PyPegen_singleton_seq(Parser *p, void *a)
1508 {
1509 assert(a != NULL);
1510 asdl_seq *seq = (asdl_seq*)_Py_asdl_generic_seq_new(1, p->arena);
1511 if (!seq) {
1512 return NULL;
1513 }
1514 asdl_seq_SET_UNTYPED(seq, 0, a);
1515 return seq;
1516 }
1517
1518 /* Creates a copy of seq and prepends a to it */
1519 asdl_seq *
_PyPegen_seq_insert_in_front(Parser * p,void * a,asdl_seq * seq)1520 _PyPegen_seq_insert_in_front(Parser *p, void *a, asdl_seq *seq)
1521 {
1522 assert(a != NULL);
1523 if (!seq) {
1524 return _PyPegen_singleton_seq(p, a);
1525 }
1526
1527 asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(asdl_seq_LEN(seq) + 1, p->arena);
1528 if (!new_seq) {
1529 return NULL;
1530 }
1531
1532 asdl_seq_SET_UNTYPED(new_seq, 0, a);
1533 for (Py_ssize_t i = 1, l = asdl_seq_LEN(new_seq); i < l; i++) {
1534 asdl_seq_SET_UNTYPED(new_seq, i, asdl_seq_GET_UNTYPED(seq, i - 1));
1535 }
1536 return new_seq;
1537 }
1538
1539 /* Creates a copy of seq and appends a to it */
1540 asdl_seq *
_PyPegen_seq_append_to_end(Parser * p,asdl_seq * seq,void * a)1541 _PyPegen_seq_append_to_end(Parser *p, asdl_seq *seq, void *a)
1542 {
1543 assert(a != NULL);
1544 if (!seq) {
1545 return _PyPegen_singleton_seq(p, a);
1546 }
1547
1548 asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(asdl_seq_LEN(seq) + 1, p->arena);
1549 if (!new_seq) {
1550 return NULL;
1551 }
1552
1553 for (Py_ssize_t i = 0, l = asdl_seq_LEN(new_seq); i + 1 < l; i++) {
1554 asdl_seq_SET_UNTYPED(new_seq, i, asdl_seq_GET_UNTYPED(seq, i));
1555 }
1556 asdl_seq_SET_UNTYPED(new_seq, asdl_seq_LEN(new_seq) - 1, a);
1557 return new_seq;
1558 }
1559
1560 static Py_ssize_t
_get_flattened_seq_size(asdl_seq * seqs)1561 _get_flattened_seq_size(asdl_seq *seqs)
1562 {
1563 Py_ssize_t size = 0;
1564 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) {
1565 asdl_seq *inner_seq = asdl_seq_GET_UNTYPED(seqs, i);
1566 size += asdl_seq_LEN(inner_seq);
1567 }
1568 return size;
1569 }
1570
1571 /* Flattens an asdl_seq* of asdl_seq*s */
1572 asdl_seq *
_PyPegen_seq_flatten(Parser * p,asdl_seq * seqs)1573 _PyPegen_seq_flatten(Parser *p, asdl_seq *seqs)
1574 {
1575 Py_ssize_t flattened_seq_size = _get_flattened_seq_size(seqs);
1576 assert(flattened_seq_size > 0);
1577
1578 asdl_seq *flattened_seq = (asdl_seq*)_Py_asdl_generic_seq_new(flattened_seq_size, p->arena);
1579 if (!flattened_seq) {
1580 return NULL;
1581 }
1582
1583 int flattened_seq_idx = 0;
1584 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) {
1585 asdl_seq *inner_seq = asdl_seq_GET_UNTYPED(seqs, i);
1586 for (Py_ssize_t j = 0, li = asdl_seq_LEN(inner_seq); j < li; j++) {
1587 asdl_seq_SET_UNTYPED(flattened_seq, flattened_seq_idx++, asdl_seq_GET_UNTYPED(inner_seq, j));
1588 }
1589 }
1590 assert(flattened_seq_idx == flattened_seq_size);
1591
1592 return flattened_seq;
1593 }
1594
1595 void *
_PyPegen_seq_last_item(asdl_seq * seq)1596 _PyPegen_seq_last_item(asdl_seq *seq)
1597 {
1598 Py_ssize_t len = asdl_seq_LEN(seq);
1599 return asdl_seq_GET_UNTYPED(seq, len - 1);
1600 }
1601
1602 void *
_PyPegen_seq_first_item(asdl_seq * seq)1603 _PyPegen_seq_first_item(asdl_seq *seq)
1604 {
1605 return asdl_seq_GET_UNTYPED(seq, 0);
1606 }
1607
1608
1609 /* Creates a new name of the form <first_name>.<second_name> */
1610 expr_ty
_PyPegen_join_names_with_dot(Parser * p,expr_ty first_name,expr_ty second_name)1611 _PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name)
1612 {
1613 assert(first_name != NULL && second_name != NULL);
1614 PyObject *first_identifier = first_name->v.Name.id;
1615 PyObject *second_identifier = second_name->v.Name.id;
1616
1617 if (PyUnicode_READY(first_identifier) == -1) {
1618 return NULL;
1619 }
1620 if (PyUnicode_READY(second_identifier) == -1) {
1621 return NULL;
1622 }
1623 const char *first_str = PyUnicode_AsUTF8(first_identifier);
1624 if (!first_str) {
1625 return NULL;
1626 }
1627 const char *second_str = PyUnicode_AsUTF8(second_identifier);
1628 if (!second_str) {
1629 return NULL;
1630 }
1631 Py_ssize_t len = strlen(first_str) + strlen(second_str) + 1; // +1 for the dot
1632
1633 PyObject *str = PyBytes_FromStringAndSize(NULL, len);
1634 if (!str) {
1635 return NULL;
1636 }
1637
1638 char *s = PyBytes_AS_STRING(str);
1639 if (!s) {
1640 return NULL;
1641 }
1642
1643 strcpy(s, first_str);
1644 s += strlen(first_str);
1645 *s++ = '.';
1646 strcpy(s, second_str);
1647 s += strlen(second_str);
1648 *s = '\0';
1649
1650 PyObject *uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str), PyBytes_GET_SIZE(str), NULL);
1651 Py_DECREF(str);
1652 if (!uni) {
1653 return NULL;
1654 }
1655 PyUnicode_InternInPlace(&uni);
1656 if (_PyArena_AddPyObject(p->arena, uni) < 0) {
1657 Py_DECREF(uni);
1658 return NULL;
1659 }
1660
1661 return _PyAST_Name(uni, Load, EXTRA_EXPR(first_name, second_name));
1662 }
1663
1664 /* Counts the total number of dots in seq's tokens */
1665 int
_PyPegen_seq_count_dots(asdl_seq * seq)1666 _PyPegen_seq_count_dots(asdl_seq *seq)
1667 {
1668 int number_of_dots = 0;
1669 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) {
1670 Token *current_expr = asdl_seq_GET_UNTYPED(seq, i);
1671 switch (current_expr->type) {
1672 case ELLIPSIS:
1673 number_of_dots += 3;
1674 break;
1675 case DOT:
1676 number_of_dots += 1;
1677 break;
1678 default:
1679 Py_UNREACHABLE();
1680 }
1681 }
1682
1683 return number_of_dots;
1684 }
1685
1686 /* Creates an alias with '*' as the identifier name */
1687 alias_ty
_PyPegen_alias_for_star(Parser * p,int lineno,int col_offset,int end_lineno,int end_col_offset,PyArena * arena)1688 _PyPegen_alias_for_star(Parser *p, int lineno, int col_offset, int end_lineno,
1689 int end_col_offset, PyArena *arena) {
1690 PyObject *str = PyUnicode_InternFromString("*");
1691 if (!str) {
1692 return NULL;
1693 }
1694 if (_PyArena_AddPyObject(p->arena, str) < 0) {
1695 Py_DECREF(str);
1696 return NULL;
1697 }
1698 return _PyAST_alias(str, NULL, lineno, col_offset, end_lineno, end_col_offset, arena);
1699 }
1700
1701 /* Creates a new asdl_seq* with the identifiers of all the names in seq */
1702 asdl_identifier_seq *
_PyPegen_map_names_to_ids(Parser * p,asdl_expr_seq * seq)1703 _PyPegen_map_names_to_ids(Parser *p, asdl_expr_seq *seq)
1704 {
1705 Py_ssize_t len = asdl_seq_LEN(seq);
1706 assert(len > 0);
1707
1708 asdl_identifier_seq *new_seq = _Py_asdl_identifier_seq_new(len, p->arena);
1709 if (!new_seq) {
1710 return NULL;
1711 }
1712 for (Py_ssize_t i = 0; i < len; i++) {
1713 expr_ty e = asdl_seq_GET(seq, i);
1714 asdl_seq_SET(new_seq, i, e->v.Name.id);
1715 }
1716 return new_seq;
1717 }
1718
1719 /* Constructs a CmpopExprPair */
1720 CmpopExprPair *
_PyPegen_cmpop_expr_pair(Parser * p,cmpop_ty cmpop,expr_ty expr)1721 _PyPegen_cmpop_expr_pair(Parser *p, cmpop_ty cmpop, expr_ty expr)
1722 {
1723 assert(expr != NULL);
1724 CmpopExprPair *a = _PyArena_Malloc(p->arena, sizeof(CmpopExprPair));
1725 if (!a) {
1726 return NULL;
1727 }
1728 a->cmpop = cmpop;
1729 a->expr = expr;
1730 return a;
1731 }
1732
1733 asdl_int_seq *
_PyPegen_get_cmpops(Parser * p,asdl_seq * seq)1734 _PyPegen_get_cmpops(Parser *p, asdl_seq *seq)
1735 {
1736 Py_ssize_t len = asdl_seq_LEN(seq);
1737 assert(len > 0);
1738
1739 asdl_int_seq *new_seq = _Py_asdl_int_seq_new(len, p->arena);
1740 if (!new_seq) {
1741 return NULL;
1742 }
1743 for (Py_ssize_t i = 0; i < len; i++) {
1744 CmpopExprPair *pair = asdl_seq_GET_UNTYPED(seq, i);
1745 asdl_seq_SET(new_seq, i, pair->cmpop);
1746 }
1747 return new_seq;
1748 }
1749
1750 asdl_expr_seq *
_PyPegen_get_exprs(Parser * p,asdl_seq * seq)1751 _PyPegen_get_exprs(Parser *p, asdl_seq *seq)
1752 {
1753 Py_ssize_t len = asdl_seq_LEN(seq);
1754 assert(len > 0);
1755
1756 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
1757 if (!new_seq) {
1758 return NULL;
1759 }
1760 for (Py_ssize_t i = 0; i < len; i++) {
1761 CmpopExprPair *pair = asdl_seq_GET_UNTYPED(seq, i);
1762 asdl_seq_SET(new_seq, i, pair->expr);
1763 }
1764 return new_seq;
1765 }
1766
1767 /* Creates an asdl_seq* where all the elements have been changed to have ctx as context */
1768 static asdl_expr_seq *
_set_seq_context(Parser * p,asdl_expr_seq * seq,expr_context_ty ctx)1769 _set_seq_context(Parser *p, asdl_expr_seq *seq, expr_context_ty ctx)
1770 {
1771 Py_ssize_t len = asdl_seq_LEN(seq);
1772 if (len == 0) {
1773 return NULL;
1774 }
1775
1776 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
1777 if (!new_seq) {
1778 return NULL;
1779 }
1780 for (Py_ssize_t i = 0; i < len; i++) {
1781 expr_ty e = asdl_seq_GET(seq, i);
1782 asdl_seq_SET(new_seq, i, _PyPegen_set_expr_context(p, e, ctx));
1783 }
1784 return new_seq;
1785 }
1786
1787 static expr_ty
_set_name_context(Parser * p,expr_ty e,expr_context_ty ctx)1788 _set_name_context(Parser *p, expr_ty e, expr_context_ty ctx)
1789 {
1790 return _PyAST_Name(e->v.Name.id, ctx, EXTRA_EXPR(e, e));
1791 }
1792
1793 static expr_ty
_set_tuple_context(Parser * p,expr_ty e,expr_context_ty ctx)1794 _set_tuple_context(Parser *p, expr_ty e, expr_context_ty ctx)
1795 {
1796 return _PyAST_Tuple(
1797 _set_seq_context(p, e->v.Tuple.elts, ctx),
1798 ctx,
1799 EXTRA_EXPR(e, e));
1800 }
1801
1802 static expr_ty
_set_list_context(Parser * p,expr_ty e,expr_context_ty ctx)1803 _set_list_context(Parser *p, expr_ty e, expr_context_ty ctx)
1804 {
1805 return _PyAST_List(
1806 _set_seq_context(p, e->v.List.elts, ctx),
1807 ctx,
1808 EXTRA_EXPR(e, e));
1809 }
1810
1811 static expr_ty
_set_subscript_context(Parser * p,expr_ty e,expr_context_ty ctx)1812 _set_subscript_context(Parser *p, expr_ty e, expr_context_ty ctx)
1813 {
1814 return _PyAST_Subscript(e->v.Subscript.value, e->v.Subscript.slice,
1815 ctx, EXTRA_EXPR(e, e));
1816 }
1817
1818 static expr_ty
_set_attribute_context(Parser * p,expr_ty e,expr_context_ty ctx)1819 _set_attribute_context(Parser *p, expr_ty e, expr_context_ty ctx)
1820 {
1821 return _PyAST_Attribute(e->v.Attribute.value, e->v.Attribute.attr,
1822 ctx, EXTRA_EXPR(e, e));
1823 }
1824
1825 static expr_ty
_set_starred_context(Parser * p,expr_ty e,expr_context_ty ctx)1826 _set_starred_context(Parser *p, expr_ty e, expr_context_ty ctx)
1827 {
1828 return _PyAST_Starred(_PyPegen_set_expr_context(p, e->v.Starred.value, ctx),
1829 ctx, EXTRA_EXPR(e, e));
1830 }
1831
1832 /* Creates an `expr_ty` equivalent to `expr` but with `ctx` as context */
1833 expr_ty
_PyPegen_set_expr_context(Parser * p,expr_ty expr,expr_context_ty ctx)1834 _PyPegen_set_expr_context(Parser *p, expr_ty expr, expr_context_ty ctx)
1835 {
1836 assert(expr != NULL);
1837
1838 expr_ty new = NULL;
1839 switch (expr->kind) {
1840 case Name_kind:
1841 new = _set_name_context(p, expr, ctx);
1842 break;
1843 case Tuple_kind:
1844 new = _set_tuple_context(p, expr, ctx);
1845 break;
1846 case List_kind:
1847 new = _set_list_context(p, expr, ctx);
1848 break;
1849 case Subscript_kind:
1850 new = _set_subscript_context(p, expr, ctx);
1851 break;
1852 case Attribute_kind:
1853 new = _set_attribute_context(p, expr, ctx);
1854 break;
1855 case Starred_kind:
1856 new = _set_starred_context(p, expr, ctx);
1857 break;
1858 default:
1859 new = expr;
1860 }
1861 return new;
1862 }
1863
1864 /* Constructs a KeyValuePair that is used when parsing a dict's key value pairs */
1865 KeyValuePair *
_PyPegen_key_value_pair(Parser * p,expr_ty key,expr_ty value)1866 _PyPegen_key_value_pair(Parser *p, expr_ty key, expr_ty value)
1867 {
1868 KeyValuePair *a = _PyArena_Malloc(p->arena, sizeof(KeyValuePair));
1869 if (!a) {
1870 return NULL;
1871 }
1872 a->key = key;
1873 a->value = value;
1874 return a;
1875 }
1876
1877 /* Extracts all keys from an asdl_seq* of KeyValuePair*'s */
1878 asdl_expr_seq *
_PyPegen_get_keys(Parser * p,asdl_seq * seq)1879 _PyPegen_get_keys(Parser *p, asdl_seq *seq)
1880 {
1881 Py_ssize_t len = asdl_seq_LEN(seq);
1882 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
1883 if (!new_seq) {
1884 return NULL;
1885 }
1886 for (Py_ssize_t i = 0; i < len; i++) {
1887 KeyValuePair *pair = asdl_seq_GET_UNTYPED(seq, i);
1888 asdl_seq_SET(new_seq, i, pair->key);
1889 }
1890 return new_seq;
1891 }
1892
1893 /* Extracts all values from an asdl_seq* of KeyValuePair*'s */
1894 asdl_expr_seq *
_PyPegen_get_values(Parser * p,asdl_seq * seq)1895 _PyPegen_get_values(Parser *p, asdl_seq *seq)
1896 {
1897 Py_ssize_t len = asdl_seq_LEN(seq);
1898 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
1899 if (!new_seq) {
1900 return NULL;
1901 }
1902 for (Py_ssize_t i = 0; i < len; i++) {
1903 KeyValuePair *pair = asdl_seq_GET_UNTYPED(seq, i);
1904 asdl_seq_SET(new_seq, i, pair->value);
1905 }
1906 return new_seq;
1907 }
1908
1909 /* Constructs a KeyPatternPair that is used when parsing mapping & class patterns */
1910 KeyPatternPair *
_PyPegen_key_pattern_pair(Parser * p,expr_ty key,pattern_ty pattern)1911 _PyPegen_key_pattern_pair(Parser *p, expr_ty key, pattern_ty pattern)
1912 {
1913 KeyPatternPair *a = _PyArena_Malloc(p->arena, sizeof(KeyPatternPair));
1914 if (!a) {
1915 return NULL;
1916 }
1917 a->key = key;
1918 a->pattern = pattern;
1919 return a;
1920 }
1921
1922 /* Extracts all keys from an asdl_seq* of KeyPatternPair*'s */
1923 asdl_expr_seq *
_PyPegen_get_pattern_keys(Parser * p,asdl_seq * seq)1924 _PyPegen_get_pattern_keys(Parser *p, asdl_seq *seq)
1925 {
1926 Py_ssize_t len = asdl_seq_LEN(seq);
1927 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
1928 if (!new_seq) {
1929 return NULL;
1930 }
1931 for (Py_ssize_t i = 0; i < len; i++) {
1932 KeyPatternPair *pair = asdl_seq_GET_UNTYPED(seq, i);
1933 asdl_seq_SET(new_seq, i, pair->key);
1934 }
1935 return new_seq;
1936 }
1937
1938 /* Extracts all patterns from an asdl_seq* of KeyPatternPair*'s */
1939 asdl_pattern_seq *
_PyPegen_get_patterns(Parser * p,asdl_seq * seq)1940 _PyPegen_get_patterns(Parser *p, asdl_seq *seq)
1941 {
1942 Py_ssize_t len = asdl_seq_LEN(seq);
1943 asdl_pattern_seq *new_seq = _Py_asdl_pattern_seq_new(len, p->arena);
1944 if (!new_seq) {
1945 return NULL;
1946 }
1947 for (Py_ssize_t i = 0; i < len; i++) {
1948 KeyPatternPair *pair = asdl_seq_GET_UNTYPED(seq, i);
1949 asdl_seq_SET(new_seq, i, pair->pattern);
1950 }
1951 return new_seq;
1952 }
1953
1954 /* Constructs a NameDefaultPair */
1955 NameDefaultPair *
_PyPegen_name_default_pair(Parser * p,arg_ty arg,expr_ty value,Token * tc)1956 _PyPegen_name_default_pair(Parser *p, arg_ty arg, expr_ty value, Token *tc)
1957 {
1958 NameDefaultPair *a = _PyArena_Malloc(p->arena, sizeof(NameDefaultPair));
1959 if (!a) {
1960 return NULL;
1961 }
1962 a->arg = _PyPegen_add_type_comment_to_arg(p, arg, tc);
1963 a->value = value;
1964 return a;
1965 }
1966
1967 /* Constructs a SlashWithDefault */
1968 SlashWithDefault *
_PyPegen_slash_with_default(Parser * p,asdl_arg_seq * plain_names,asdl_seq * names_with_defaults)1969 _PyPegen_slash_with_default(Parser *p, asdl_arg_seq *plain_names, asdl_seq *names_with_defaults)
1970 {
1971 SlashWithDefault *a = _PyArena_Malloc(p->arena, sizeof(SlashWithDefault));
1972 if (!a) {
1973 return NULL;
1974 }
1975 a->plain_names = plain_names;
1976 a->names_with_defaults = names_with_defaults;
1977 return a;
1978 }
1979
1980 /* Constructs a StarEtc */
1981 StarEtc *
_PyPegen_star_etc(Parser * p,arg_ty vararg,asdl_seq * kwonlyargs,arg_ty kwarg)1982 _PyPegen_star_etc(Parser *p, arg_ty vararg, asdl_seq *kwonlyargs, arg_ty kwarg)
1983 {
1984 StarEtc *a = _PyArena_Malloc(p->arena, sizeof(StarEtc));
1985 if (!a) {
1986 return NULL;
1987 }
1988 a->vararg = vararg;
1989 a->kwonlyargs = kwonlyargs;
1990 a->kwarg = kwarg;
1991 return a;
1992 }
1993
1994 asdl_seq *
_PyPegen_join_sequences(Parser * p,asdl_seq * a,asdl_seq * b)1995 _PyPegen_join_sequences(Parser *p, asdl_seq *a, asdl_seq *b)
1996 {
1997 Py_ssize_t first_len = asdl_seq_LEN(a);
1998 Py_ssize_t second_len = asdl_seq_LEN(b);
1999 asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(first_len + second_len, p->arena);
2000 if (!new_seq) {
2001 return NULL;
2002 }
2003
2004 int k = 0;
2005 for (Py_ssize_t i = 0; i < first_len; i++) {
2006 asdl_seq_SET_UNTYPED(new_seq, k++, asdl_seq_GET_UNTYPED(a, i));
2007 }
2008 for (Py_ssize_t i = 0; i < second_len; i++) {
2009 asdl_seq_SET_UNTYPED(new_seq, k++, asdl_seq_GET_UNTYPED(b, i));
2010 }
2011
2012 return new_seq;
2013 }
2014
2015 static asdl_arg_seq*
_get_names(Parser * p,asdl_seq * names_with_defaults)2016 _get_names(Parser *p, asdl_seq *names_with_defaults)
2017 {
2018 Py_ssize_t len = asdl_seq_LEN(names_with_defaults);
2019 asdl_arg_seq *seq = _Py_asdl_arg_seq_new(len, p->arena);
2020 if (!seq) {
2021 return NULL;
2022 }
2023 for (Py_ssize_t i = 0; i < len; i++) {
2024 NameDefaultPair *pair = asdl_seq_GET_UNTYPED(names_with_defaults, i);
2025 asdl_seq_SET(seq, i, pair->arg);
2026 }
2027 return seq;
2028 }
2029
2030 static asdl_expr_seq *
_get_defaults(Parser * p,asdl_seq * names_with_defaults)2031 _get_defaults(Parser *p, asdl_seq *names_with_defaults)
2032 {
2033 Py_ssize_t len = asdl_seq_LEN(names_with_defaults);
2034 asdl_expr_seq *seq = _Py_asdl_expr_seq_new(len, p->arena);
2035 if (!seq) {
2036 return NULL;
2037 }
2038 for (Py_ssize_t i = 0; i < len; i++) {
2039 NameDefaultPair *pair = asdl_seq_GET_UNTYPED(names_with_defaults, i);
2040 asdl_seq_SET(seq, i, pair->value);
2041 }
2042 return seq;
2043 }
2044
2045 static int
_make_posonlyargs(Parser * p,asdl_arg_seq * slash_without_default,SlashWithDefault * slash_with_default,asdl_arg_seq ** posonlyargs)2046 _make_posonlyargs(Parser *p,
2047 asdl_arg_seq *slash_without_default,
2048 SlashWithDefault *slash_with_default,
2049 asdl_arg_seq **posonlyargs) {
2050 if (slash_without_default != NULL) {
2051 *posonlyargs = slash_without_default;
2052 }
2053 else if (slash_with_default != NULL) {
2054 asdl_arg_seq *slash_with_default_names =
2055 _get_names(p, slash_with_default->names_with_defaults);
2056 if (!slash_with_default_names) {
2057 return -1;
2058 }
2059 *posonlyargs = (asdl_arg_seq*)_PyPegen_join_sequences(
2060 p,
2061 (asdl_seq*)slash_with_default->plain_names,
2062 (asdl_seq*)slash_with_default_names);
2063 }
2064 else {
2065 *posonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
2066 }
2067 return *posonlyargs == NULL ? -1 : 0;
2068 }
2069
2070 static int
_make_posargs(Parser * p,asdl_arg_seq * plain_names,asdl_seq * names_with_default,asdl_arg_seq ** posargs)2071 _make_posargs(Parser *p,
2072 asdl_arg_seq *plain_names,
2073 asdl_seq *names_with_default,
2074 asdl_arg_seq **posargs) {
2075 if (plain_names != NULL && names_with_default != NULL) {
2076 asdl_arg_seq *names_with_default_names = _get_names(p, names_with_default);
2077 if (!names_with_default_names) {
2078 return -1;
2079 }
2080 *posargs = (asdl_arg_seq*)_PyPegen_join_sequences(
2081 p,(asdl_seq*)plain_names, (asdl_seq*)names_with_default_names);
2082 }
2083 else if (plain_names == NULL && names_with_default != NULL) {
2084 *posargs = _get_names(p, names_with_default);
2085 }
2086 else if (plain_names != NULL && names_with_default == NULL) {
2087 *posargs = plain_names;
2088 }
2089 else {
2090 *posargs = _Py_asdl_arg_seq_new(0, p->arena);
2091 }
2092 return *posargs == NULL ? -1 : 0;
2093 }
2094
2095 static int
_make_posdefaults(Parser * p,SlashWithDefault * slash_with_default,asdl_seq * names_with_default,asdl_expr_seq ** posdefaults)2096 _make_posdefaults(Parser *p,
2097 SlashWithDefault *slash_with_default,
2098 asdl_seq *names_with_default,
2099 asdl_expr_seq **posdefaults) {
2100 if (slash_with_default != NULL && names_with_default != NULL) {
2101 asdl_expr_seq *slash_with_default_values =
2102 _get_defaults(p, slash_with_default->names_with_defaults);
2103 if (!slash_with_default_values) {
2104 return -1;
2105 }
2106 asdl_expr_seq *names_with_default_values = _get_defaults(p, names_with_default);
2107 if (!names_with_default_values) {
2108 return -1;
2109 }
2110 *posdefaults = (asdl_expr_seq*)_PyPegen_join_sequences(
2111 p,
2112 (asdl_seq*)slash_with_default_values,
2113 (asdl_seq*)names_with_default_values);
2114 }
2115 else if (slash_with_default == NULL && names_with_default != NULL) {
2116 *posdefaults = _get_defaults(p, names_with_default);
2117 }
2118 else if (slash_with_default != NULL && names_with_default == NULL) {
2119 *posdefaults = _get_defaults(p, slash_with_default->names_with_defaults);
2120 }
2121 else {
2122 *posdefaults = _Py_asdl_expr_seq_new(0, p->arena);
2123 }
2124 return *posdefaults == NULL ? -1 : 0;
2125 }
2126
2127 static int
_make_kwargs(Parser * p,StarEtc * star_etc,asdl_arg_seq ** kwonlyargs,asdl_expr_seq ** kwdefaults)2128 _make_kwargs(Parser *p, StarEtc *star_etc,
2129 asdl_arg_seq **kwonlyargs,
2130 asdl_expr_seq **kwdefaults) {
2131 if (star_etc != NULL && star_etc->kwonlyargs != NULL) {
2132 *kwonlyargs = _get_names(p, star_etc->kwonlyargs);
2133 }
2134 else {
2135 *kwonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
2136 }
2137
2138 if (*kwonlyargs == NULL) {
2139 return -1;
2140 }
2141
2142 if (star_etc != NULL && star_etc->kwonlyargs != NULL) {
2143 *kwdefaults = _get_defaults(p, star_etc->kwonlyargs);
2144 }
2145 else {
2146 *kwdefaults = _Py_asdl_expr_seq_new(0, p->arena);
2147 }
2148
2149 if (*kwdefaults == NULL) {
2150 return -1;
2151 }
2152
2153 return 0;
2154 }
2155
2156 /* Constructs an arguments_ty object out of all the parsed constructs in the parameters rule */
2157 arguments_ty
_PyPegen_make_arguments(Parser * p,asdl_arg_seq * slash_without_default,SlashWithDefault * slash_with_default,asdl_arg_seq * plain_names,asdl_seq * names_with_default,StarEtc * star_etc)2158 _PyPegen_make_arguments(Parser *p, asdl_arg_seq *slash_without_default,
2159 SlashWithDefault *slash_with_default, asdl_arg_seq *plain_names,
2160 asdl_seq *names_with_default, StarEtc *star_etc)
2161 {
2162 asdl_arg_seq *posonlyargs;
2163 if (_make_posonlyargs(p, slash_without_default, slash_with_default, &posonlyargs) == -1) {
2164 return NULL;
2165 }
2166
2167 asdl_arg_seq *posargs;
2168 if (_make_posargs(p, plain_names, names_with_default, &posargs) == -1) {
2169 return NULL;
2170 }
2171
2172 asdl_expr_seq *posdefaults;
2173 if (_make_posdefaults(p,slash_with_default, names_with_default, &posdefaults) == -1) {
2174 return NULL;
2175 }
2176
2177 arg_ty vararg = NULL;
2178 if (star_etc != NULL && star_etc->vararg != NULL) {
2179 vararg = star_etc->vararg;
2180 }
2181
2182 asdl_arg_seq *kwonlyargs;
2183 asdl_expr_seq *kwdefaults;
2184 if (_make_kwargs(p, star_etc, &kwonlyargs, &kwdefaults) == -1) {
2185 return NULL;
2186 }
2187
2188 arg_ty kwarg = NULL;
2189 if (star_etc != NULL && star_etc->kwarg != NULL) {
2190 kwarg = star_etc->kwarg;
2191 }
2192
2193 return _PyAST_arguments(posonlyargs, posargs, vararg, kwonlyargs,
2194 kwdefaults, kwarg, posdefaults, p->arena);
2195 }
2196
2197
2198 /* Constructs an empty arguments_ty object, that gets used when a function accepts no
2199 * arguments. */
2200 arguments_ty
_PyPegen_empty_arguments(Parser * p)2201 _PyPegen_empty_arguments(Parser *p)
2202 {
2203 asdl_arg_seq *posonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
2204 if (!posonlyargs) {
2205 return NULL;
2206 }
2207 asdl_arg_seq *posargs = _Py_asdl_arg_seq_new(0, p->arena);
2208 if (!posargs) {
2209 return NULL;
2210 }
2211 asdl_expr_seq *posdefaults = _Py_asdl_expr_seq_new(0, p->arena);
2212 if (!posdefaults) {
2213 return NULL;
2214 }
2215 asdl_arg_seq *kwonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
2216 if (!kwonlyargs) {
2217 return NULL;
2218 }
2219 asdl_expr_seq *kwdefaults = _Py_asdl_expr_seq_new(0, p->arena);
2220 if (!kwdefaults) {
2221 return NULL;
2222 }
2223
2224 return _PyAST_arguments(posonlyargs, posargs, NULL, kwonlyargs,
2225 kwdefaults, NULL, posdefaults, p->arena);
2226 }
2227
2228 /* Encapsulates the value of an operator_ty into an AugOperator struct */
2229 AugOperator *
_PyPegen_augoperator(Parser * p,operator_ty kind)2230 _PyPegen_augoperator(Parser *p, operator_ty kind)
2231 {
2232 AugOperator *a = _PyArena_Malloc(p->arena, sizeof(AugOperator));
2233 if (!a) {
2234 return NULL;
2235 }
2236 a->kind = kind;
2237 return a;
2238 }
2239
2240 /* Construct a FunctionDef equivalent to function_def, but with decorators */
2241 stmt_ty
_PyPegen_function_def_decorators(Parser * p,asdl_expr_seq * decorators,stmt_ty function_def)2242 _PyPegen_function_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty function_def)
2243 {
2244 assert(function_def != NULL);
2245 if (function_def->kind == AsyncFunctionDef_kind) {
2246 return _PyAST_AsyncFunctionDef(
2247 function_def->v.FunctionDef.name, function_def->v.FunctionDef.args,
2248 function_def->v.FunctionDef.body, decorators, function_def->v.FunctionDef.returns,
2249 function_def->v.FunctionDef.type_comment, function_def->lineno,
2250 function_def->col_offset, function_def->end_lineno, function_def->end_col_offset,
2251 p->arena);
2252 }
2253
2254 return _PyAST_FunctionDef(
2255 function_def->v.FunctionDef.name, function_def->v.FunctionDef.args,
2256 function_def->v.FunctionDef.body, decorators,
2257 function_def->v.FunctionDef.returns,
2258 function_def->v.FunctionDef.type_comment, function_def->lineno,
2259 function_def->col_offset, function_def->end_lineno,
2260 function_def->end_col_offset, p->arena);
2261 }
2262
2263 /* Construct a ClassDef equivalent to class_def, but with decorators */
2264 stmt_ty
_PyPegen_class_def_decorators(Parser * p,asdl_expr_seq * decorators,stmt_ty class_def)2265 _PyPegen_class_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty class_def)
2266 {
2267 assert(class_def != NULL);
2268 return _PyAST_ClassDef(
2269 class_def->v.ClassDef.name, class_def->v.ClassDef.bases,
2270 class_def->v.ClassDef.keywords, class_def->v.ClassDef.body, decorators,
2271 class_def->lineno, class_def->col_offset, class_def->end_lineno,
2272 class_def->end_col_offset, p->arena);
2273 }
2274
2275 /* Construct a KeywordOrStarred */
2276 KeywordOrStarred *
_PyPegen_keyword_or_starred(Parser * p,void * element,int is_keyword)2277 _PyPegen_keyword_or_starred(Parser *p, void *element, int is_keyword)
2278 {
2279 KeywordOrStarred *a = _PyArena_Malloc(p->arena, sizeof(KeywordOrStarred));
2280 if (!a) {
2281 return NULL;
2282 }
2283 a->element = element;
2284 a->is_keyword = is_keyword;
2285 return a;
2286 }
2287
2288 /* Get the number of starred expressions in an asdl_seq* of KeywordOrStarred*s */
2289 static int
_seq_number_of_starred_exprs(asdl_seq * seq)2290 _seq_number_of_starred_exprs(asdl_seq *seq)
2291 {
2292 int n = 0;
2293 for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) {
2294 KeywordOrStarred *k = asdl_seq_GET_UNTYPED(seq, i);
2295 if (!k->is_keyword) {
2296 n++;
2297 }
2298 }
2299 return n;
2300 }
2301
2302 /* Extract the starred expressions of an asdl_seq* of KeywordOrStarred*s */
2303 asdl_expr_seq *
_PyPegen_seq_extract_starred_exprs(Parser * p,asdl_seq * kwargs)2304 _PyPegen_seq_extract_starred_exprs(Parser *p, asdl_seq *kwargs)
2305 {
2306 int new_len = _seq_number_of_starred_exprs(kwargs);
2307 if (new_len == 0) {
2308 return NULL;
2309 }
2310 asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(new_len, p->arena);
2311 if (!new_seq) {
2312 return NULL;
2313 }
2314
2315 int idx = 0;
2316 for (Py_ssize_t i = 0, len = asdl_seq_LEN(kwargs); i < len; i++) {
2317 KeywordOrStarred *k = asdl_seq_GET_UNTYPED(kwargs, i);
2318 if (!k->is_keyword) {
2319 asdl_seq_SET(new_seq, idx++, k->element);
2320 }
2321 }
2322 return new_seq;
2323 }
2324
2325 /* Return a new asdl_seq* with only the keywords in kwargs */
2326 asdl_keyword_seq*
_PyPegen_seq_delete_starred_exprs(Parser * p,asdl_seq * kwargs)2327 _PyPegen_seq_delete_starred_exprs(Parser *p, asdl_seq *kwargs)
2328 {
2329 Py_ssize_t len = asdl_seq_LEN(kwargs);
2330 Py_ssize_t new_len = len - _seq_number_of_starred_exprs(kwargs);
2331 if (new_len == 0) {
2332 return NULL;
2333 }
2334 asdl_keyword_seq *new_seq = _Py_asdl_keyword_seq_new(new_len, p->arena);
2335 if (!new_seq) {
2336 return NULL;
2337 }
2338
2339 int idx = 0;
2340 for (Py_ssize_t i = 0; i < len; i++) {
2341 KeywordOrStarred *k = asdl_seq_GET_UNTYPED(kwargs, i);
2342 if (k->is_keyword) {
2343 asdl_seq_SET(new_seq, idx++, k->element);
2344 }
2345 }
2346 return new_seq;
2347 }
2348
2349 expr_ty
_PyPegen_concatenate_strings(Parser * p,asdl_seq * strings)2350 _PyPegen_concatenate_strings(Parser *p, asdl_seq *strings)
2351 {
2352 Py_ssize_t len = asdl_seq_LEN(strings);
2353 assert(len > 0);
2354
2355 Token *first = asdl_seq_GET_UNTYPED(strings, 0);
2356 Token *last = asdl_seq_GET_UNTYPED(strings, len - 1);
2357
2358 int bytesmode = 0;
2359 PyObject *bytes_str = NULL;
2360
2361 FstringParser state;
2362 _PyPegen_FstringParser_Init(&state);
2363
2364 for (Py_ssize_t i = 0; i < len; i++) {
2365 Token *t = asdl_seq_GET_UNTYPED(strings, i);
2366
2367 int this_bytesmode;
2368 int this_rawmode;
2369 PyObject *s;
2370 const char *fstr;
2371 Py_ssize_t fstrlen = -1;
2372
2373 if (_PyPegen_parsestr(p, &this_bytesmode, &this_rawmode, &s, &fstr, &fstrlen, t) != 0) {
2374 goto error;
2375 }
2376
2377 /* Check that we are not mixing bytes with unicode. */
2378 if (i != 0 && bytesmode != this_bytesmode) {
2379 RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals");
2380 Py_XDECREF(s);
2381 goto error;
2382 }
2383 bytesmode = this_bytesmode;
2384
2385 if (fstr != NULL) {
2386 assert(s == NULL && !bytesmode);
2387
2388 int result = _PyPegen_FstringParser_ConcatFstring(p, &state, &fstr, fstr + fstrlen,
2389 this_rawmode, 0, first, t, last);
2390 if (result < 0) {
2391 goto error;
2392 }
2393 }
2394 else {
2395 /* String or byte string. */
2396 assert(s != NULL && fstr == NULL);
2397 assert(bytesmode ? PyBytes_CheckExact(s) : PyUnicode_CheckExact(s));
2398
2399 if (bytesmode) {
2400 if (i == 0) {
2401 bytes_str = s;
2402 }
2403 else {
2404 PyBytes_ConcatAndDel(&bytes_str, s);
2405 if (!bytes_str) {
2406 goto error;
2407 }
2408 }
2409 }
2410 else {
2411 /* This is a regular string. Concatenate it. */
2412 if (_PyPegen_FstringParser_ConcatAndDel(&state, s) < 0) {
2413 goto error;
2414 }
2415 }
2416 }
2417 }
2418
2419 if (bytesmode) {
2420 if (_PyArena_AddPyObject(p->arena, bytes_str) < 0) {
2421 goto error;
2422 }
2423 return _PyAST_Constant(bytes_str, NULL, first->lineno,
2424 first->col_offset, last->end_lineno,
2425 last->end_col_offset, p->arena);
2426 }
2427
2428 return _PyPegen_FstringParser_Finish(p, &state, first, last);
2429
2430 error:
2431 Py_XDECREF(bytes_str);
2432 _PyPegen_FstringParser_Dealloc(&state);
2433 if (PyErr_Occurred()) {
2434 raise_decode_error(p);
2435 }
2436 return NULL;
2437 }
2438
2439 expr_ty
_PyPegen_ensure_imaginary(Parser * p,expr_ty exp)2440 _PyPegen_ensure_imaginary(Parser *p, expr_ty exp)
2441 {
2442 if (exp->kind != Constant_kind || !PyComplex_CheckExact(exp->v.Constant.value)) {
2443 RAISE_SYNTAX_ERROR_KNOWN_LOCATION(exp, "imaginary number required in complex literal");
2444 return NULL;
2445 }
2446 return exp;
2447 }
2448
2449 expr_ty
_PyPegen_ensure_real(Parser * p,expr_ty exp)2450 _PyPegen_ensure_real(Parser *p, expr_ty exp)
2451 {
2452 if (exp->kind != Constant_kind || PyComplex_CheckExact(exp->v.Constant.value)) {
2453 RAISE_SYNTAX_ERROR_KNOWN_LOCATION(exp, "real number required in complex literal");
2454 return NULL;
2455 }
2456 return exp;
2457 }
2458
2459 mod_ty
_PyPegen_make_module(Parser * p,asdl_stmt_seq * a)2460 _PyPegen_make_module(Parser *p, asdl_stmt_seq *a) {
2461 asdl_type_ignore_seq *type_ignores = NULL;
2462 Py_ssize_t num = p->type_ignore_comments.num_items;
2463 if (num > 0) {
2464 // Turn the raw (comment, lineno) pairs into TypeIgnore objects in the arena
2465 type_ignores = _Py_asdl_type_ignore_seq_new(num, p->arena);
2466 if (type_ignores == NULL) {
2467 return NULL;
2468 }
2469 for (int i = 0; i < num; i++) {
2470 PyObject *tag = _PyPegen_new_type_comment(p, p->type_ignore_comments.items[i].comment);
2471 if (tag == NULL) {
2472 return NULL;
2473 }
2474 type_ignore_ty ti = _PyAST_TypeIgnore(p->type_ignore_comments.items[i].lineno,
2475 tag, p->arena);
2476 if (ti == NULL) {
2477 return NULL;
2478 }
2479 asdl_seq_SET(type_ignores, i, ti);
2480 }
2481 }
2482 return _PyAST_Module(a, type_ignores, p->arena);
2483 }
2484
2485 // Error reporting helpers
2486
2487 expr_ty
_PyPegen_get_invalid_target(expr_ty e,TARGETS_TYPE targets_type)2488 _PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type)
2489 {
2490 if (e == NULL) {
2491 return NULL;
2492 }
2493
2494 #define VISIT_CONTAINER(CONTAINER, TYPE) do { \
2495 Py_ssize_t len = asdl_seq_LEN((CONTAINER)->v.TYPE.elts);\
2496 for (Py_ssize_t i = 0; i < len; i++) {\
2497 expr_ty other = asdl_seq_GET((CONTAINER)->v.TYPE.elts, i);\
2498 expr_ty child = _PyPegen_get_invalid_target(other, targets_type);\
2499 if (child != NULL) {\
2500 return child;\
2501 }\
2502 }\
2503 } while (0)
2504
2505 // We only need to visit List and Tuple nodes recursively as those
2506 // are the only ones that can contain valid names in targets when
2507 // they are parsed as expressions. Any other kind of expression
2508 // that is a container (like Sets or Dicts) is directly invalid and
2509 // we don't need to visit it recursively.
2510
2511 switch (e->kind) {
2512 case List_kind:
2513 VISIT_CONTAINER(e, List);
2514 return NULL;
2515 case Tuple_kind:
2516 VISIT_CONTAINER(e, Tuple);
2517 return NULL;
2518 case Starred_kind:
2519 if (targets_type == DEL_TARGETS) {
2520 return e;
2521 }
2522 return _PyPegen_get_invalid_target(e->v.Starred.value, targets_type);
2523 case Compare_kind:
2524 // This is needed, because the `a in b` in `for a in b` gets parsed
2525 // as a comparison, and so we need to search the left side of the comparison
2526 // for invalid targets.
2527 if (targets_type == FOR_TARGETS) {
2528 cmpop_ty cmpop = (cmpop_ty) asdl_seq_GET(e->v.Compare.ops, 0);
2529 if (cmpop == In) {
2530 return _PyPegen_get_invalid_target(e->v.Compare.left, targets_type);
2531 }
2532 return NULL;
2533 }
2534 return e;
2535 case Name_kind:
2536 case Subscript_kind:
2537 case Attribute_kind:
2538 return NULL;
2539 default:
2540 return e;
2541 }
2542 }
2543
_PyPegen_arguments_parsing_error(Parser * p,expr_ty e)2544 void *_PyPegen_arguments_parsing_error(Parser *p, expr_ty e) {
2545 int kwarg_unpacking = 0;
2546 for (Py_ssize_t i = 0, l = asdl_seq_LEN(e->v.Call.keywords); i < l; i++) {
2547 keyword_ty keyword = asdl_seq_GET(e->v.Call.keywords, i);
2548 if (!keyword->arg) {
2549 kwarg_unpacking = 1;
2550 }
2551 }
2552
2553 const char *msg = NULL;
2554 if (kwarg_unpacking) {
2555 msg = "positional argument follows keyword argument unpacking";
2556 } else {
2557 msg = "positional argument follows keyword argument";
2558 }
2559
2560 return RAISE_SYNTAX_ERROR(msg);
2561 }
2562
2563
2564 static inline expr_ty
_PyPegen_get_last_comprehension_item(comprehension_ty comprehension)2565 _PyPegen_get_last_comprehension_item(comprehension_ty comprehension) {
2566 if (comprehension->ifs == NULL || asdl_seq_LEN(comprehension->ifs) == 0) {
2567 return comprehension->iter;
2568 }
2569 return PyPegen_last_item(comprehension->ifs, expr_ty);
2570 }
2571
2572 void *
_PyPegen_nonparen_genexp_in_call(Parser * p,expr_ty args,asdl_comprehension_seq * comprehensions)2573 _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions)
2574 {
2575 /* The rule that calls this function is 'args for_if_clauses'.
2576 For the input f(L, x for x in y), L and x are in args and
2577 the for is parsed as a for_if_clause. We have to check if
2578 len <= 1, so that input like dict((a, b) for a, b in x)
2579 gets successfully parsed and then we pass the last
2580 argument (x in the above example) as the location of the
2581 error */
2582 Py_ssize_t len = asdl_seq_LEN(args->v.Call.args);
2583 if (len <= 1) {
2584 return NULL;
2585 }
2586
2587 comprehension_ty last_comprehension = PyPegen_last_item(comprehensions, comprehension_ty);
2588
2589 return RAISE_SYNTAX_ERROR_KNOWN_RANGE(
2590 (expr_ty) asdl_seq_GET(args->v.Call.args, len - 1),
2591 _PyPegen_get_last_comprehension_item(last_comprehension),
2592 "Generator expression must be parenthesized"
2593 );
2594 }
2595
2596
_PyPegen_collect_call_seqs(Parser * p,asdl_expr_seq * a,asdl_seq * b,int lineno,int col_offset,int end_lineno,int end_col_offset,PyArena * arena)2597 expr_ty _PyPegen_collect_call_seqs(Parser *p, asdl_expr_seq *a, asdl_seq *b,
2598 int lineno, int col_offset, int end_lineno,
2599 int end_col_offset, PyArena *arena) {
2600 Py_ssize_t args_len = asdl_seq_LEN(a);
2601 Py_ssize_t total_len = args_len;
2602
2603 if (b == NULL) {
2604 return _PyAST_Call(_PyPegen_dummy_name(p), a, NULL, lineno, col_offset,
2605 end_lineno, end_col_offset, arena);
2606
2607 }
2608
2609 asdl_expr_seq *starreds = _PyPegen_seq_extract_starred_exprs(p, b);
2610 asdl_keyword_seq *keywords = _PyPegen_seq_delete_starred_exprs(p, b);
2611
2612 if (starreds) {
2613 total_len += asdl_seq_LEN(starreds);
2614 }
2615
2616 asdl_expr_seq *args = _Py_asdl_expr_seq_new(total_len, arena);
2617
2618 Py_ssize_t i = 0;
2619 for (i = 0; i < args_len; i++) {
2620 asdl_seq_SET(args, i, asdl_seq_GET(a, i));
2621 }
2622 for (; i < total_len; i++) {
2623 asdl_seq_SET(args, i, asdl_seq_GET(starreds, i - args_len));
2624 }
2625
2626 return _PyAST_Call(_PyPegen_dummy_name(p), args, keywords, lineno,
2627 col_offset, end_lineno, end_col_offset, arena);
2628 }
2629