• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 #include <Python.h>
2 #include "pycore_ast.h"           // _PyAST_Validate(),
3 #include "pycore_pystate.h"       // _PyThreadState_GET()
4 #include <errcode.h>
5 #include "tokenizer.h"
6 
7 #include "pegen.h"
8 #include "string_parser.h"
9 
10 PyObject *
_PyPegen_new_type_comment(Parser * p,const char * s)11 _PyPegen_new_type_comment(Parser *p, const char *s)
12 {
13     PyObject *res = PyUnicode_DecodeUTF8(s, strlen(s), NULL);
14     if (res == NULL) {
15         return NULL;
16     }
17     if (_PyArena_AddPyObject(p->arena, res) < 0) {
18         Py_DECREF(res);
19         return NULL;
20     }
21     return res;
22 }
23 
24 arg_ty
_PyPegen_add_type_comment_to_arg(Parser * p,arg_ty a,Token * tc)25 _PyPegen_add_type_comment_to_arg(Parser *p, arg_ty a, Token *tc)
26 {
27     if (tc == NULL) {
28         return a;
29     }
30     const char *bytes = PyBytes_AsString(tc->bytes);
31     if (bytes == NULL) {
32         return NULL;
33     }
34     PyObject *tco = _PyPegen_new_type_comment(p, bytes);
35     if (tco == NULL) {
36         return NULL;
37     }
38     return _PyAST_arg(a->arg, a->annotation, tco,
39                       a->lineno, a->col_offset, a->end_lineno, a->end_col_offset,
40                       p->arena);
41 }
42 
43 static int
init_normalization(Parser * p)44 init_normalization(Parser *p)
45 {
46     if (p->normalize) {
47         return 1;
48     }
49     PyObject *m = PyImport_ImportModuleNoBlock("unicodedata");
50     if (!m)
51     {
52         return 0;
53     }
54     p->normalize = PyObject_GetAttrString(m, "normalize");
55     Py_DECREF(m);
56     if (!p->normalize)
57     {
58         return 0;
59     }
60     return 1;
61 }
62 
63 /* Checks if the NOTEQUAL token is valid given the current parser flags
64 0 indicates success and nonzero indicates failure (an exception may be set) */
65 int
_PyPegen_check_barry_as_flufl(Parser * p,Token * t)66 _PyPegen_check_barry_as_flufl(Parser *p, Token* t) {
67     assert(t->bytes != NULL);
68     assert(t->type == NOTEQUAL);
69 
70     const char* tok_str = PyBytes_AS_STRING(t->bytes);
71     if (p->flags & PyPARSE_BARRY_AS_BDFL && strcmp(tok_str, "<>") != 0) {
72         RAISE_SYNTAX_ERROR("with Barry as BDFL, use '<>' instead of '!='");
73         return -1;
74     }
75     if (!(p->flags & PyPARSE_BARRY_AS_BDFL)) {
76         return strcmp(tok_str, "!=");
77     }
78     return 0;
79 }
80 
81 int
_PyPegen_check_legacy_stmt(Parser * p,expr_ty name)82 _PyPegen_check_legacy_stmt(Parser *p, expr_ty name) {
83     if (name->kind != Name_kind) {
84         return 0;
85     }
86     const char* candidates[2] = {"print", "exec"};
87     for (int i=0; i<2; i++) {
88         if (PyUnicode_CompareWithASCIIString(name->v.Name.id, candidates[i]) == 0) {
89             return 1;
90         }
91     }
92     return 0;
93 }
94 
95 PyObject *
_PyPegen_new_identifier(Parser * p,const char * n)96 _PyPegen_new_identifier(Parser *p, const char *n)
97 {
98     PyObject *id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
99     if (!id) {
100         goto error;
101     }
102     /* PyUnicode_DecodeUTF8 should always return a ready string. */
103     assert(PyUnicode_IS_READY(id));
104     /* Check whether there are non-ASCII characters in the
105        identifier; if so, normalize to NFKC. */
106     if (!PyUnicode_IS_ASCII(id))
107     {
108         PyObject *id2;
109         if (!init_normalization(p))
110         {
111             Py_DECREF(id);
112             goto error;
113         }
114         PyObject *form = PyUnicode_InternFromString("NFKC");
115         if (form == NULL)
116         {
117             Py_DECREF(id);
118             goto error;
119         }
120         PyObject *args[2] = {form, id};
121         id2 = _PyObject_FastCall(p->normalize, args, 2);
122         Py_DECREF(id);
123         Py_DECREF(form);
124         if (!id2) {
125             goto error;
126         }
127         if (!PyUnicode_Check(id2))
128         {
129             PyErr_Format(PyExc_TypeError,
130                          "unicodedata.normalize() must return a string, not "
131                          "%.200s",
132                          _PyType_Name(Py_TYPE(id2)));
133             Py_DECREF(id2);
134             goto error;
135         }
136         id = id2;
137     }
138     PyUnicode_InternInPlace(&id);
139     if (_PyArena_AddPyObject(p->arena, id) < 0)
140     {
141         Py_DECREF(id);
142         goto error;
143     }
144     return id;
145 
146 error:
147     p->error_indicator = 1;
148     return NULL;
149 }
150 
151 static PyObject *
_create_dummy_identifier(Parser * p)152 _create_dummy_identifier(Parser *p)
153 {
154     return _PyPegen_new_identifier(p, "");
155 }
156 
157 static inline Py_ssize_t
byte_offset_to_character_offset(PyObject * line,Py_ssize_t col_offset)158 byte_offset_to_character_offset(PyObject *line, Py_ssize_t col_offset)
159 {
160     const char *str = PyUnicode_AsUTF8(line);
161     if (!str) {
162         return 0;
163     }
164     Py_ssize_t len = strlen(str);
165     if (col_offset > len + 1) {
166         col_offset = len + 1;
167     }
168     assert(col_offset >= 0);
169     PyObject *text = PyUnicode_DecodeUTF8(str, col_offset, "replace");
170     if (!text) {
171         return 0;
172     }
173     Py_ssize_t size = PyUnicode_GET_LENGTH(text);
174     Py_DECREF(text);
175     return size;
176 }
177 
178 const char *
_PyPegen_get_expr_name(expr_ty e)179 _PyPegen_get_expr_name(expr_ty e)
180 {
181     assert(e != NULL);
182     switch (e->kind) {
183         case Attribute_kind:
184             return "attribute";
185         case Subscript_kind:
186             return "subscript";
187         case Starred_kind:
188             return "starred";
189         case Name_kind:
190             return "name";
191         case List_kind:
192             return "list";
193         case Tuple_kind:
194             return "tuple";
195         case Lambda_kind:
196             return "lambda";
197         case Call_kind:
198             return "function call";
199         case BoolOp_kind:
200         case BinOp_kind:
201         case UnaryOp_kind:
202             return "expression";
203         case GeneratorExp_kind:
204             return "generator expression";
205         case Yield_kind:
206         case YieldFrom_kind:
207             return "yield expression";
208         case Await_kind:
209             return "await expression";
210         case ListComp_kind:
211             return "list comprehension";
212         case SetComp_kind:
213             return "set comprehension";
214         case DictComp_kind:
215             return "dict comprehension";
216         case Dict_kind:
217             return "dict literal";
218         case Set_kind:
219             return "set display";
220         case JoinedStr_kind:
221         case FormattedValue_kind:
222             return "f-string expression";
223         case Constant_kind: {
224             PyObject *value = e->v.Constant.value;
225             if (value == Py_None) {
226                 return "None";
227             }
228             if (value == Py_False) {
229                 return "False";
230             }
231             if (value == Py_True) {
232                 return "True";
233             }
234             if (value == Py_Ellipsis) {
235                 return "ellipsis";
236             }
237             return "literal";
238         }
239         case Compare_kind:
240             return "comparison";
241         case IfExp_kind:
242             return "conditional expression";
243         case NamedExpr_kind:
244             return "named expression";
245         default:
246             PyErr_Format(PyExc_SystemError,
247                          "unexpected expression in assignment %d (line %d)",
248                          e->kind, e->lineno);
249             return NULL;
250     }
251 }
252 
253 static int
raise_decode_error(Parser * p)254 raise_decode_error(Parser *p)
255 {
256     assert(PyErr_Occurred());
257     const char *errtype = NULL;
258     if (PyErr_ExceptionMatches(PyExc_UnicodeError)) {
259         errtype = "unicode error";
260     }
261     else if (PyErr_ExceptionMatches(PyExc_ValueError)) {
262         errtype = "value error";
263     }
264     if (errtype) {
265         PyObject *type;
266         PyObject *value;
267         PyObject *tback;
268         PyObject *errstr;
269         PyErr_Fetch(&type, &value, &tback);
270         errstr = PyObject_Str(value);
271         if (errstr) {
272             RAISE_SYNTAX_ERROR("(%s) %U", errtype, errstr);
273             Py_DECREF(errstr);
274         }
275         else {
276             PyErr_Clear();
277             RAISE_SYNTAX_ERROR("(%s) unknown error", errtype);
278         }
279         Py_XDECREF(type);
280         Py_XDECREF(value);
281         Py_XDECREF(tback);
282     }
283 
284     return -1;
285 }
286 
287 static inline void
raise_unclosed_parentheses_error(Parser * p)288 raise_unclosed_parentheses_error(Parser *p) {
289        int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
290        int error_col = p->tok->parencolstack[p->tok->level-1];
291        RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError,
292                                   error_lineno, error_col, error_lineno, -1,
293                                   "'%c' was never closed",
294                                   p->tok->parenstack[p->tok->level-1]);
295 }
296 
297 static void
raise_tokenizer_init_error(PyObject * filename)298 raise_tokenizer_init_error(PyObject *filename)
299 {
300     if (!(PyErr_ExceptionMatches(PyExc_LookupError)
301           || PyErr_ExceptionMatches(PyExc_SyntaxError)
302           || PyErr_ExceptionMatches(PyExc_ValueError)
303           || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) {
304         return;
305     }
306     PyObject *errstr = NULL;
307     PyObject *tuple = NULL;
308     PyObject *type;
309     PyObject *value;
310     PyObject *tback;
311     PyErr_Fetch(&type, &value, &tback);
312     errstr = PyObject_Str(value);
313     if (!errstr) {
314         goto error;
315     }
316 
317     PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None);
318     if (!tmp) {
319         goto error;
320     }
321 
322     tuple = PyTuple_Pack(2, errstr, tmp);
323     Py_DECREF(tmp);
324     if (!value) {
325         goto error;
326     }
327     PyErr_SetObject(PyExc_SyntaxError, tuple);
328 
329 error:
330     Py_XDECREF(type);
331     Py_XDECREF(value);
332     Py_XDECREF(tback);
333     Py_XDECREF(errstr);
334     Py_XDECREF(tuple);
335 }
336 
337 static int
tokenizer_error(Parser * p)338 tokenizer_error(Parser *p)
339 {
340     if (PyErr_Occurred()) {
341         return -1;
342     }
343 
344     const char *msg = NULL;
345     PyObject* errtype = PyExc_SyntaxError;
346     Py_ssize_t col_offset = -1;
347     switch (p->tok->done) {
348         case E_TOKEN:
349             msg = "invalid token";
350             break;
351         case E_EOF:
352             if (p->tok->level) {
353                 raise_unclosed_parentheses_error(p);
354             } else {
355                 RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
356             }
357             return -1;
358         case E_DEDENT:
359             RAISE_INDENTATION_ERROR("unindent does not match any outer indentation level");
360             return -1;
361         case E_INTR:
362             if (!PyErr_Occurred()) {
363                 PyErr_SetNone(PyExc_KeyboardInterrupt);
364             }
365             return -1;
366         case E_NOMEM:
367             PyErr_NoMemory();
368             return -1;
369         case E_TABSPACE:
370             errtype = PyExc_TabError;
371             msg = "inconsistent use of tabs and spaces in indentation";
372             break;
373         case E_TOODEEP:
374             errtype = PyExc_IndentationError;
375             msg = "too many levels of indentation";
376             break;
377         case E_LINECONT: {
378             col_offset = p->tok->cur - p->tok->buf - 1;
379             msg = "unexpected character after line continuation character";
380             break;
381         }
382         default:
383             msg = "unknown parsing error";
384     }
385 
386     RAISE_ERROR_KNOWN_LOCATION(p, errtype, p->tok->lineno,
387                                col_offset >= 0 ? col_offset : 0,
388                                p->tok->lineno, -1, msg);
389     return -1;
390 }
391 
392 void *
_PyPegen_raise_error(Parser * p,PyObject * errtype,const char * errmsg,...)393 _PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...)
394 {
395     if (p->fill == 0) {
396         va_list va;
397         va_start(va, errmsg);
398         _PyPegen_raise_error_known_location(p, errtype, 0, 0, 0, -1, errmsg, va);
399         va_end(va);
400         return NULL;
401     }
402 
403     Token *t = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
404     Py_ssize_t col_offset;
405     Py_ssize_t end_col_offset = -1;
406     if (t->col_offset == -1) {
407         if (p->tok->cur == p->tok->buf) {
408             col_offset = 0;
409         } else {
410             const char* start = p->tok->buf  ? p->tok->line_start : p->tok->buf;
411             col_offset = Py_SAFE_DOWNCAST(p->tok->cur - start, intptr_t, int);
412         }
413     } else {
414         col_offset = t->col_offset + 1;
415     }
416 
417     if (t->end_col_offset != -1) {
418         end_col_offset = t->end_col_offset + 1;
419     }
420 
421     va_list va;
422     va_start(va, errmsg);
423     _PyPegen_raise_error_known_location(p, errtype, t->lineno, col_offset, t->end_lineno, end_col_offset, errmsg, va);
424     va_end(va);
425 
426     return NULL;
427 }
428 
429 static PyObject *
get_error_line(Parser * p,Py_ssize_t lineno)430 get_error_line(Parser *p, Py_ssize_t lineno)
431 {
432     /* If the file descriptor is interactive, the source lines of the current
433      * (multi-line) statement are stored in p->tok->interactive_src_start.
434      * If not, we're parsing from a string, which means that the whole source
435      * is stored in p->tok->str. */
436     assert((p->tok->fp == NULL && p->tok->str != NULL) || p->tok->fp == stdin);
437 
438     char *cur_line = p->tok->fp_interactive ? p->tok->interactive_src_start : p->tok->str;
439     assert(cur_line != NULL);
440 
441     for (int i = 0; i < lineno - 1; i++) {
442         cur_line = strchr(cur_line, '\n') + 1;
443     }
444 
445     char *next_newline;
446     if ((next_newline = strchr(cur_line, '\n')) == NULL) { // This is the last line
447         next_newline = cur_line + strlen(cur_line);
448     }
449     return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
450 }
451 
452 void *
_PyPegen_raise_error_known_location(Parser * p,PyObject * errtype,Py_ssize_t lineno,Py_ssize_t col_offset,Py_ssize_t end_lineno,Py_ssize_t end_col_offset,const char * errmsg,va_list va)453 _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
454                                     Py_ssize_t lineno, Py_ssize_t col_offset,
455                                     Py_ssize_t end_lineno, Py_ssize_t end_col_offset,
456                                     const char *errmsg, va_list va)
457 {
458     PyObject *value = NULL;
459     PyObject *errstr = NULL;
460     PyObject *error_line = NULL;
461     PyObject *tmp = NULL;
462     p->error_indicator = 1;
463 
464     if (end_lineno == CURRENT_POS) {
465         end_lineno = p->tok->lineno;
466     }
467     if (end_col_offset == CURRENT_POS) {
468         end_col_offset = p->tok->cur - p->tok->line_start;
469     }
470 
471     if (p->start_rule == Py_fstring_input) {
472         const char *fstring_msg = "f-string: ";
473         Py_ssize_t len = strlen(fstring_msg) + strlen(errmsg);
474 
475         char *new_errmsg = PyMem_Malloc(len + 1); // Lengths of both strings plus NULL character
476         if (!new_errmsg) {
477             return (void *) PyErr_NoMemory();
478         }
479 
480         // Copy both strings into new buffer
481         memcpy(new_errmsg, fstring_msg, strlen(fstring_msg));
482         memcpy(new_errmsg + strlen(fstring_msg), errmsg, strlen(errmsg));
483         new_errmsg[len] = 0;
484         errmsg = new_errmsg;
485     }
486     errstr = PyUnicode_FromFormatV(errmsg, va);
487     if (!errstr) {
488         goto error;
489     }
490 
491     if (p->tok->fp_interactive) {
492         error_line = get_error_line(p, lineno);
493     }
494     else if (p->start_rule == Py_file_input) {
495         error_line = _PyErr_ProgramDecodedTextObject(p->tok->filename,
496                                                      (int) lineno, p->tok->encoding);
497     }
498 
499     if (!error_line) {
500         /* PyErr_ProgramTextObject was not called or returned NULL. If it was not called,
501            then we need to find the error line from some other source, because
502            p->start_rule != Py_file_input. If it returned NULL, then it either unexpectedly
503            failed or we're parsing from a string or the REPL. There's a third edge case where
504            we're actually parsing from a file, which has an E_EOF SyntaxError and in that case
505            `PyErr_ProgramTextObject` fails because lineno points to last_file_line + 1, which
506            does not physically exist */
507         assert(p->tok->fp == NULL || p->tok->fp == stdin || p->tok->done == E_EOF);
508 
509         if (p->tok->lineno <= lineno && p->tok->inp > p->tok->buf) {
510             Py_ssize_t size = p->tok->inp - p->tok->buf;
511             error_line = PyUnicode_DecodeUTF8(p->tok->buf, size, "replace");
512         }
513         else if (p->tok->fp == NULL || p->tok->fp == stdin) {
514             error_line = get_error_line(p, lineno);
515         }
516         else {
517             error_line = PyUnicode_FromStringAndSize("", 0);
518         }
519         if (!error_line) {
520             goto error;
521         }
522     }
523 
524     if (p->start_rule == Py_fstring_input) {
525         col_offset -= p->starting_col_offset;
526         end_col_offset -= p->starting_col_offset;
527     }
528 
529     Py_ssize_t col_number = col_offset;
530     Py_ssize_t end_col_number = end_col_offset;
531 
532     if (p->tok->encoding != NULL) {
533         col_number = byte_offset_to_character_offset(error_line, col_offset);
534         end_col_number = end_col_number > 0 ?
535                          byte_offset_to_character_offset(error_line, end_col_offset) :
536                          end_col_number;
537     }
538     tmp = Py_BuildValue("(OiiNii)", p->tok->filename, lineno, col_number, error_line, end_lineno, end_col_number);
539     if (!tmp) {
540         goto error;
541     }
542     value = PyTuple_Pack(2, errstr, tmp);
543     Py_DECREF(tmp);
544     if (!value) {
545         goto error;
546     }
547     PyErr_SetObject(errtype, value);
548 
549     Py_DECREF(errstr);
550     Py_DECREF(value);
551     if (p->start_rule == Py_fstring_input) {
552         PyMem_Free((void *)errmsg);
553     }
554     return NULL;
555 
556 error:
557     Py_XDECREF(errstr);
558     Py_XDECREF(error_line);
559     if (p->start_rule == Py_fstring_input) {
560         PyMem_Free((void *)errmsg);
561     }
562     return NULL;
563 }
564 
565 #if 0
566 static const char *
567 token_name(int type)
568 {
569     if (0 <= type && type <= N_TOKENS) {
570         return _PyParser_TokenNames[type];
571     }
572     return "<Huh?>";
573 }
574 #endif
575 
576 // Here, mark is the start of the node, while p->mark is the end.
577 // If node==NULL, they should be the same.
578 int
_PyPegen_insert_memo(Parser * p,int mark,int type,void * node)579 _PyPegen_insert_memo(Parser *p, int mark, int type, void *node)
580 {
581     // Insert in front
582     Memo *m = _PyArena_Malloc(p->arena, sizeof(Memo));
583     if (m == NULL) {
584         return -1;
585     }
586     m->type = type;
587     m->node = node;
588     m->mark = p->mark;
589     m->next = p->tokens[mark]->memo;
590     p->tokens[mark]->memo = m;
591     return 0;
592 }
593 
594 // Like _PyPegen_insert_memo(), but updates an existing node if found.
595 int
_PyPegen_update_memo(Parser * p,int mark,int type,void * node)596 _PyPegen_update_memo(Parser *p, int mark, int type, void *node)
597 {
598     for (Memo *m = p->tokens[mark]->memo; m != NULL; m = m->next) {
599         if (m->type == type) {
600             // Update existing node.
601             m->node = node;
602             m->mark = p->mark;
603             return 0;
604         }
605     }
606     // Insert new node.
607     return _PyPegen_insert_memo(p, mark, type, node);
608 }
609 
610 // Return dummy NAME.
611 void *
_PyPegen_dummy_name(Parser * p,...)612 _PyPegen_dummy_name(Parser *p, ...)
613 {
614     static void *cache = NULL;
615 
616     if (cache != NULL) {
617         return cache;
618     }
619 
620     PyObject *id = _create_dummy_identifier(p);
621     if (!id) {
622         return NULL;
623     }
624     cache = _PyAST_Name(id, Load, 1, 0, 1, 0, p->arena);
625     return cache;
626 }
627 
628 static int
_get_keyword_or_name_type(Parser * p,const char * name,int name_len)629 _get_keyword_or_name_type(Parser *p, const char *name, int name_len)
630 {
631     assert(name_len > 0);
632     if (name_len >= p->n_keyword_lists ||
633         p->keywords[name_len] == NULL ||
634         p->keywords[name_len]->type == -1) {
635         return NAME;
636     }
637     for (KeywordToken *k = p->keywords[name_len]; k != NULL && k->type != -1; k++) {
638         if (strncmp(k->str, name, name_len) == 0) {
639             return k->type;
640         }
641     }
642     return NAME;
643 }
644 
645 static int
growable_comment_array_init(growable_comment_array * arr,size_t initial_size)646 growable_comment_array_init(growable_comment_array *arr, size_t initial_size) {
647     assert(initial_size > 0);
648     arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items));
649     arr->size = initial_size;
650     arr->num_items = 0;
651 
652     return arr->items != NULL;
653 }
654 
655 static int
growable_comment_array_add(growable_comment_array * arr,int lineno,char * comment)656 growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) {
657     if (arr->num_items >= arr->size) {
658         size_t new_size = arr->size * 2;
659         void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items));
660         if (!new_items_array) {
661             return 0;
662         }
663         arr->items = new_items_array;
664         arr->size = new_size;
665     }
666 
667     arr->items[arr->num_items].lineno = lineno;
668     arr->items[arr->num_items].comment = comment;  // Take ownership
669     arr->num_items++;
670     return 1;
671 }
672 
673 static void
growable_comment_array_deallocate(growable_comment_array * arr)674 growable_comment_array_deallocate(growable_comment_array *arr) {
675     for (unsigned i = 0; i < arr->num_items; i++) {
676         PyMem_Free(arr->items[i].comment);
677     }
678     PyMem_Free(arr->items);
679 }
680 
681 static int
initialize_token(Parser * p,Token * token,const char * start,const char * end,int token_type)682 initialize_token(Parser *p, Token *token, const char *start, const char *end, int token_type) {
683     assert(token != NULL);
684 
685     token->type = (token_type == NAME) ? _get_keyword_or_name_type(p, start, (int)(end - start)) : token_type;
686     token->bytes = PyBytes_FromStringAndSize(start, end - start);
687     if (token->bytes == NULL) {
688         return -1;
689     }
690 
691     if (_PyArena_AddPyObject(p->arena, token->bytes) < 0) {
692         Py_DECREF(token->bytes);
693         return -1;
694     }
695 
696     token->level = p->tok->level;
697 
698     const char *line_start = token_type == STRING ? p->tok->multi_line_start : p->tok->line_start;
699     int lineno = token_type == STRING ? p->tok->first_lineno : p->tok->lineno;
700     int end_lineno = p->tok->lineno;
701 
702     int col_offset = (start != NULL && start >= line_start) ? (int)(start - line_start) : -1;
703     int end_col_offset = (end != NULL && end >= p->tok->line_start) ? (int)(end - p->tok->line_start) : -1;
704 
705     token->lineno = lineno;
706     token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + col_offset : col_offset;
707     token->end_lineno = end_lineno;
708     token->end_col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + end_col_offset : end_col_offset;
709 
710     p->fill += 1;
711 
712     if (token_type == ERRORTOKEN && p->tok->done == E_DECODE) {
713         return raise_decode_error(p);
714     }
715 
716     return (token_type == ERRORTOKEN ? tokenizer_error(p) : 0);
717 }
718 
719 static int
_resize_tokens_array(Parser * p)720 _resize_tokens_array(Parser *p) {
721     int newsize = p->size * 2;
722     Token **new_tokens = PyMem_Realloc(p->tokens, newsize * sizeof(Token *));
723     if (new_tokens == NULL) {
724         PyErr_NoMemory();
725         return -1;
726     }
727     p->tokens = new_tokens;
728 
729     for (int i = p->size; i < newsize; i++) {
730         p->tokens[i] = PyMem_Calloc(1, sizeof(Token));
731         if (p->tokens[i] == NULL) {
732             p->size = i; // Needed, in order to cleanup correctly after parser fails
733             PyErr_NoMemory();
734             return -1;
735         }
736     }
737     p->size = newsize;
738     return 0;
739 }
740 
741 int
_PyPegen_fill_token(Parser * p)742 _PyPegen_fill_token(Parser *p)
743 {
744     const char *start;
745     const char *end;
746     int type = PyTokenizer_Get(p->tok, &start, &end);
747 
748     // Record and skip '# type: ignore' comments
749     while (type == TYPE_IGNORE) {
750         Py_ssize_t len = end - start;
751         char *tag = PyMem_Malloc(len + 1);
752         if (tag == NULL) {
753             PyErr_NoMemory();
754             return -1;
755         }
756         strncpy(tag, start, len);
757         tag[len] = '\0';
758         // Ownership of tag passes to the growable array
759         if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
760             PyErr_NoMemory();
761             return -1;
762         }
763         type = PyTokenizer_Get(p->tok, &start, &end);
764     }
765 
766     // If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
767     if (p->start_rule == Py_single_input && type == ENDMARKER && p->parsing_started) {
768         type = NEWLINE; /* Add an extra newline */
769         p->parsing_started = 0;
770 
771         if (p->tok->indent && !(p->flags & PyPARSE_DONT_IMPLY_DEDENT)) {
772             p->tok->pendin = -p->tok->indent;
773             p->tok->indent = 0;
774         }
775     }
776     else {
777         p->parsing_started = 1;
778     }
779 
780     // Check if we are at the limit of the token array capacity and resize if needed
781     if ((p->fill == p->size) && (_resize_tokens_array(p) != 0)) {
782         return -1;
783     }
784 
785     Token *t = p->tokens[p->fill];
786     return initialize_token(p, t, start, end, type);
787 }
788 
789 
790 #if defined(Py_DEBUG)
791 // Instrumentation to count the effectiveness of memoization.
792 // The array counts the number of tokens skipped by memoization,
793 // indexed by type.
794 
795 #define NSTATISTICS 2000
796 static long memo_statistics[NSTATISTICS];
797 
798 void
_PyPegen_clear_memo_statistics()799 _PyPegen_clear_memo_statistics()
800 {
801     for (int i = 0; i < NSTATISTICS; i++) {
802         memo_statistics[i] = 0;
803     }
804 }
805 
806 PyObject *
_PyPegen_get_memo_statistics()807 _PyPegen_get_memo_statistics()
808 {
809     PyObject *ret = PyList_New(NSTATISTICS);
810     if (ret == NULL) {
811         return NULL;
812     }
813     for (int i = 0; i < NSTATISTICS; i++) {
814         PyObject *value = PyLong_FromLong(memo_statistics[i]);
815         if (value == NULL) {
816             Py_DECREF(ret);
817             return NULL;
818         }
819         // PyList_SetItem borrows a reference to value.
820         if (PyList_SetItem(ret, i, value) < 0) {
821             Py_DECREF(ret);
822             return NULL;
823         }
824     }
825     return ret;
826 }
827 #endif
828 
829 int  // bool
_PyPegen_is_memoized(Parser * p,int type,void * pres)830 _PyPegen_is_memoized(Parser *p, int type, void *pres)
831 {
832     if (p->mark == p->fill) {
833         if (_PyPegen_fill_token(p) < 0) {
834             p->error_indicator = 1;
835             return -1;
836         }
837     }
838 
839     Token *t = p->tokens[p->mark];
840 
841     for (Memo *m = t->memo; m != NULL; m = m->next) {
842         if (m->type == type) {
843 #if defined(PY_DEBUG)
844             if (0 <= type && type < NSTATISTICS) {
845                 long count = m->mark - p->mark;
846                 // A memoized negative result counts for one.
847                 if (count <= 0) {
848                     count = 1;
849                 }
850                 memo_statistics[type] += count;
851             }
852 #endif
853             p->mark = m->mark;
854             *(void **)(pres) = m->node;
855             return 1;
856         }
857     }
858     return 0;
859 }
860 
861 int
_PyPegen_lookahead_with_name(int positive,expr_ty (func)(Parser *),Parser * p)862 _PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p)
863 {
864     int mark = p->mark;
865     void *res = func(p);
866     p->mark = mark;
867     return (res != NULL) == positive;
868 }
869 
870 int
_PyPegen_lookahead_with_string(int positive,expr_ty (func)(Parser *,const char *),Parser * p,const char * arg)871 _PyPegen_lookahead_with_string(int positive, expr_ty (func)(Parser *, const char*), Parser *p, const char* arg)
872 {
873     int mark = p->mark;
874     void *res = func(p, arg);
875     p->mark = mark;
876     return (res != NULL) == positive;
877 }
878 
879 int
_PyPegen_lookahead_with_int(int positive,Token * (func)(Parser *,int),Parser * p,int arg)880 _PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg)
881 {
882     int mark = p->mark;
883     void *res = func(p, arg);
884     p->mark = mark;
885     return (res != NULL) == positive;
886 }
887 
888 int
_PyPegen_lookahead(int positive,void * (func)(Parser *),Parser * p)889 _PyPegen_lookahead(int positive, void *(func)(Parser *), Parser *p)
890 {
891     int mark = p->mark;
892     void *res = (void*)func(p);
893     p->mark = mark;
894     return (res != NULL) == positive;
895 }
896 
897 Token *
_PyPegen_expect_token(Parser * p,int type)898 _PyPegen_expect_token(Parser *p, int type)
899 {
900     if (p->mark == p->fill) {
901         if (_PyPegen_fill_token(p) < 0) {
902             p->error_indicator = 1;
903             return NULL;
904         }
905     }
906     Token *t = p->tokens[p->mark];
907     if (t->type != type) {
908         return NULL;
909     }
910     p->mark += 1;
911     return t;
912 }
913 
914 Token *
_PyPegen_expect_forced_token(Parser * p,int type,const char * expected)915 _PyPegen_expect_forced_token(Parser *p, int type, const char* expected) {
916 
917     if (p->error_indicator == 1) {
918         return NULL;
919     }
920 
921     if (p->mark == p->fill) {
922         if (_PyPegen_fill_token(p) < 0) {
923             p->error_indicator = 1;
924             return NULL;
925         }
926     }
927     Token *t = p->tokens[p->mark];
928     if (t->type != type) {
929         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "expected '%s'", expected);
930         return NULL;
931     }
932     p->mark += 1;
933     return t;
934 }
935 
936 expr_ty
_PyPegen_expect_soft_keyword(Parser * p,const char * keyword)937 _PyPegen_expect_soft_keyword(Parser *p, const char *keyword)
938 {
939     if (p->mark == p->fill) {
940         if (_PyPegen_fill_token(p) < 0) {
941             p->error_indicator = 1;
942             return NULL;
943         }
944     }
945     Token *t = p->tokens[p->mark];
946     if (t->type != NAME) {
947         return NULL;
948     }
949     const char *s = PyBytes_AsString(t->bytes);
950     if (!s) {
951         p->error_indicator = 1;
952         return NULL;
953     }
954     if (strcmp(s, keyword) != 0) {
955         return NULL;
956     }
957     return _PyPegen_name_token(p);
958 }
959 
960 Token *
_PyPegen_get_last_nonnwhitespace_token(Parser * p)961 _PyPegen_get_last_nonnwhitespace_token(Parser *p)
962 {
963     assert(p->mark >= 0);
964     Token *token = NULL;
965     for (int m = p->mark - 1; m >= 0; m--) {
966         token = p->tokens[m];
967         if (token->type != ENDMARKER && (token->type < NEWLINE || token->type > DEDENT)) {
968             break;
969         }
970     }
971     return token;
972 }
973 
974 static expr_ty
_PyPegen_name_from_token(Parser * p,Token * t)975 _PyPegen_name_from_token(Parser *p, Token* t)
976 {
977     if (t == NULL) {
978         return NULL;
979     }
980     const char *s = PyBytes_AsString(t->bytes);
981     if (!s) {
982         p->error_indicator = 1;
983         return NULL;
984     }
985     PyObject *id = _PyPegen_new_identifier(p, s);
986     if (id == NULL) {
987         p->error_indicator = 1;
988         return NULL;
989     }
990     return _PyAST_Name(id, Load, t->lineno, t->col_offset, t->end_lineno,
991                        t->end_col_offset, p->arena);
992 }
993 
994 
995 expr_ty
_PyPegen_name_token(Parser * p)996 _PyPegen_name_token(Parser *p)
997 {
998     Token *t = _PyPegen_expect_token(p, NAME);
999     return _PyPegen_name_from_token(p, t);
1000 }
1001 
1002 void *
_PyPegen_string_token(Parser * p)1003 _PyPegen_string_token(Parser *p)
1004 {
1005     return _PyPegen_expect_token(p, STRING);
1006 }
1007 
1008 
_PyPegen_soft_keyword_token(Parser * p)1009 expr_ty _PyPegen_soft_keyword_token(Parser *p) {
1010     Token *t = _PyPegen_expect_token(p, NAME);
1011     if (t == NULL) {
1012         return NULL;
1013     }
1014     char *the_token;
1015     Py_ssize_t size;
1016     PyBytes_AsStringAndSize(t->bytes, &the_token, &size);
1017     for (char **keyword = p->soft_keywords; *keyword != NULL; keyword++) {
1018         if (strncmp(*keyword, the_token, size) == 0) {
1019             return _PyPegen_name_from_token(p, t);
1020         }
1021     }
1022     return NULL;
1023 }
1024 
1025 static PyObject *
parsenumber_raw(const char * s)1026 parsenumber_raw(const char *s)
1027 {
1028     const char *end;
1029     long x;
1030     double dx;
1031     Py_complex compl;
1032     int imflag;
1033 
1034     assert(s != NULL);
1035     errno = 0;
1036     end = s + strlen(s) - 1;
1037     imflag = *end == 'j' || *end == 'J';
1038     if (s[0] == '0') {
1039         x = (long)PyOS_strtoul(s, (char **)&end, 0);
1040         if (x < 0 && errno == 0) {
1041             return PyLong_FromString(s, (char **)0, 0);
1042         }
1043     }
1044     else {
1045         x = PyOS_strtol(s, (char **)&end, 0);
1046     }
1047     if (*end == '\0') {
1048         if (errno != 0) {
1049             return PyLong_FromString(s, (char **)0, 0);
1050         }
1051         return PyLong_FromLong(x);
1052     }
1053     /* XXX Huge floats may silently fail */
1054     if (imflag) {
1055         compl.real = 0.;
1056         compl.imag = PyOS_string_to_double(s, (char **)&end, NULL);
1057         if (compl.imag == -1.0 && PyErr_Occurred()) {
1058             return NULL;
1059         }
1060         return PyComplex_FromCComplex(compl);
1061     }
1062     dx = PyOS_string_to_double(s, NULL, NULL);
1063     if (dx == -1.0 && PyErr_Occurred()) {
1064         return NULL;
1065     }
1066     return PyFloat_FromDouble(dx);
1067 }
1068 
1069 static PyObject *
parsenumber(const char * s)1070 parsenumber(const char *s)
1071 {
1072     char *dup;
1073     char *end;
1074     PyObject *res = NULL;
1075 
1076     assert(s != NULL);
1077 
1078     if (strchr(s, '_') == NULL) {
1079         return parsenumber_raw(s);
1080     }
1081     /* Create a duplicate without underscores. */
1082     dup = PyMem_Malloc(strlen(s) + 1);
1083     if (dup == NULL) {
1084         return PyErr_NoMemory();
1085     }
1086     end = dup;
1087     for (; *s; s++) {
1088         if (*s != '_') {
1089             *end++ = *s;
1090         }
1091     }
1092     *end = '\0';
1093     res = parsenumber_raw(dup);
1094     PyMem_Free(dup);
1095     return res;
1096 }
1097 
1098 expr_ty
_PyPegen_number_token(Parser * p)1099 _PyPegen_number_token(Parser *p)
1100 {
1101     Token *t = _PyPegen_expect_token(p, NUMBER);
1102     if (t == NULL) {
1103         return NULL;
1104     }
1105 
1106     const char *num_raw = PyBytes_AsString(t->bytes);
1107     if (num_raw == NULL) {
1108         p->error_indicator = 1;
1109         return NULL;
1110     }
1111 
1112     if (p->feature_version < 6 && strchr(num_raw, '_') != NULL) {
1113         p->error_indicator = 1;
1114         return RAISE_SYNTAX_ERROR("Underscores in numeric literals are only supported "
1115                                   "in Python 3.6 and greater");
1116     }
1117 
1118     PyObject *c = parsenumber(num_raw);
1119 
1120     if (c == NULL) {
1121         p->error_indicator = 1;
1122         PyThreadState *tstate = _PyThreadState_GET();
1123         // The only way a ValueError should happen in _this_ code is via
1124         // PyLong_FromString hitting a length limit.
1125         if (tstate->curexc_type == PyExc_ValueError &&
1126             tstate->curexc_value != NULL) {
1127             PyObject *type, *value, *tb;
1128             // This acts as PyErr_Clear() as we're replacing curexc.
1129             PyErr_Fetch(&type, &value, &tb);
1130             Py_XDECREF(tb);
1131             Py_DECREF(type);
1132             /* Intentionally omitting columns to avoid a wall of 1000s of '^'s
1133              * on the error message. Nobody is going to overlook their huge
1134              * numeric literal once given the line. */
1135             RAISE_ERROR_KNOWN_LOCATION(
1136                 p, PyExc_SyntaxError,
1137                 t->lineno, -1 /* col_offset */,
1138                 t->end_lineno, -1 /* end_col_offset */,
1139                 "%S - Consider hexadecimal for huge integer literals "
1140                 "to avoid decimal conversion limits.",
1141                 value);
1142             Py_DECREF(value);
1143         }
1144         return NULL;
1145     }
1146 
1147     if (_PyArena_AddPyObject(p->arena, c) < 0) {
1148         Py_DECREF(c);
1149         p->error_indicator = 1;
1150         return NULL;
1151     }
1152 
1153     return _PyAST_Constant(c, NULL, t->lineno, t->col_offset, t->end_lineno,
1154                            t->end_col_offset, p->arena);
1155 }
1156 
1157 /* Check that the source for a single input statement really is a single
1158    statement by looking at what is left in the buffer after parsing.
1159    Trailing whitespace and comments are OK. */
1160 static int // bool
bad_single_statement(Parser * p)1161 bad_single_statement(Parser *p)
1162 {
1163     char *cur = p->tok->cur;
1164     char c = *cur;
1165 
1166     for (;;) {
1167         while (c == ' ' || c == '\t' || c == '\n' || c == '\014') {
1168             c = *++cur;
1169         }
1170 
1171         if (!c) {
1172             return 0;
1173         }
1174 
1175         if (c != '#') {
1176             return 1;
1177         }
1178 
1179         /* Suck up comment. */
1180         while (c && c != '\n') {
1181             c = *++cur;
1182         }
1183     }
1184 }
1185 
1186 void
_PyPegen_Parser_Free(Parser * p)1187 _PyPegen_Parser_Free(Parser *p)
1188 {
1189     Py_XDECREF(p->normalize);
1190     for (int i = 0; i < p->size; i++) {
1191         PyMem_Free(p->tokens[i]);
1192     }
1193     PyMem_Free(p->tokens);
1194     growable_comment_array_deallocate(&p->type_ignore_comments);
1195     PyMem_Free(p);
1196 }
1197 
1198 static int
compute_parser_flags(PyCompilerFlags * flags)1199 compute_parser_flags(PyCompilerFlags *flags)
1200 {
1201     int parser_flags = 0;
1202     if (!flags) {
1203         return 0;
1204     }
1205     if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT) {
1206         parser_flags |= PyPARSE_DONT_IMPLY_DEDENT;
1207     }
1208     if (flags->cf_flags & PyCF_IGNORE_COOKIE) {
1209         parser_flags |= PyPARSE_IGNORE_COOKIE;
1210     }
1211     if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) {
1212         parser_flags |= PyPARSE_BARRY_AS_BDFL;
1213     }
1214     if (flags->cf_flags & PyCF_TYPE_COMMENTS) {
1215         parser_flags |= PyPARSE_TYPE_COMMENTS;
1216     }
1217     if ((flags->cf_flags & PyCF_ONLY_AST) && flags->cf_feature_version < 7) {
1218         parser_flags |= PyPARSE_ASYNC_HACKS;
1219     }
1220     return parser_flags;
1221 }
1222 
1223 Parser *
_PyPegen_Parser_New(struct tok_state * tok,int start_rule,int flags,int feature_version,int * errcode,PyArena * arena)1224 _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
1225                     int feature_version, int *errcode, PyArena *arena)
1226 {
1227     Parser *p = PyMem_Malloc(sizeof(Parser));
1228     if (p == NULL) {
1229         return (Parser *) PyErr_NoMemory();
1230     }
1231     assert(tok != NULL);
1232     tok->type_comments = (flags & PyPARSE_TYPE_COMMENTS) > 0;
1233     tok->async_hacks = (flags & PyPARSE_ASYNC_HACKS) > 0;
1234     p->tok = tok;
1235     p->keywords = NULL;
1236     p->n_keyword_lists = -1;
1237     p->soft_keywords = NULL;
1238     p->tokens = PyMem_Malloc(sizeof(Token *));
1239     if (!p->tokens) {
1240         PyMem_Free(p);
1241         return (Parser *) PyErr_NoMemory();
1242     }
1243     p->tokens[0] = PyMem_Calloc(1, sizeof(Token));
1244     if (!p->tokens) {
1245         PyMem_Free(p->tokens);
1246         PyMem_Free(p);
1247         return (Parser *) PyErr_NoMemory();
1248     }
1249     if (!growable_comment_array_init(&p->type_ignore_comments, 10)) {
1250         PyMem_Free(p->tokens[0]);
1251         PyMem_Free(p->tokens);
1252         PyMem_Free(p);
1253         return (Parser *) PyErr_NoMemory();
1254     }
1255 
1256     p->mark = 0;
1257     p->fill = 0;
1258     p->size = 1;
1259 
1260     p->errcode = errcode;
1261     p->arena = arena;
1262     p->start_rule = start_rule;
1263     p->parsing_started = 0;
1264     p->normalize = NULL;
1265     p->error_indicator = 0;
1266 
1267     p->starting_lineno = 0;
1268     p->starting_col_offset = 0;
1269     p->flags = flags;
1270     p->feature_version = feature_version;
1271     p->known_err_token = NULL;
1272     p->level = 0;
1273     p->call_invalid_rules = 0;
1274     p->in_raw_rule = 0;
1275     return p;
1276 }
1277 
1278 static void
reset_parser_state(Parser * p)1279 reset_parser_state(Parser *p)
1280 {
1281     for (int i = 0; i < p->fill; i++) {
1282         p->tokens[i]->memo = NULL;
1283     }
1284     p->mark = 0;
1285     p->call_invalid_rules = 1;
1286     // Don't try to get extra tokens in interactive mode when trying to
1287     // raise specialized errors in the second pass.
1288     p->tok->interactive_underflow = IUNDERFLOW_STOP;
1289 }
1290 
1291 static int
_PyPegen_check_tokenizer_errors(Parser * p)1292 _PyPegen_check_tokenizer_errors(Parser *p) {
1293     // Tokenize the whole input to see if there are any tokenization
1294     // errors such as mistmatching parentheses. These will get priority
1295     // over generic syntax errors only if the line number of the error is
1296     // before the one that we had for the generic error.
1297 
1298     // We don't want to tokenize to the end for interactive input
1299     if (p->tok->prompt != NULL) {
1300         return 0;
1301     }
1302 
1303     PyObject *type, *value, *traceback;
1304     PyErr_Fetch(&type, &value, &traceback);
1305 
1306     Token *current_token = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
1307     Py_ssize_t current_err_line = current_token->lineno;
1308 
1309     int ret = 0;
1310 
1311     for (;;) {
1312         const char *start;
1313         const char *end;
1314         switch (PyTokenizer_Get(p->tok, &start, &end)) {
1315             case ERRORTOKEN:
1316                 if (p->tok->level != 0) {
1317                     int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
1318                     if (current_err_line > error_lineno) {
1319                         raise_unclosed_parentheses_error(p);
1320                         ret = -1;
1321                         goto exit;
1322                     }
1323                 }
1324                 break;
1325             case ENDMARKER:
1326                 break;
1327             default:
1328                 continue;
1329         }
1330         break;
1331     }
1332 
1333 
1334 exit:
1335     if (PyErr_Occurred()) {
1336         Py_XDECREF(value);
1337         Py_XDECREF(type);
1338         Py_XDECREF(traceback);
1339     } else {
1340         PyErr_Restore(type, value, traceback);
1341     }
1342     return ret;
1343 }
1344 
1345 void *
_PyPegen_run_parser(Parser * p)1346 _PyPegen_run_parser(Parser *p)
1347 {
1348     void *res = _PyPegen_parse(p);
1349     assert(p->level == 0);
1350     if (res == NULL) {
1351         if (PyErr_Occurred() && !PyErr_ExceptionMatches(PyExc_SyntaxError)) {
1352             return NULL;
1353         }
1354         Token *last_token = p->tokens[p->fill - 1];
1355         reset_parser_state(p);
1356         _PyPegen_parse(p);
1357         if (PyErr_Occurred()) {
1358             // Prioritize tokenizer errors to custom syntax errors raised
1359             // on the second phase only if the errors come from the parser.
1360             if (p->tok->done == E_DONE && PyErr_ExceptionMatches(PyExc_SyntaxError)) {
1361                 _PyPegen_check_tokenizer_errors(p);
1362             }
1363             return NULL;
1364         }
1365         if (p->fill == 0) {
1366             RAISE_SYNTAX_ERROR("error at start before reading any input");
1367         }
1368         else if (last_token->type == ERRORTOKEN && p->tok->done == E_EOF) {
1369             if (p->tok->level) {
1370                 raise_unclosed_parentheses_error(p);
1371             } else {
1372                 RAISE_SYNTAX_ERROR("unexpected EOF while parsing");
1373             }
1374         }
1375         else {
1376             if (p->tokens[p->fill-1]->type == INDENT) {
1377                 RAISE_INDENTATION_ERROR("unexpected indent");
1378             }
1379             else if (p->tokens[p->fill-1]->type == DEDENT) {
1380                 RAISE_INDENTATION_ERROR("unexpected unindent");
1381             }
1382             else {
1383                 // Use the last token we found on the first pass to avoid reporting
1384                 // incorrect locations for generic syntax errors just because we reached
1385                 // further away when trying to find specific syntax errors in the second
1386                 // pass.
1387                 RAISE_SYNTAX_ERROR_KNOWN_LOCATION(last_token, "invalid syntax");
1388                 // _PyPegen_check_tokenizer_errors will override the existing
1389                 // generic SyntaxError we just raised if errors are found.
1390                 _PyPegen_check_tokenizer_errors(p);
1391             }
1392         }
1393         return NULL;
1394     }
1395 
1396     if (p->start_rule == Py_single_input && bad_single_statement(p)) {
1397         p->tok->done = E_BADSINGLE; // This is not necessary for now, but might be in the future
1398         return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
1399     }
1400 
1401     // test_peg_generator defines _Py_TEST_PEGEN to not call PyAST_Validate()
1402 #if defined(Py_DEBUG) && !defined(_Py_TEST_PEGEN)
1403     if (p->start_rule == Py_single_input ||
1404         p->start_rule == Py_file_input ||
1405         p->start_rule == Py_eval_input)
1406     {
1407         if (!_PyAST_Validate(res)) {
1408             return NULL;
1409         }
1410     }
1411 #endif
1412     return res;
1413 }
1414 
1415 mod_ty
_PyPegen_run_parser_from_file_pointer(FILE * fp,int start_rule,PyObject * filename_ob,const char * enc,const char * ps1,const char * ps2,PyCompilerFlags * flags,int * errcode,PyArena * arena)1416 _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob,
1417                              const char *enc, const char *ps1, const char *ps2,
1418                              PyCompilerFlags *flags, int *errcode, PyArena *arena)
1419 {
1420     struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2);
1421     if (tok == NULL) {
1422         if (PyErr_Occurred()) {
1423             raise_tokenizer_init_error(filename_ob);
1424             return NULL;
1425         }
1426         return NULL;
1427     }
1428     if (!tok->fp || ps1 != NULL || ps2 != NULL ||
1429         PyUnicode_CompareWithASCIIString(filename_ob, "<stdin>") == 0) {
1430         tok->fp_interactive = 1;
1431     }
1432     // This transfers the ownership to the tokenizer
1433     tok->filename = filename_ob;
1434     Py_INCREF(filename_ob);
1435 
1436     // From here on we need to clean up even if there's an error
1437     mod_ty result = NULL;
1438 
1439     int parser_flags = compute_parser_flags(flags);
1440     Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
1441                                     errcode, arena);
1442     if (p == NULL) {
1443         goto error;
1444     }
1445 
1446     result = _PyPegen_run_parser(p);
1447     _PyPegen_Parser_Free(p);
1448 
1449 error:
1450     PyTokenizer_Free(tok);
1451     return result;
1452 }
1453 
1454 mod_ty
_PyPegen_run_parser_from_string(const char * str,int start_rule,PyObject * filename_ob,PyCompilerFlags * flags,PyArena * arena)1455 _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob,
1456                        PyCompilerFlags *flags, PyArena *arena)
1457 {
1458     int exec_input = start_rule == Py_file_input;
1459 
1460     struct tok_state *tok;
1461     if (flags != NULL && flags->cf_flags & PyCF_IGNORE_COOKIE) {
1462         tok = PyTokenizer_FromUTF8(str, exec_input);
1463     } else {
1464         tok = PyTokenizer_FromString(str, exec_input);
1465     }
1466     if (tok == NULL) {
1467         if (PyErr_Occurred()) {
1468             raise_tokenizer_init_error(filename_ob);
1469         }
1470         return NULL;
1471     }
1472     // This transfers the ownership to the tokenizer
1473     tok->filename = filename_ob;
1474     Py_INCREF(filename_ob);
1475 
1476     // We need to clear up from here on
1477     mod_ty result = NULL;
1478 
1479     int parser_flags = compute_parser_flags(flags);
1480     int feature_version = flags && (flags->cf_flags & PyCF_ONLY_AST) ?
1481         flags->cf_feature_version : PY_MINOR_VERSION;
1482     Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
1483                                     NULL, arena);
1484     if (p == NULL) {
1485         goto error;
1486     }
1487 
1488     result = _PyPegen_run_parser(p);
1489     _PyPegen_Parser_Free(p);
1490 
1491 error:
1492     PyTokenizer_Free(tok);
1493     return result;
1494 }
1495 
1496 asdl_stmt_seq*
_PyPegen_interactive_exit(Parser * p)1497 _PyPegen_interactive_exit(Parser *p)
1498 {
1499     if (p->errcode) {
1500         *(p->errcode) = E_EOF;
1501     }
1502     return NULL;
1503 }
1504 
1505 /* Creates a single-element asdl_seq* that contains a */
1506 asdl_seq *
_PyPegen_singleton_seq(Parser * p,void * a)1507 _PyPegen_singleton_seq(Parser *p, void *a)
1508 {
1509     assert(a != NULL);
1510     asdl_seq *seq = (asdl_seq*)_Py_asdl_generic_seq_new(1, p->arena);
1511     if (!seq) {
1512         return NULL;
1513     }
1514     asdl_seq_SET_UNTYPED(seq, 0, a);
1515     return seq;
1516 }
1517 
1518 /* Creates a copy of seq and prepends a to it */
1519 asdl_seq *
_PyPegen_seq_insert_in_front(Parser * p,void * a,asdl_seq * seq)1520 _PyPegen_seq_insert_in_front(Parser *p, void *a, asdl_seq *seq)
1521 {
1522     assert(a != NULL);
1523     if (!seq) {
1524         return _PyPegen_singleton_seq(p, a);
1525     }
1526 
1527     asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(asdl_seq_LEN(seq) + 1, p->arena);
1528     if (!new_seq) {
1529         return NULL;
1530     }
1531 
1532     asdl_seq_SET_UNTYPED(new_seq, 0, a);
1533     for (Py_ssize_t i = 1, l = asdl_seq_LEN(new_seq); i < l; i++) {
1534         asdl_seq_SET_UNTYPED(new_seq, i, asdl_seq_GET_UNTYPED(seq, i - 1));
1535     }
1536     return new_seq;
1537 }
1538 
1539 /* Creates a copy of seq and appends a to it */
1540 asdl_seq *
_PyPegen_seq_append_to_end(Parser * p,asdl_seq * seq,void * a)1541 _PyPegen_seq_append_to_end(Parser *p, asdl_seq *seq, void *a)
1542 {
1543     assert(a != NULL);
1544     if (!seq) {
1545         return _PyPegen_singleton_seq(p, a);
1546     }
1547 
1548     asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(asdl_seq_LEN(seq) + 1, p->arena);
1549     if (!new_seq) {
1550         return NULL;
1551     }
1552 
1553     for (Py_ssize_t i = 0, l = asdl_seq_LEN(new_seq); i + 1 < l; i++) {
1554         asdl_seq_SET_UNTYPED(new_seq, i, asdl_seq_GET_UNTYPED(seq, i));
1555     }
1556     asdl_seq_SET_UNTYPED(new_seq, asdl_seq_LEN(new_seq) - 1, a);
1557     return new_seq;
1558 }
1559 
1560 static Py_ssize_t
_get_flattened_seq_size(asdl_seq * seqs)1561 _get_flattened_seq_size(asdl_seq *seqs)
1562 {
1563     Py_ssize_t size = 0;
1564     for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) {
1565         asdl_seq *inner_seq = asdl_seq_GET_UNTYPED(seqs, i);
1566         size += asdl_seq_LEN(inner_seq);
1567     }
1568     return size;
1569 }
1570 
1571 /* Flattens an asdl_seq* of asdl_seq*s */
1572 asdl_seq *
_PyPegen_seq_flatten(Parser * p,asdl_seq * seqs)1573 _PyPegen_seq_flatten(Parser *p, asdl_seq *seqs)
1574 {
1575     Py_ssize_t flattened_seq_size = _get_flattened_seq_size(seqs);
1576     assert(flattened_seq_size > 0);
1577 
1578     asdl_seq *flattened_seq = (asdl_seq*)_Py_asdl_generic_seq_new(flattened_seq_size, p->arena);
1579     if (!flattened_seq) {
1580         return NULL;
1581     }
1582 
1583     int flattened_seq_idx = 0;
1584     for (Py_ssize_t i = 0, l = asdl_seq_LEN(seqs); i < l; i++) {
1585         asdl_seq *inner_seq = asdl_seq_GET_UNTYPED(seqs, i);
1586         for (Py_ssize_t j = 0, li = asdl_seq_LEN(inner_seq); j < li; j++) {
1587             asdl_seq_SET_UNTYPED(flattened_seq, flattened_seq_idx++, asdl_seq_GET_UNTYPED(inner_seq, j));
1588         }
1589     }
1590     assert(flattened_seq_idx == flattened_seq_size);
1591 
1592     return flattened_seq;
1593 }
1594 
1595 void *
_PyPegen_seq_last_item(asdl_seq * seq)1596 _PyPegen_seq_last_item(asdl_seq *seq)
1597 {
1598     Py_ssize_t len = asdl_seq_LEN(seq);
1599     return asdl_seq_GET_UNTYPED(seq, len - 1);
1600 }
1601 
1602 void *
_PyPegen_seq_first_item(asdl_seq * seq)1603 _PyPegen_seq_first_item(asdl_seq *seq)
1604 {
1605     return asdl_seq_GET_UNTYPED(seq, 0);
1606 }
1607 
1608 
1609 /* Creates a new name of the form <first_name>.<second_name> */
1610 expr_ty
_PyPegen_join_names_with_dot(Parser * p,expr_ty first_name,expr_ty second_name)1611 _PyPegen_join_names_with_dot(Parser *p, expr_ty first_name, expr_ty second_name)
1612 {
1613     assert(first_name != NULL && second_name != NULL);
1614     PyObject *first_identifier = first_name->v.Name.id;
1615     PyObject *second_identifier = second_name->v.Name.id;
1616 
1617     if (PyUnicode_READY(first_identifier) == -1) {
1618         return NULL;
1619     }
1620     if (PyUnicode_READY(second_identifier) == -1) {
1621         return NULL;
1622     }
1623     const char *first_str = PyUnicode_AsUTF8(first_identifier);
1624     if (!first_str) {
1625         return NULL;
1626     }
1627     const char *second_str = PyUnicode_AsUTF8(second_identifier);
1628     if (!second_str) {
1629         return NULL;
1630     }
1631     Py_ssize_t len = strlen(first_str) + strlen(second_str) + 1;  // +1 for the dot
1632 
1633     PyObject *str = PyBytes_FromStringAndSize(NULL, len);
1634     if (!str) {
1635         return NULL;
1636     }
1637 
1638     char *s = PyBytes_AS_STRING(str);
1639     if (!s) {
1640         return NULL;
1641     }
1642 
1643     strcpy(s, first_str);
1644     s += strlen(first_str);
1645     *s++ = '.';
1646     strcpy(s, second_str);
1647     s += strlen(second_str);
1648     *s = '\0';
1649 
1650     PyObject *uni = PyUnicode_DecodeUTF8(PyBytes_AS_STRING(str), PyBytes_GET_SIZE(str), NULL);
1651     Py_DECREF(str);
1652     if (!uni) {
1653         return NULL;
1654     }
1655     PyUnicode_InternInPlace(&uni);
1656     if (_PyArena_AddPyObject(p->arena, uni) < 0) {
1657         Py_DECREF(uni);
1658         return NULL;
1659     }
1660 
1661     return _PyAST_Name(uni, Load, EXTRA_EXPR(first_name, second_name));
1662 }
1663 
1664 /* Counts the total number of dots in seq's tokens */
1665 int
_PyPegen_seq_count_dots(asdl_seq * seq)1666 _PyPegen_seq_count_dots(asdl_seq *seq)
1667 {
1668     int number_of_dots = 0;
1669     for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) {
1670         Token *current_expr = asdl_seq_GET_UNTYPED(seq, i);
1671         switch (current_expr->type) {
1672             case ELLIPSIS:
1673                 number_of_dots += 3;
1674                 break;
1675             case DOT:
1676                 number_of_dots += 1;
1677                 break;
1678             default:
1679                 Py_UNREACHABLE();
1680         }
1681     }
1682 
1683     return number_of_dots;
1684 }
1685 
1686 /* Creates an alias with '*' as the identifier name */
1687 alias_ty
_PyPegen_alias_for_star(Parser * p,int lineno,int col_offset,int end_lineno,int end_col_offset,PyArena * arena)1688 _PyPegen_alias_for_star(Parser *p, int lineno, int col_offset, int end_lineno,
1689                         int end_col_offset, PyArena *arena) {
1690     PyObject *str = PyUnicode_InternFromString("*");
1691     if (!str) {
1692         return NULL;
1693     }
1694     if (_PyArena_AddPyObject(p->arena, str) < 0) {
1695         Py_DECREF(str);
1696         return NULL;
1697     }
1698     return _PyAST_alias(str, NULL, lineno, col_offset, end_lineno, end_col_offset, arena);
1699 }
1700 
1701 /* Creates a new asdl_seq* with the identifiers of all the names in seq */
1702 asdl_identifier_seq *
_PyPegen_map_names_to_ids(Parser * p,asdl_expr_seq * seq)1703 _PyPegen_map_names_to_ids(Parser *p, asdl_expr_seq *seq)
1704 {
1705     Py_ssize_t len = asdl_seq_LEN(seq);
1706     assert(len > 0);
1707 
1708     asdl_identifier_seq *new_seq = _Py_asdl_identifier_seq_new(len, p->arena);
1709     if (!new_seq) {
1710         return NULL;
1711     }
1712     for (Py_ssize_t i = 0; i < len; i++) {
1713         expr_ty e = asdl_seq_GET(seq, i);
1714         asdl_seq_SET(new_seq, i, e->v.Name.id);
1715     }
1716     return new_seq;
1717 }
1718 
1719 /* Constructs a CmpopExprPair */
1720 CmpopExprPair *
_PyPegen_cmpop_expr_pair(Parser * p,cmpop_ty cmpop,expr_ty expr)1721 _PyPegen_cmpop_expr_pair(Parser *p, cmpop_ty cmpop, expr_ty expr)
1722 {
1723     assert(expr != NULL);
1724     CmpopExprPair *a = _PyArena_Malloc(p->arena, sizeof(CmpopExprPair));
1725     if (!a) {
1726         return NULL;
1727     }
1728     a->cmpop = cmpop;
1729     a->expr = expr;
1730     return a;
1731 }
1732 
1733 asdl_int_seq *
_PyPegen_get_cmpops(Parser * p,asdl_seq * seq)1734 _PyPegen_get_cmpops(Parser *p, asdl_seq *seq)
1735 {
1736     Py_ssize_t len = asdl_seq_LEN(seq);
1737     assert(len > 0);
1738 
1739     asdl_int_seq *new_seq = _Py_asdl_int_seq_new(len, p->arena);
1740     if (!new_seq) {
1741         return NULL;
1742     }
1743     for (Py_ssize_t i = 0; i < len; i++) {
1744         CmpopExprPair *pair = asdl_seq_GET_UNTYPED(seq, i);
1745         asdl_seq_SET(new_seq, i, pair->cmpop);
1746     }
1747     return new_seq;
1748 }
1749 
1750 asdl_expr_seq *
_PyPegen_get_exprs(Parser * p,asdl_seq * seq)1751 _PyPegen_get_exprs(Parser *p, asdl_seq *seq)
1752 {
1753     Py_ssize_t len = asdl_seq_LEN(seq);
1754     assert(len > 0);
1755 
1756     asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
1757     if (!new_seq) {
1758         return NULL;
1759     }
1760     for (Py_ssize_t i = 0; i < len; i++) {
1761         CmpopExprPair *pair = asdl_seq_GET_UNTYPED(seq, i);
1762         asdl_seq_SET(new_seq, i, pair->expr);
1763     }
1764     return new_seq;
1765 }
1766 
1767 /* Creates an asdl_seq* where all the elements have been changed to have ctx as context */
1768 static asdl_expr_seq *
_set_seq_context(Parser * p,asdl_expr_seq * seq,expr_context_ty ctx)1769 _set_seq_context(Parser *p, asdl_expr_seq *seq, expr_context_ty ctx)
1770 {
1771     Py_ssize_t len = asdl_seq_LEN(seq);
1772     if (len == 0) {
1773         return NULL;
1774     }
1775 
1776     asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
1777     if (!new_seq) {
1778         return NULL;
1779     }
1780     for (Py_ssize_t i = 0; i < len; i++) {
1781         expr_ty e = asdl_seq_GET(seq, i);
1782         asdl_seq_SET(new_seq, i, _PyPegen_set_expr_context(p, e, ctx));
1783     }
1784     return new_seq;
1785 }
1786 
1787 static expr_ty
_set_name_context(Parser * p,expr_ty e,expr_context_ty ctx)1788 _set_name_context(Parser *p, expr_ty e, expr_context_ty ctx)
1789 {
1790     return _PyAST_Name(e->v.Name.id, ctx, EXTRA_EXPR(e, e));
1791 }
1792 
1793 static expr_ty
_set_tuple_context(Parser * p,expr_ty e,expr_context_ty ctx)1794 _set_tuple_context(Parser *p, expr_ty e, expr_context_ty ctx)
1795 {
1796     return _PyAST_Tuple(
1797             _set_seq_context(p, e->v.Tuple.elts, ctx),
1798             ctx,
1799             EXTRA_EXPR(e, e));
1800 }
1801 
1802 static expr_ty
_set_list_context(Parser * p,expr_ty e,expr_context_ty ctx)1803 _set_list_context(Parser *p, expr_ty e, expr_context_ty ctx)
1804 {
1805     return _PyAST_List(
1806             _set_seq_context(p, e->v.List.elts, ctx),
1807             ctx,
1808             EXTRA_EXPR(e, e));
1809 }
1810 
1811 static expr_ty
_set_subscript_context(Parser * p,expr_ty e,expr_context_ty ctx)1812 _set_subscript_context(Parser *p, expr_ty e, expr_context_ty ctx)
1813 {
1814     return _PyAST_Subscript(e->v.Subscript.value, e->v.Subscript.slice,
1815                             ctx, EXTRA_EXPR(e, e));
1816 }
1817 
1818 static expr_ty
_set_attribute_context(Parser * p,expr_ty e,expr_context_ty ctx)1819 _set_attribute_context(Parser *p, expr_ty e, expr_context_ty ctx)
1820 {
1821     return _PyAST_Attribute(e->v.Attribute.value, e->v.Attribute.attr,
1822                             ctx, EXTRA_EXPR(e, e));
1823 }
1824 
1825 static expr_ty
_set_starred_context(Parser * p,expr_ty e,expr_context_ty ctx)1826 _set_starred_context(Parser *p, expr_ty e, expr_context_ty ctx)
1827 {
1828     return _PyAST_Starred(_PyPegen_set_expr_context(p, e->v.Starred.value, ctx),
1829                           ctx, EXTRA_EXPR(e, e));
1830 }
1831 
1832 /* Creates an `expr_ty` equivalent to `expr` but with `ctx` as context */
1833 expr_ty
_PyPegen_set_expr_context(Parser * p,expr_ty expr,expr_context_ty ctx)1834 _PyPegen_set_expr_context(Parser *p, expr_ty expr, expr_context_ty ctx)
1835 {
1836     assert(expr != NULL);
1837 
1838     expr_ty new = NULL;
1839     switch (expr->kind) {
1840         case Name_kind:
1841             new = _set_name_context(p, expr, ctx);
1842             break;
1843         case Tuple_kind:
1844             new = _set_tuple_context(p, expr, ctx);
1845             break;
1846         case List_kind:
1847             new = _set_list_context(p, expr, ctx);
1848             break;
1849         case Subscript_kind:
1850             new = _set_subscript_context(p, expr, ctx);
1851             break;
1852         case Attribute_kind:
1853             new = _set_attribute_context(p, expr, ctx);
1854             break;
1855         case Starred_kind:
1856             new = _set_starred_context(p, expr, ctx);
1857             break;
1858         default:
1859             new = expr;
1860     }
1861     return new;
1862 }
1863 
1864 /* Constructs a KeyValuePair that is used when parsing a dict's key value pairs */
1865 KeyValuePair *
_PyPegen_key_value_pair(Parser * p,expr_ty key,expr_ty value)1866 _PyPegen_key_value_pair(Parser *p, expr_ty key, expr_ty value)
1867 {
1868     KeyValuePair *a = _PyArena_Malloc(p->arena, sizeof(KeyValuePair));
1869     if (!a) {
1870         return NULL;
1871     }
1872     a->key = key;
1873     a->value = value;
1874     return a;
1875 }
1876 
1877 /* Extracts all keys from an asdl_seq* of KeyValuePair*'s */
1878 asdl_expr_seq *
_PyPegen_get_keys(Parser * p,asdl_seq * seq)1879 _PyPegen_get_keys(Parser *p, asdl_seq *seq)
1880 {
1881     Py_ssize_t len = asdl_seq_LEN(seq);
1882     asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
1883     if (!new_seq) {
1884         return NULL;
1885     }
1886     for (Py_ssize_t i = 0; i < len; i++) {
1887         KeyValuePair *pair = asdl_seq_GET_UNTYPED(seq, i);
1888         asdl_seq_SET(new_seq, i, pair->key);
1889     }
1890     return new_seq;
1891 }
1892 
1893 /* Extracts all values from an asdl_seq* of KeyValuePair*'s */
1894 asdl_expr_seq *
_PyPegen_get_values(Parser * p,asdl_seq * seq)1895 _PyPegen_get_values(Parser *p, asdl_seq *seq)
1896 {
1897     Py_ssize_t len = asdl_seq_LEN(seq);
1898     asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
1899     if (!new_seq) {
1900         return NULL;
1901     }
1902     for (Py_ssize_t i = 0; i < len; i++) {
1903         KeyValuePair *pair = asdl_seq_GET_UNTYPED(seq, i);
1904         asdl_seq_SET(new_seq, i, pair->value);
1905     }
1906     return new_seq;
1907 }
1908 
1909 /* Constructs a KeyPatternPair that is used when parsing mapping & class patterns */
1910 KeyPatternPair *
_PyPegen_key_pattern_pair(Parser * p,expr_ty key,pattern_ty pattern)1911 _PyPegen_key_pattern_pair(Parser *p, expr_ty key, pattern_ty pattern)
1912 {
1913     KeyPatternPair *a = _PyArena_Malloc(p->arena, sizeof(KeyPatternPair));
1914     if (!a) {
1915         return NULL;
1916     }
1917     a->key = key;
1918     a->pattern = pattern;
1919     return a;
1920 }
1921 
1922 /* Extracts all keys from an asdl_seq* of KeyPatternPair*'s */
1923 asdl_expr_seq *
_PyPegen_get_pattern_keys(Parser * p,asdl_seq * seq)1924 _PyPegen_get_pattern_keys(Parser *p, asdl_seq *seq)
1925 {
1926     Py_ssize_t len = asdl_seq_LEN(seq);
1927     asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(len, p->arena);
1928     if (!new_seq) {
1929         return NULL;
1930     }
1931     for (Py_ssize_t i = 0; i < len; i++) {
1932         KeyPatternPair *pair = asdl_seq_GET_UNTYPED(seq, i);
1933         asdl_seq_SET(new_seq, i, pair->key);
1934     }
1935     return new_seq;
1936 }
1937 
1938 /* Extracts all patterns from an asdl_seq* of KeyPatternPair*'s */
1939 asdl_pattern_seq *
_PyPegen_get_patterns(Parser * p,asdl_seq * seq)1940 _PyPegen_get_patterns(Parser *p, asdl_seq *seq)
1941 {
1942     Py_ssize_t len = asdl_seq_LEN(seq);
1943     asdl_pattern_seq *new_seq = _Py_asdl_pattern_seq_new(len, p->arena);
1944     if (!new_seq) {
1945         return NULL;
1946     }
1947     for (Py_ssize_t i = 0; i < len; i++) {
1948         KeyPatternPair *pair = asdl_seq_GET_UNTYPED(seq, i);
1949         asdl_seq_SET(new_seq, i, pair->pattern);
1950     }
1951     return new_seq;
1952 }
1953 
1954 /* Constructs a NameDefaultPair */
1955 NameDefaultPair *
_PyPegen_name_default_pair(Parser * p,arg_ty arg,expr_ty value,Token * tc)1956 _PyPegen_name_default_pair(Parser *p, arg_ty arg, expr_ty value, Token *tc)
1957 {
1958     NameDefaultPair *a = _PyArena_Malloc(p->arena, sizeof(NameDefaultPair));
1959     if (!a) {
1960         return NULL;
1961     }
1962     a->arg = _PyPegen_add_type_comment_to_arg(p, arg, tc);
1963     a->value = value;
1964     return a;
1965 }
1966 
1967 /* Constructs a SlashWithDefault */
1968 SlashWithDefault *
_PyPegen_slash_with_default(Parser * p,asdl_arg_seq * plain_names,asdl_seq * names_with_defaults)1969 _PyPegen_slash_with_default(Parser *p, asdl_arg_seq *plain_names, asdl_seq *names_with_defaults)
1970 {
1971     SlashWithDefault *a = _PyArena_Malloc(p->arena, sizeof(SlashWithDefault));
1972     if (!a) {
1973         return NULL;
1974     }
1975     a->plain_names = plain_names;
1976     a->names_with_defaults = names_with_defaults;
1977     return a;
1978 }
1979 
1980 /* Constructs a StarEtc */
1981 StarEtc *
_PyPegen_star_etc(Parser * p,arg_ty vararg,asdl_seq * kwonlyargs,arg_ty kwarg)1982 _PyPegen_star_etc(Parser *p, arg_ty vararg, asdl_seq *kwonlyargs, arg_ty kwarg)
1983 {
1984     StarEtc *a = _PyArena_Malloc(p->arena, sizeof(StarEtc));
1985     if (!a) {
1986         return NULL;
1987     }
1988     a->vararg = vararg;
1989     a->kwonlyargs = kwonlyargs;
1990     a->kwarg = kwarg;
1991     return a;
1992 }
1993 
1994 asdl_seq *
_PyPegen_join_sequences(Parser * p,asdl_seq * a,asdl_seq * b)1995 _PyPegen_join_sequences(Parser *p, asdl_seq *a, asdl_seq *b)
1996 {
1997     Py_ssize_t first_len = asdl_seq_LEN(a);
1998     Py_ssize_t second_len = asdl_seq_LEN(b);
1999     asdl_seq *new_seq = (asdl_seq*)_Py_asdl_generic_seq_new(first_len + second_len, p->arena);
2000     if (!new_seq) {
2001         return NULL;
2002     }
2003 
2004     int k = 0;
2005     for (Py_ssize_t i = 0; i < first_len; i++) {
2006         asdl_seq_SET_UNTYPED(new_seq, k++, asdl_seq_GET_UNTYPED(a, i));
2007     }
2008     for (Py_ssize_t i = 0; i < second_len; i++) {
2009         asdl_seq_SET_UNTYPED(new_seq, k++, asdl_seq_GET_UNTYPED(b, i));
2010     }
2011 
2012     return new_seq;
2013 }
2014 
2015 static asdl_arg_seq*
_get_names(Parser * p,asdl_seq * names_with_defaults)2016 _get_names(Parser *p, asdl_seq *names_with_defaults)
2017 {
2018     Py_ssize_t len = asdl_seq_LEN(names_with_defaults);
2019     asdl_arg_seq *seq = _Py_asdl_arg_seq_new(len, p->arena);
2020     if (!seq) {
2021         return NULL;
2022     }
2023     for (Py_ssize_t i = 0; i < len; i++) {
2024         NameDefaultPair *pair = asdl_seq_GET_UNTYPED(names_with_defaults, i);
2025         asdl_seq_SET(seq, i, pair->arg);
2026     }
2027     return seq;
2028 }
2029 
2030 static asdl_expr_seq *
_get_defaults(Parser * p,asdl_seq * names_with_defaults)2031 _get_defaults(Parser *p, asdl_seq *names_with_defaults)
2032 {
2033     Py_ssize_t len = asdl_seq_LEN(names_with_defaults);
2034     asdl_expr_seq *seq = _Py_asdl_expr_seq_new(len, p->arena);
2035     if (!seq) {
2036         return NULL;
2037     }
2038     for (Py_ssize_t i = 0; i < len; i++) {
2039         NameDefaultPair *pair = asdl_seq_GET_UNTYPED(names_with_defaults, i);
2040         asdl_seq_SET(seq, i, pair->value);
2041     }
2042     return seq;
2043 }
2044 
2045 static int
_make_posonlyargs(Parser * p,asdl_arg_seq * slash_without_default,SlashWithDefault * slash_with_default,asdl_arg_seq ** posonlyargs)2046 _make_posonlyargs(Parser *p,
2047                   asdl_arg_seq *slash_without_default,
2048                   SlashWithDefault *slash_with_default,
2049                   asdl_arg_seq **posonlyargs) {
2050     if (slash_without_default != NULL) {
2051         *posonlyargs = slash_without_default;
2052     }
2053     else if (slash_with_default != NULL) {
2054         asdl_arg_seq *slash_with_default_names =
2055                 _get_names(p, slash_with_default->names_with_defaults);
2056         if (!slash_with_default_names) {
2057             return -1;
2058         }
2059         *posonlyargs = (asdl_arg_seq*)_PyPegen_join_sequences(
2060                 p,
2061                 (asdl_seq*)slash_with_default->plain_names,
2062                 (asdl_seq*)slash_with_default_names);
2063     }
2064     else {
2065         *posonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
2066     }
2067     return *posonlyargs == NULL ? -1 : 0;
2068 }
2069 
2070 static int
_make_posargs(Parser * p,asdl_arg_seq * plain_names,asdl_seq * names_with_default,asdl_arg_seq ** posargs)2071 _make_posargs(Parser *p,
2072               asdl_arg_seq *plain_names,
2073               asdl_seq *names_with_default,
2074               asdl_arg_seq **posargs) {
2075     if (plain_names != NULL && names_with_default != NULL) {
2076         asdl_arg_seq *names_with_default_names = _get_names(p, names_with_default);
2077         if (!names_with_default_names) {
2078             return -1;
2079         }
2080         *posargs = (asdl_arg_seq*)_PyPegen_join_sequences(
2081                 p,(asdl_seq*)plain_names, (asdl_seq*)names_with_default_names);
2082     }
2083     else if (plain_names == NULL && names_with_default != NULL) {
2084         *posargs = _get_names(p, names_with_default);
2085     }
2086     else if (plain_names != NULL && names_with_default == NULL) {
2087         *posargs = plain_names;
2088     }
2089     else {
2090         *posargs = _Py_asdl_arg_seq_new(0, p->arena);
2091     }
2092     return *posargs == NULL ? -1 : 0;
2093 }
2094 
2095 static int
_make_posdefaults(Parser * p,SlashWithDefault * slash_with_default,asdl_seq * names_with_default,asdl_expr_seq ** posdefaults)2096 _make_posdefaults(Parser *p,
2097                   SlashWithDefault *slash_with_default,
2098                   asdl_seq *names_with_default,
2099                   asdl_expr_seq **posdefaults) {
2100     if (slash_with_default != NULL && names_with_default != NULL) {
2101         asdl_expr_seq *slash_with_default_values =
2102                 _get_defaults(p, slash_with_default->names_with_defaults);
2103         if (!slash_with_default_values) {
2104             return -1;
2105         }
2106         asdl_expr_seq *names_with_default_values = _get_defaults(p, names_with_default);
2107         if (!names_with_default_values) {
2108             return -1;
2109         }
2110         *posdefaults = (asdl_expr_seq*)_PyPegen_join_sequences(
2111                 p,
2112                 (asdl_seq*)slash_with_default_values,
2113                 (asdl_seq*)names_with_default_values);
2114     }
2115     else if (slash_with_default == NULL && names_with_default != NULL) {
2116         *posdefaults = _get_defaults(p, names_with_default);
2117     }
2118     else if (slash_with_default != NULL && names_with_default == NULL) {
2119         *posdefaults = _get_defaults(p, slash_with_default->names_with_defaults);
2120     }
2121     else {
2122         *posdefaults = _Py_asdl_expr_seq_new(0, p->arena);
2123     }
2124     return *posdefaults == NULL ? -1 : 0;
2125 }
2126 
2127 static int
_make_kwargs(Parser * p,StarEtc * star_etc,asdl_arg_seq ** kwonlyargs,asdl_expr_seq ** kwdefaults)2128 _make_kwargs(Parser *p, StarEtc *star_etc,
2129              asdl_arg_seq **kwonlyargs,
2130              asdl_expr_seq **kwdefaults) {
2131     if (star_etc != NULL && star_etc->kwonlyargs != NULL) {
2132         *kwonlyargs = _get_names(p, star_etc->kwonlyargs);
2133     }
2134     else {
2135         *kwonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
2136     }
2137 
2138     if (*kwonlyargs == NULL) {
2139         return -1;
2140     }
2141 
2142     if (star_etc != NULL && star_etc->kwonlyargs != NULL) {
2143         *kwdefaults = _get_defaults(p, star_etc->kwonlyargs);
2144     }
2145     else {
2146         *kwdefaults = _Py_asdl_expr_seq_new(0, p->arena);
2147     }
2148 
2149     if (*kwdefaults == NULL) {
2150         return -1;
2151     }
2152 
2153     return 0;
2154 }
2155 
2156 /* Constructs an arguments_ty object out of all the parsed constructs in the parameters rule */
2157 arguments_ty
_PyPegen_make_arguments(Parser * p,asdl_arg_seq * slash_without_default,SlashWithDefault * slash_with_default,asdl_arg_seq * plain_names,asdl_seq * names_with_default,StarEtc * star_etc)2158 _PyPegen_make_arguments(Parser *p, asdl_arg_seq *slash_without_default,
2159                         SlashWithDefault *slash_with_default, asdl_arg_seq *plain_names,
2160                         asdl_seq *names_with_default, StarEtc *star_etc)
2161 {
2162     asdl_arg_seq *posonlyargs;
2163     if (_make_posonlyargs(p, slash_without_default, slash_with_default, &posonlyargs) == -1) {
2164         return NULL;
2165     }
2166 
2167     asdl_arg_seq *posargs;
2168     if (_make_posargs(p, plain_names, names_with_default, &posargs) == -1) {
2169         return NULL;
2170     }
2171 
2172     asdl_expr_seq *posdefaults;
2173     if (_make_posdefaults(p,slash_with_default, names_with_default, &posdefaults) == -1) {
2174         return NULL;
2175     }
2176 
2177     arg_ty vararg = NULL;
2178     if (star_etc != NULL && star_etc->vararg != NULL) {
2179         vararg = star_etc->vararg;
2180     }
2181 
2182     asdl_arg_seq *kwonlyargs;
2183     asdl_expr_seq *kwdefaults;
2184     if (_make_kwargs(p, star_etc, &kwonlyargs, &kwdefaults) == -1) {
2185         return NULL;
2186     }
2187 
2188     arg_ty kwarg = NULL;
2189     if (star_etc != NULL && star_etc->kwarg != NULL) {
2190         kwarg = star_etc->kwarg;
2191     }
2192 
2193     return _PyAST_arguments(posonlyargs, posargs, vararg, kwonlyargs,
2194                             kwdefaults, kwarg, posdefaults, p->arena);
2195 }
2196 
2197 
2198 /* Constructs an empty arguments_ty object, that gets used when a function accepts no
2199  * arguments. */
2200 arguments_ty
_PyPegen_empty_arguments(Parser * p)2201 _PyPegen_empty_arguments(Parser *p)
2202 {
2203     asdl_arg_seq *posonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
2204     if (!posonlyargs) {
2205         return NULL;
2206     }
2207     asdl_arg_seq *posargs = _Py_asdl_arg_seq_new(0, p->arena);
2208     if (!posargs) {
2209         return NULL;
2210     }
2211     asdl_expr_seq *posdefaults = _Py_asdl_expr_seq_new(0, p->arena);
2212     if (!posdefaults) {
2213         return NULL;
2214     }
2215     asdl_arg_seq *kwonlyargs = _Py_asdl_arg_seq_new(0, p->arena);
2216     if (!kwonlyargs) {
2217         return NULL;
2218     }
2219     asdl_expr_seq *kwdefaults = _Py_asdl_expr_seq_new(0, p->arena);
2220     if (!kwdefaults) {
2221         return NULL;
2222     }
2223 
2224     return _PyAST_arguments(posonlyargs, posargs, NULL, kwonlyargs,
2225                             kwdefaults, NULL, posdefaults, p->arena);
2226 }
2227 
2228 /* Encapsulates the value of an operator_ty into an AugOperator struct */
2229 AugOperator *
_PyPegen_augoperator(Parser * p,operator_ty kind)2230 _PyPegen_augoperator(Parser *p, operator_ty kind)
2231 {
2232     AugOperator *a = _PyArena_Malloc(p->arena, sizeof(AugOperator));
2233     if (!a) {
2234         return NULL;
2235     }
2236     a->kind = kind;
2237     return a;
2238 }
2239 
2240 /* Construct a FunctionDef equivalent to function_def, but with decorators */
2241 stmt_ty
_PyPegen_function_def_decorators(Parser * p,asdl_expr_seq * decorators,stmt_ty function_def)2242 _PyPegen_function_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty function_def)
2243 {
2244     assert(function_def != NULL);
2245     if (function_def->kind == AsyncFunctionDef_kind) {
2246         return _PyAST_AsyncFunctionDef(
2247             function_def->v.FunctionDef.name, function_def->v.FunctionDef.args,
2248             function_def->v.FunctionDef.body, decorators, function_def->v.FunctionDef.returns,
2249             function_def->v.FunctionDef.type_comment, function_def->lineno,
2250             function_def->col_offset, function_def->end_lineno, function_def->end_col_offset,
2251             p->arena);
2252     }
2253 
2254     return _PyAST_FunctionDef(
2255         function_def->v.FunctionDef.name, function_def->v.FunctionDef.args,
2256         function_def->v.FunctionDef.body, decorators,
2257         function_def->v.FunctionDef.returns,
2258         function_def->v.FunctionDef.type_comment, function_def->lineno,
2259         function_def->col_offset, function_def->end_lineno,
2260         function_def->end_col_offset, p->arena);
2261 }
2262 
2263 /* Construct a ClassDef equivalent to class_def, but with decorators */
2264 stmt_ty
_PyPegen_class_def_decorators(Parser * p,asdl_expr_seq * decorators,stmt_ty class_def)2265 _PyPegen_class_def_decorators(Parser *p, asdl_expr_seq *decorators, stmt_ty class_def)
2266 {
2267     assert(class_def != NULL);
2268     return _PyAST_ClassDef(
2269         class_def->v.ClassDef.name, class_def->v.ClassDef.bases,
2270         class_def->v.ClassDef.keywords, class_def->v.ClassDef.body, decorators,
2271         class_def->lineno, class_def->col_offset, class_def->end_lineno,
2272         class_def->end_col_offset, p->arena);
2273 }
2274 
2275 /* Construct a KeywordOrStarred */
2276 KeywordOrStarred *
_PyPegen_keyword_or_starred(Parser * p,void * element,int is_keyword)2277 _PyPegen_keyword_or_starred(Parser *p, void *element, int is_keyword)
2278 {
2279     KeywordOrStarred *a = _PyArena_Malloc(p->arena, sizeof(KeywordOrStarred));
2280     if (!a) {
2281         return NULL;
2282     }
2283     a->element = element;
2284     a->is_keyword = is_keyword;
2285     return a;
2286 }
2287 
2288 /* Get the number of starred expressions in an asdl_seq* of KeywordOrStarred*s */
2289 static int
_seq_number_of_starred_exprs(asdl_seq * seq)2290 _seq_number_of_starred_exprs(asdl_seq *seq)
2291 {
2292     int n = 0;
2293     for (Py_ssize_t i = 0, l = asdl_seq_LEN(seq); i < l; i++) {
2294         KeywordOrStarred *k = asdl_seq_GET_UNTYPED(seq, i);
2295         if (!k->is_keyword) {
2296             n++;
2297         }
2298     }
2299     return n;
2300 }
2301 
2302 /* Extract the starred expressions of an asdl_seq* of KeywordOrStarred*s */
2303 asdl_expr_seq *
_PyPegen_seq_extract_starred_exprs(Parser * p,asdl_seq * kwargs)2304 _PyPegen_seq_extract_starred_exprs(Parser *p, asdl_seq *kwargs)
2305 {
2306     int new_len = _seq_number_of_starred_exprs(kwargs);
2307     if (new_len == 0) {
2308         return NULL;
2309     }
2310     asdl_expr_seq *new_seq = _Py_asdl_expr_seq_new(new_len, p->arena);
2311     if (!new_seq) {
2312         return NULL;
2313     }
2314 
2315     int idx = 0;
2316     for (Py_ssize_t i = 0, len = asdl_seq_LEN(kwargs); i < len; i++) {
2317         KeywordOrStarred *k = asdl_seq_GET_UNTYPED(kwargs, i);
2318         if (!k->is_keyword) {
2319             asdl_seq_SET(new_seq, idx++, k->element);
2320         }
2321     }
2322     return new_seq;
2323 }
2324 
2325 /* Return a new asdl_seq* with only the keywords in kwargs */
2326 asdl_keyword_seq*
_PyPegen_seq_delete_starred_exprs(Parser * p,asdl_seq * kwargs)2327 _PyPegen_seq_delete_starred_exprs(Parser *p, asdl_seq *kwargs)
2328 {
2329     Py_ssize_t len = asdl_seq_LEN(kwargs);
2330     Py_ssize_t new_len = len - _seq_number_of_starred_exprs(kwargs);
2331     if (new_len == 0) {
2332         return NULL;
2333     }
2334     asdl_keyword_seq *new_seq = _Py_asdl_keyword_seq_new(new_len, p->arena);
2335     if (!new_seq) {
2336         return NULL;
2337     }
2338 
2339     int idx = 0;
2340     for (Py_ssize_t i = 0; i < len; i++) {
2341         KeywordOrStarred *k = asdl_seq_GET_UNTYPED(kwargs, i);
2342         if (k->is_keyword) {
2343             asdl_seq_SET(new_seq, idx++, k->element);
2344         }
2345     }
2346     return new_seq;
2347 }
2348 
2349 expr_ty
_PyPegen_concatenate_strings(Parser * p,asdl_seq * strings)2350 _PyPegen_concatenate_strings(Parser *p, asdl_seq *strings)
2351 {
2352     Py_ssize_t len = asdl_seq_LEN(strings);
2353     assert(len > 0);
2354 
2355     Token *first = asdl_seq_GET_UNTYPED(strings, 0);
2356     Token *last = asdl_seq_GET_UNTYPED(strings, len - 1);
2357 
2358     int bytesmode = 0;
2359     PyObject *bytes_str = NULL;
2360 
2361     FstringParser state;
2362     _PyPegen_FstringParser_Init(&state);
2363 
2364     for (Py_ssize_t i = 0; i < len; i++) {
2365         Token *t = asdl_seq_GET_UNTYPED(strings, i);
2366 
2367         int this_bytesmode;
2368         int this_rawmode;
2369         PyObject *s;
2370         const char *fstr;
2371         Py_ssize_t fstrlen = -1;
2372 
2373         if (_PyPegen_parsestr(p, &this_bytesmode, &this_rawmode, &s, &fstr, &fstrlen, t) != 0) {
2374             goto error;
2375         }
2376 
2377         /* Check that we are not mixing bytes with unicode. */
2378         if (i != 0 && bytesmode != this_bytesmode) {
2379             RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals");
2380             Py_XDECREF(s);
2381             goto error;
2382         }
2383         bytesmode = this_bytesmode;
2384 
2385         if (fstr != NULL) {
2386             assert(s == NULL && !bytesmode);
2387 
2388             int result = _PyPegen_FstringParser_ConcatFstring(p, &state, &fstr, fstr + fstrlen,
2389                                                      this_rawmode, 0, first, t, last);
2390             if (result < 0) {
2391                 goto error;
2392             }
2393         }
2394         else {
2395             /* String or byte string. */
2396             assert(s != NULL && fstr == NULL);
2397             assert(bytesmode ? PyBytes_CheckExact(s) : PyUnicode_CheckExact(s));
2398 
2399             if (bytesmode) {
2400                 if (i == 0) {
2401                     bytes_str = s;
2402                 }
2403                 else {
2404                     PyBytes_ConcatAndDel(&bytes_str, s);
2405                     if (!bytes_str) {
2406                         goto error;
2407                     }
2408                 }
2409             }
2410             else {
2411                 /* This is a regular string. Concatenate it. */
2412                 if (_PyPegen_FstringParser_ConcatAndDel(&state, s) < 0) {
2413                     goto error;
2414                 }
2415             }
2416         }
2417     }
2418 
2419     if (bytesmode) {
2420         if (_PyArena_AddPyObject(p->arena, bytes_str) < 0) {
2421             goto error;
2422         }
2423         return _PyAST_Constant(bytes_str, NULL, first->lineno,
2424                                first->col_offset, last->end_lineno,
2425                                last->end_col_offset, p->arena);
2426     }
2427 
2428     return _PyPegen_FstringParser_Finish(p, &state, first, last);
2429 
2430 error:
2431     Py_XDECREF(bytes_str);
2432     _PyPegen_FstringParser_Dealloc(&state);
2433     if (PyErr_Occurred()) {
2434         raise_decode_error(p);
2435     }
2436     return NULL;
2437 }
2438 
2439 expr_ty
_PyPegen_ensure_imaginary(Parser * p,expr_ty exp)2440 _PyPegen_ensure_imaginary(Parser *p, expr_ty exp)
2441 {
2442     if (exp->kind != Constant_kind || !PyComplex_CheckExact(exp->v.Constant.value)) {
2443         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(exp, "imaginary number required in complex literal");
2444         return NULL;
2445     }
2446     return exp;
2447 }
2448 
2449 expr_ty
_PyPegen_ensure_real(Parser * p,expr_ty exp)2450 _PyPegen_ensure_real(Parser *p, expr_ty exp)
2451 {
2452     if (exp->kind != Constant_kind || PyComplex_CheckExact(exp->v.Constant.value)) {
2453         RAISE_SYNTAX_ERROR_KNOWN_LOCATION(exp, "real number required in complex literal");
2454         return NULL;
2455     }
2456     return exp;
2457 }
2458 
2459 mod_ty
_PyPegen_make_module(Parser * p,asdl_stmt_seq * a)2460 _PyPegen_make_module(Parser *p, asdl_stmt_seq *a) {
2461     asdl_type_ignore_seq *type_ignores = NULL;
2462     Py_ssize_t num = p->type_ignore_comments.num_items;
2463     if (num > 0) {
2464         // Turn the raw (comment, lineno) pairs into TypeIgnore objects in the arena
2465         type_ignores = _Py_asdl_type_ignore_seq_new(num, p->arena);
2466         if (type_ignores == NULL) {
2467             return NULL;
2468         }
2469         for (int i = 0; i < num; i++) {
2470             PyObject *tag = _PyPegen_new_type_comment(p, p->type_ignore_comments.items[i].comment);
2471             if (tag == NULL) {
2472                 return NULL;
2473             }
2474             type_ignore_ty ti = _PyAST_TypeIgnore(p->type_ignore_comments.items[i].lineno,
2475                                                   tag, p->arena);
2476             if (ti == NULL) {
2477                 return NULL;
2478             }
2479             asdl_seq_SET(type_ignores, i, ti);
2480         }
2481     }
2482     return _PyAST_Module(a, type_ignores, p->arena);
2483 }
2484 
2485 // Error reporting helpers
2486 
2487 expr_ty
_PyPegen_get_invalid_target(expr_ty e,TARGETS_TYPE targets_type)2488 _PyPegen_get_invalid_target(expr_ty e, TARGETS_TYPE targets_type)
2489 {
2490     if (e == NULL) {
2491         return NULL;
2492     }
2493 
2494 #define VISIT_CONTAINER(CONTAINER, TYPE) do { \
2495         Py_ssize_t len = asdl_seq_LEN((CONTAINER)->v.TYPE.elts);\
2496         for (Py_ssize_t i = 0; i < len; i++) {\
2497             expr_ty other = asdl_seq_GET((CONTAINER)->v.TYPE.elts, i);\
2498             expr_ty child = _PyPegen_get_invalid_target(other, targets_type);\
2499             if (child != NULL) {\
2500                 return child;\
2501             }\
2502         }\
2503     } while (0)
2504 
2505     // We only need to visit List and Tuple nodes recursively as those
2506     // are the only ones that can contain valid names in targets when
2507     // they are parsed as expressions. Any other kind of expression
2508     // that is a container (like Sets or Dicts) is directly invalid and
2509     // we don't need to visit it recursively.
2510 
2511     switch (e->kind) {
2512         case List_kind:
2513             VISIT_CONTAINER(e, List);
2514             return NULL;
2515         case Tuple_kind:
2516             VISIT_CONTAINER(e, Tuple);
2517             return NULL;
2518         case Starred_kind:
2519             if (targets_type == DEL_TARGETS) {
2520                 return e;
2521             }
2522             return _PyPegen_get_invalid_target(e->v.Starred.value, targets_type);
2523         case Compare_kind:
2524             // This is needed, because the `a in b` in `for a in b` gets parsed
2525             // as a comparison, and so we need to search the left side of the comparison
2526             // for invalid targets.
2527             if (targets_type == FOR_TARGETS) {
2528                 cmpop_ty cmpop = (cmpop_ty) asdl_seq_GET(e->v.Compare.ops, 0);
2529                 if (cmpop == In) {
2530                     return _PyPegen_get_invalid_target(e->v.Compare.left, targets_type);
2531                 }
2532                 return NULL;
2533             }
2534             return e;
2535         case Name_kind:
2536         case Subscript_kind:
2537         case Attribute_kind:
2538             return NULL;
2539         default:
2540             return e;
2541     }
2542 }
2543 
_PyPegen_arguments_parsing_error(Parser * p,expr_ty e)2544 void *_PyPegen_arguments_parsing_error(Parser *p, expr_ty e) {
2545     int kwarg_unpacking = 0;
2546     for (Py_ssize_t i = 0, l = asdl_seq_LEN(e->v.Call.keywords); i < l; i++) {
2547         keyword_ty keyword = asdl_seq_GET(e->v.Call.keywords, i);
2548         if (!keyword->arg) {
2549             kwarg_unpacking = 1;
2550         }
2551     }
2552 
2553     const char *msg = NULL;
2554     if (kwarg_unpacking) {
2555         msg = "positional argument follows keyword argument unpacking";
2556     } else {
2557         msg = "positional argument follows keyword argument";
2558     }
2559 
2560     return RAISE_SYNTAX_ERROR(msg);
2561 }
2562 
2563 
2564 static inline expr_ty
_PyPegen_get_last_comprehension_item(comprehension_ty comprehension)2565 _PyPegen_get_last_comprehension_item(comprehension_ty comprehension) {
2566     if (comprehension->ifs == NULL || asdl_seq_LEN(comprehension->ifs) == 0) {
2567         return comprehension->iter;
2568     }
2569     return PyPegen_last_item(comprehension->ifs, expr_ty);
2570 }
2571 
2572 void *
_PyPegen_nonparen_genexp_in_call(Parser * p,expr_ty args,asdl_comprehension_seq * comprehensions)2573 _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq *comprehensions)
2574 {
2575     /* The rule that calls this function is 'args for_if_clauses'.
2576        For the input f(L, x for x in y), L and x are in args and
2577        the for is parsed as a for_if_clause. We have to check if
2578        len <= 1, so that input like dict((a, b) for a, b in x)
2579        gets successfully parsed and then we pass the last
2580        argument (x in the above example) as the location of the
2581        error */
2582     Py_ssize_t len = asdl_seq_LEN(args->v.Call.args);
2583     if (len <= 1) {
2584         return NULL;
2585     }
2586 
2587     comprehension_ty last_comprehension = PyPegen_last_item(comprehensions, comprehension_ty);
2588 
2589     return RAISE_SYNTAX_ERROR_KNOWN_RANGE(
2590         (expr_ty) asdl_seq_GET(args->v.Call.args, len - 1),
2591         _PyPegen_get_last_comprehension_item(last_comprehension),
2592         "Generator expression must be parenthesized"
2593     );
2594 }
2595 
2596 
_PyPegen_collect_call_seqs(Parser * p,asdl_expr_seq * a,asdl_seq * b,int lineno,int col_offset,int end_lineno,int end_col_offset,PyArena * arena)2597 expr_ty _PyPegen_collect_call_seqs(Parser *p, asdl_expr_seq *a, asdl_seq *b,
2598                      int lineno, int col_offset, int end_lineno,
2599                      int end_col_offset, PyArena *arena) {
2600     Py_ssize_t args_len = asdl_seq_LEN(a);
2601     Py_ssize_t total_len = args_len;
2602 
2603     if (b == NULL) {
2604         return _PyAST_Call(_PyPegen_dummy_name(p), a, NULL, lineno, col_offset,
2605                         end_lineno, end_col_offset, arena);
2606 
2607     }
2608 
2609     asdl_expr_seq *starreds = _PyPegen_seq_extract_starred_exprs(p, b);
2610     asdl_keyword_seq *keywords = _PyPegen_seq_delete_starred_exprs(p, b);
2611 
2612     if (starreds) {
2613         total_len += asdl_seq_LEN(starreds);
2614     }
2615 
2616     asdl_expr_seq *args = _Py_asdl_expr_seq_new(total_len, arena);
2617 
2618     Py_ssize_t i = 0;
2619     for (i = 0; i < args_len; i++) {
2620         asdl_seq_SET(args, i, asdl_seq_GET(a, i));
2621     }
2622     for (; i < total_len; i++) {
2623         asdl_seq_SET(args, i, asdl_seq_GET(starreds, i - args_len));
2624     }
2625 
2626     return _PyAST_Call(_PyPegen_dummy_name(p), args, keywords, lineno,
2627                        col_offset, end_lineno, end_col_offset, arena);
2628 }
2629