1 #include "Python.h"
2 #include "pycore_ast.h" // identifier, stmt_ty
3 #include "pycore_compile.h" // _Py_Mangle(), _PyFuture_FromAST()
4 #include "pycore_parser.h" // _PyParser_ASTFromString()
5 #include "pycore_pystate.h" // _PyThreadState_GET()
6 #include "pycore_symtable.h" // PySTEntryObject
7 #include "structmember.h" // PyMemberDef
8
9 /* error strings used for warnings */
10 #define GLOBAL_PARAM \
11 "name '%U' is parameter and global"
12
13 #define NONLOCAL_PARAM \
14 "name '%U' is parameter and nonlocal"
15
16 #define GLOBAL_AFTER_ASSIGN \
17 "name '%U' is assigned to before global declaration"
18
19 #define NONLOCAL_AFTER_ASSIGN \
20 "name '%U' is assigned to before nonlocal declaration"
21
22 #define GLOBAL_AFTER_USE \
23 "name '%U' is used prior to global declaration"
24
25 #define NONLOCAL_AFTER_USE \
26 "name '%U' is used prior to nonlocal declaration"
27
28 #define GLOBAL_ANNOT \
29 "annotated name '%U' can't be global"
30
31 #define NONLOCAL_ANNOT \
32 "annotated name '%U' can't be nonlocal"
33
34 #define IMPORT_STAR_WARNING "import * only allowed at module level"
35
36 #define NAMED_EXPR_COMP_IN_CLASS \
37 "assignment expression within a comprehension cannot be used in a class body"
38
39 #define NAMED_EXPR_COMP_CONFLICT \
40 "assignment expression cannot rebind comprehension iteration variable '%U'"
41
42 #define NAMED_EXPR_COMP_INNER_LOOP_CONFLICT \
43 "comprehension inner loop cannot rebind assignment expression target '%U'"
44
45 #define NAMED_EXPR_COMP_ITER_EXPR \
46 "assignment expression cannot be used in a comprehension iterable expression"
47
48 #define ANNOTATION_NOT_ALLOWED \
49 "'%s' can not be used within an annotation"
50
51
52 #define LOCATION(x) \
53 (x)->lineno, (x)->col_offset, (x)->end_lineno, (x)->end_col_offset
54
55 #define ST_LOCATION(x) \
56 (x)->ste_lineno, (x)->ste_col_offset, (x)->ste_end_lineno, (x)->ste_end_col_offset
57
58 static PySTEntryObject *
ste_new(struct symtable * st,identifier name,_Py_block_ty block,void * key,int lineno,int col_offset,int end_lineno,int end_col_offset)59 ste_new(struct symtable *st, identifier name, _Py_block_ty block,
60 void *key, int lineno, int col_offset,
61 int end_lineno, int end_col_offset)
62 {
63 PySTEntryObject *ste = NULL;
64 PyObject *k = NULL;
65
66 k = PyLong_FromVoidPtr(key);
67 if (k == NULL)
68 goto fail;
69 ste = PyObject_New(PySTEntryObject, &PySTEntry_Type);
70 if (ste == NULL) {
71 Py_DECREF(k);
72 goto fail;
73 }
74 ste->ste_table = st;
75 ste->ste_id = k; /* ste owns reference to k */
76
77 Py_INCREF(name);
78 ste->ste_name = name;
79
80 ste->ste_symbols = NULL;
81 ste->ste_varnames = NULL;
82 ste->ste_children = NULL;
83
84 ste->ste_directives = NULL;
85
86 ste->ste_type = block;
87 ste->ste_nested = 0;
88 ste->ste_free = 0;
89 ste->ste_varargs = 0;
90 ste->ste_varkeywords = 0;
91 ste->ste_opt_lineno = 0;
92 ste->ste_opt_col_offset = 0;
93 ste->ste_lineno = lineno;
94 ste->ste_col_offset = col_offset;
95 ste->ste_end_lineno = end_lineno;
96 ste->ste_end_col_offset = end_col_offset;
97
98 if (st->st_cur != NULL &&
99 (st->st_cur->ste_nested ||
100 st->st_cur->ste_type == FunctionBlock))
101 ste->ste_nested = 1;
102 ste->ste_child_free = 0;
103 ste->ste_generator = 0;
104 ste->ste_coroutine = 0;
105 ste->ste_comprehension = NoComprehension;
106 ste->ste_returns_value = 0;
107 ste->ste_needs_class_closure = 0;
108 ste->ste_comp_iter_target = 0;
109 ste->ste_comp_iter_expr = 0;
110
111 ste->ste_symbols = PyDict_New();
112 ste->ste_varnames = PyList_New(0);
113 ste->ste_children = PyList_New(0);
114 if (ste->ste_symbols == NULL
115 || ste->ste_varnames == NULL
116 || ste->ste_children == NULL)
117 goto fail;
118
119 if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0)
120 goto fail;
121
122 return ste;
123 fail:
124 Py_XDECREF(ste);
125 return NULL;
126 }
127
128 static PyObject *
ste_repr(PySTEntryObject * ste)129 ste_repr(PySTEntryObject *ste)
130 {
131 return PyUnicode_FromFormat("<symtable entry %U(%ld), line %d>",
132 ste->ste_name,
133 PyLong_AS_LONG(ste->ste_id), ste->ste_lineno);
134 }
135
136 static void
ste_dealloc(PySTEntryObject * ste)137 ste_dealloc(PySTEntryObject *ste)
138 {
139 ste->ste_table = NULL;
140 Py_XDECREF(ste->ste_id);
141 Py_XDECREF(ste->ste_name);
142 Py_XDECREF(ste->ste_symbols);
143 Py_XDECREF(ste->ste_varnames);
144 Py_XDECREF(ste->ste_children);
145 Py_XDECREF(ste->ste_directives);
146 PyObject_Free(ste);
147 }
148
149 #define OFF(x) offsetof(PySTEntryObject, x)
150
151 static PyMemberDef ste_memberlist[] = {
152 {"id", T_OBJECT, OFF(ste_id), READONLY},
153 {"name", T_OBJECT, OFF(ste_name), READONLY},
154 {"symbols", T_OBJECT, OFF(ste_symbols), READONLY},
155 {"varnames", T_OBJECT, OFF(ste_varnames), READONLY},
156 {"children", T_OBJECT, OFF(ste_children), READONLY},
157 {"nested", T_INT, OFF(ste_nested), READONLY},
158 {"type", T_INT, OFF(ste_type), READONLY},
159 {"lineno", T_INT, OFF(ste_lineno), READONLY},
160 {NULL}
161 };
162
163 PyTypeObject PySTEntry_Type = {
164 PyVarObject_HEAD_INIT(&PyType_Type, 0)
165 "symtable entry",
166 sizeof(PySTEntryObject),
167 0,
168 (destructor)ste_dealloc, /* tp_dealloc */
169 0, /* tp_vectorcall_offset */
170 0, /* tp_getattr */
171 0, /* tp_setattr */
172 0, /* tp_as_async */
173 (reprfunc)ste_repr, /* tp_repr */
174 0, /* tp_as_number */
175 0, /* tp_as_sequence */
176 0, /* tp_as_mapping */
177 0, /* tp_hash */
178 0, /* tp_call */
179 0, /* tp_str */
180 PyObject_GenericGetAttr, /* tp_getattro */
181 0, /* tp_setattro */
182 0, /* tp_as_buffer */
183 Py_TPFLAGS_DEFAULT, /* tp_flags */
184 0, /* tp_doc */
185 0, /* tp_traverse */
186 0, /* tp_clear */
187 0, /* tp_richcompare */
188 0, /* tp_weaklistoffset */
189 0, /* tp_iter */
190 0, /* tp_iternext */
191 0, /* tp_methods */
192 ste_memberlist, /* tp_members */
193 0, /* tp_getset */
194 0, /* tp_base */
195 0, /* tp_dict */
196 0, /* tp_descr_get */
197 0, /* tp_descr_set */
198 0, /* tp_dictoffset */
199 0, /* tp_init */
200 0, /* tp_alloc */
201 0, /* tp_new */
202 };
203
204 static int symtable_analyze(struct symtable *st);
205 static int symtable_enter_block(struct symtable *st, identifier name,
206 _Py_block_ty block, void *ast,
207 int lineno, int col_offset,
208 int end_lineno, int end_col_offset);
209 static int symtable_exit_block(struct symtable *st);
210 static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
211 static int symtable_visit_expr(struct symtable *st, expr_ty s);
212 static int symtable_visit_genexp(struct symtable *st, expr_ty s);
213 static int symtable_visit_listcomp(struct symtable *st, expr_ty s);
214 static int symtable_visit_setcomp(struct symtable *st, expr_ty s);
215 static int symtable_visit_dictcomp(struct symtable *st, expr_ty s);
216 static int symtable_visit_arguments(struct symtable *st, arguments_ty);
217 static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty);
218 static int symtable_visit_alias(struct symtable *st, alias_ty);
219 static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
220 static int symtable_visit_keyword(struct symtable *st, keyword_ty);
221 static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args);
222 static int symtable_visit_annotation(struct symtable *st, expr_ty annotation);
223 static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args);
224 static int symtable_implicit_arg(struct symtable *st, int pos);
225 static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty);
226 static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
227 static int symtable_visit_match_case(struct symtable *st, match_case_ty m);
228 static int symtable_visit_pattern(struct symtable *st, pattern_ty s);
229 static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty);
230 static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty);
231
232
233 static identifier top = NULL, lambda = NULL, genexpr = NULL,
234 listcomp = NULL, setcomp = NULL, dictcomp = NULL,
235 __class__ = NULL, _annotation = NULL;
236
237 #define GET_IDENTIFIER(VAR) \
238 ((VAR) ? (VAR) : ((VAR) = PyUnicode_InternFromString(# VAR)))
239
240 #define DUPLICATE_ARGUMENT \
241 "duplicate argument '%U' in function definition"
242
243 static struct symtable *
symtable_new(void)244 symtable_new(void)
245 {
246 struct symtable *st;
247
248 st = (struct symtable *)PyMem_Malloc(sizeof(struct symtable));
249 if (st == NULL) {
250 PyErr_NoMemory();
251 return NULL;
252 }
253
254 st->st_filename = NULL;
255 st->st_blocks = NULL;
256
257 if ((st->st_stack = PyList_New(0)) == NULL)
258 goto fail;
259 if ((st->st_blocks = PyDict_New()) == NULL)
260 goto fail;
261 st->st_cur = NULL;
262 st->st_private = NULL;
263 return st;
264 fail:
265 _PySymtable_Free(st);
266 return NULL;
267 }
268
269 /* When compiling the use of C stack is probably going to be a lot
270 lighter than when executing Python code but still can overflow
271 and causing a Python crash if not checked (e.g. eval("()"*300000)).
272 Using the current recursion limit for the compiler seems too
273 restrictive (it caused at least one test to fail) so a factor is
274 used to allow deeper recursion when compiling an expression.
275
276 Using a scaling factor means this should automatically adjust when
277 the recursion limit is adjusted for small or large C stack allocations.
278 */
279 #define COMPILER_STACK_FRAME_SCALE 3
280
281 struct symtable *
_PySymtable_Build(mod_ty mod,PyObject * filename,PyFutureFeatures * future)282 _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
283 {
284 struct symtable *st = symtable_new();
285 asdl_stmt_seq *seq;
286 int i;
287 PyThreadState *tstate;
288 int recursion_limit = Py_GetRecursionLimit();
289 int starting_recursion_depth;
290
291 if (st == NULL)
292 return NULL;
293 if (filename == NULL) {
294 _PySymtable_Free(st);
295 return NULL;
296 }
297 Py_INCREF(filename);
298 st->st_filename = filename;
299 st->st_future = future;
300
301 /* Setup recursion depth check counters */
302 tstate = _PyThreadState_GET();
303 if (!tstate) {
304 _PySymtable_Free(st);
305 return NULL;
306 }
307 /* Be careful here to prevent overflow. */
308 starting_recursion_depth = (tstate->recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
309 tstate->recursion_depth * COMPILER_STACK_FRAME_SCALE : tstate->recursion_depth;
310 st->recursion_depth = starting_recursion_depth;
311 st->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
312 recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit;
313
314 /* Make the initial symbol information gathering pass */
315 if (!GET_IDENTIFIER(top) ||
316 !symtable_enter_block(st, top, ModuleBlock, (void *)mod, 0, 0, 0, 0)) {
317 _PySymtable_Free(st);
318 return NULL;
319 }
320
321 st->st_top = st->st_cur;
322 switch (mod->kind) {
323 case Module_kind:
324 seq = mod->v.Module.body;
325 for (i = 0; i < asdl_seq_LEN(seq); i++)
326 if (!symtable_visit_stmt(st,
327 (stmt_ty)asdl_seq_GET(seq, i)))
328 goto error;
329 break;
330 case Expression_kind:
331 if (!symtable_visit_expr(st, mod->v.Expression.body))
332 goto error;
333 break;
334 case Interactive_kind:
335 seq = mod->v.Interactive.body;
336 for (i = 0; i < asdl_seq_LEN(seq); i++)
337 if (!symtable_visit_stmt(st,
338 (stmt_ty)asdl_seq_GET(seq, i)))
339 goto error;
340 break;
341 case FunctionType_kind:
342 PyErr_SetString(PyExc_RuntimeError,
343 "this compiler does not handle FunctionTypes");
344 goto error;
345 }
346 if (!symtable_exit_block(st)) {
347 _PySymtable_Free(st);
348 return NULL;
349 }
350 /* Check that the recursion depth counting balanced correctly */
351 if (st->recursion_depth != starting_recursion_depth) {
352 PyErr_Format(PyExc_SystemError,
353 "symtable analysis recursion depth mismatch (before=%d, after=%d)",
354 starting_recursion_depth, st->recursion_depth);
355 _PySymtable_Free(st);
356 return NULL;
357 }
358 /* Make the second symbol analysis pass */
359 if (symtable_analyze(st))
360 return st;
361 _PySymtable_Free(st);
362 return NULL;
363 error:
364 (void) symtable_exit_block(st);
365 _PySymtable_Free(st);
366 return NULL;
367 }
368
369
370 void
_PySymtable_Free(struct symtable * st)371 _PySymtable_Free(struct symtable *st)
372 {
373 Py_XDECREF(st->st_filename);
374 Py_XDECREF(st->st_blocks);
375 Py_XDECREF(st->st_stack);
376 PyMem_Free((void *)st);
377 }
378
379 PySTEntryObject *
PySymtable_Lookup(struct symtable * st,void * key)380 PySymtable_Lookup(struct symtable *st, void *key)
381 {
382 PyObject *k, *v;
383
384 k = PyLong_FromVoidPtr(key);
385 if (k == NULL)
386 return NULL;
387 v = PyDict_GetItemWithError(st->st_blocks, k);
388 if (v) {
389 assert(PySTEntry_Check(v));
390 Py_INCREF(v);
391 }
392 else if (!PyErr_Occurred()) {
393 PyErr_SetString(PyExc_KeyError,
394 "unknown symbol table entry");
395 }
396
397 Py_DECREF(k);
398 return (PySTEntryObject *)v;
399 }
400
401 static long
_PyST_GetSymbol(PySTEntryObject * ste,PyObject * name)402 _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name)
403 {
404 PyObject *v = PyDict_GetItemWithError(ste->ste_symbols, name);
405 if (!v)
406 return 0;
407 assert(PyLong_Check(v));
408 return PyLong_AS_LONG(v);
409 }
410
411 int
_PyST_GetScope(PySTEntryObject * ste,PyObject * name)412 _PyST_GetScope(PySTEntryObject *ste, PyObject *name)
413 {
414 long symbol = _PyST_GetSymbol(ste, name);
415 return (symbol >> SCOPE_OFFSET) & SCOPE_MASK;
416 }
417
418 static int
error_at_directive(PySTEntryObject * ste,PyObject * name)419 error_at_directive(PySTEntryObject *ste, PyObject *name)
420 {
421 Py_ssize_t i;
422 PyObject *data;
423 assert(ste->ste_directives);
424 for (i = 0; i < PyList_GET_SIZE(ste->ste_directives); i++) {
425 data = PyList_GET_ITEM(ste->ste_directives, i);
426 assert(PyTuple_CheckExact(data));
427 assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(data, 0)));
428 if (PyUnicode_Compare(PyTuple_GET_ITEM(data, 0), name) == 0) {
429 PyErr_RangedSyntaxLocationObject(ste->ste_table->st_filename,
430 PyLong_AsLong(PyTuple_GET_ITEM(data, 1)),
431 PyLong_AsLong(PyTuple_GET_ITEM(data, 2)) + 1,
432 PyLong_AsLong(PyTuple_GET_ITEM(data, 3)),
433 PyLong_AsLong(PyTuple_GET_ITEM(data, 4)) + 1);
434
435 return 0;
436 }
437 }
438 PyErr_SetString(PyExc_RuntimeError,
439 "BUG: internal directive bookkeeping broken");
440 return 0;
441 }
442
443
444 /* Analyze raw symbol information to determine scope of each name.
445
446 The next several functions are helpers for symtable_analyze(),
447 which determines whether a name is local, global, or free. In addition,
448 it determines which local variables are cell variables; they provide
449 bindings that are used for free variables in enclosed blocks.
450
451 There are also two kinds of global variables, implicit and explicit. An
452 explicit global is declared with the global statement. An implicit
453 global is a free variable for which the compiler has found no binding
454 in an enclosing function scope. The implicit global is either a global
455 or a builtin. Python's module and class blocks use the xxx_NAME opcodes
456 to handle these names to implement slightly odd semantics. In such a
457 block, the name is treated as global until it is assigned to; then it
458 is treated as a local.
459
460 The symbol table requires two passes to determine the scope of each name.
461 The first pass collects raw facts from the AST via the symtable_visit_*
462 functions: the name is a parameter here, the name is used but not defined
463 here, etc. The second pass analyzes these facts during a pass over the
464 PySTEntryObjects created during pass 1.
465
466 When a function is entered during the second pass, the parent passes
467 the set of all name bindings visible to its children. These bindings
468 are used to determine if non-local variables are free or implicit globals.
469 Names which are explicitly declared nonlocal must exist in this set of
470 visible names - if they do not, a syntax error is raised. After doing
471 the local analysis, it analyzes each of its child blocks using an
472 updated set of name bindings.
473
474 The children update the free variable set. If a local variable is added to
475 the free variable set by the child, the variable is marked as a cell. The
476 function object being defined must provide runtime storage for the variable
477 that may outlive the function's frame. Cell variables are removed from the
478 free set before the analyze function returns to its parent.
479
480 During analysis, the names are:
481 symbols: dict mapping from symbol names to flag values (including offset scope values)
482 scopes: dict mapping from symbol names to scope values (no offset)
483 local: set of all symbol names local to the current scope
484 bound: set of all symbol names local to a containing function scope
485 free: set of all symbol names referenced but not bound in child scopes
486 global: set of all symbol names explicitly declared as global
487 */
488
489 #define SET_SCOPE(DICT, NAME, I) { \
490 PyObject *o = PyLong_FromLong(I); \
491 if (!o) \
492 return 0; \
493 if (PyDict_SetItem((DICT), (NAME), o) < 0) { \
494 Py_DECREF(o); \
495 return 0; \
496 } \
497 Py_DECREF(o); \
498 }
499
500 /* Decide on scope of name, given flags.
501
502 The namespace dictionaries may be modified to record information
503 about the new name. For example, a new global will add an entry to
504 global. A name that was global can be changed to local.
505 */
506
507 static int
analyze_name(PySTEntryObject * ste,PyObject * scopes,PyObject * name,long flags,PyObject * bound,PyObject * local,PyObject * free,PyObject * global)508 analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
509 PyObject *bound, PyObject *local, PyObject *free,
510 PyObject *global)
511 {
512 if (flags & DEF_GLOBAL) {
513 if (flags & DEF_NONLOCAL) {
514 PyErr_Format(PyExc_SyntaxError,
515 "name '%U' is nonlocal and global",
516 name);
517 return error_at_directive(ste, name);
518 }
519 SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
520 if (PySet_Add(global, name) < 0)
521 return 0;
522 if (bound && (PySet_Discard(bound, name) < 0))
523 return 0;
524 return 1;
525 }
526 if (flags & DEF_NONLOCAL) {
527 if (!bound) {
528 PyErr_Format(PyExc_SyntaxError,
529 "nonlocal declaration not allowed at module level");
530 return error_at_directive(ste, name);
531 }
532 if (!PySet_Contains(bound, name)) {
533 PyErr_Format(PyExc_SyntaxError,
534 "no binding for nonlocal '%U' found",
535 name);
536
537 return error_at_directive(ste, name);
538 }
539 SET_SCOPE(scopes, name, FREE);
540 ste->ste_free = 1;
541 return PySet_Add(free, name) >= 0;
542 }
543 if (flags & DEF_BOUND) {
544 SET_SCOPE(scopes, name, LOCAL);
545 if (PySet_Add(local, name) < 0)
546 return 0;
547 if (PySet_Discard(global, name) < 0)
548 return 0;
549 return 1;
550 }
551 /* If an enclosing block has a binding for this name, it
552 is a free variable rather than a global variable.
553 Note that having a non-NULL bound implies that the block
554 is nested.
555 */
556 if (bound && PySet_Contains(bound, name)) {
557 SET_SCOPE(scopes, name, FREE);
558 ste->ste_free = 1;
559 return PySet_Add(free, name) >= 0;
560 }
561 /* If a parent has a global statement, then call it global
562 explicit? It could also be global implicit.
563 */
564 if (global && PySet_Contains(global, name)) {
565 SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
566 return 1;
567 }
568 if (ste->ste_nested)
569 ste->ste_free = 1;
570 SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
571 return 1;
572 }
573
574 #undef SET_SCOPE
575
576 /* If a name is defined in free and also in locals, then this block
577 provides the binding for the free variable. The name should be
578 marked CELL in this block and removed from the free list.
579
580 Note that the current block's free variables are included in free.
581 That's safe because no name can be free and local in the same scope.
582 */
583
584 static int
analyze_cells(PyObject * scopes,PyObject * free)585 analyze_cells(PyObject *scopes, PyObject *free)
586 {
587 PyObject *name, *v, *v_cell;
588 int success = 0;
589 Py_ssize_t pos = 0;
590
591 v_cell = PyLong_FromLong(CELL);
592 if (!v_cell)
593 return 0;
594 while (PyDict_Next(scopes, &pos, &name, &v)) {
595 long scope;
596 assert(PyLong_Check(v));
597 scope = PyLong_AS_LONG(v);
598 if (scope != LOCAL)
599 continue;
600 if (!PySet_Contains(free, name))
601 continue;
602 /* Replace LOCAL with CELL for this name, and remove
603 from free. It is safe to replace the value of name
604 in the dict, because it will not cause a resize.
605 */
606 if (PyDict_SetItem(scopes, name, v_cell) < 0)
607 goto error;
608 if (PySet_Discard(free, name) < 0)
609 goto error;
610 }
611 success = 1;
612 error:
613 Py_DECREF(v_cell);
614 return success;
615 }
616
617 static int
drop_class_free(PySTEntryObject * ste,PyObject * free)618 drop_class_free(PySTEntryObject *ste, PyObject *free)
619 {
620 int res;
621 if (!GET_IDENTIFIER(__class__))
622 return 0;
623 res = PySet_Discard(free, __class__);
624 if (res < 0)
625 return 0;
626 if (res)
627 ste->ste_needs_class_closure = 1;
628 return 1;
629 }
630
631 /* Enter the final scope information into the ste_symbols dict.
632 *
633 * All arguments are dicts. Modifies symbols, others are read-only.
634 */
635 static int
update_symbols(PyObject * symbols,PyObject * scopes,PyObject * bound,PyObject * free,int classflag)636 update_symbols(PyObject *symbols, PyObject *scopes,
637 PyObject *bound, PyObject *free, int classflag)
638 {
639 PyObject *name = NULL, *itr = NULL;
640 PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL;
641 Py_ssize_t pos = 0;
642
643 /* Update scope information for all symbols in this scope */
644 while (PyDict_Next(symbols, &pos, &name, &v)) {
645 long scope, flags;
646 assert(PyLong_Check(v));
647 flags = PyLong_AS_LONG(v);
648 v_scope = PyDict_GetItemWithError(scopes, name);
649 assert(v_scope && PyLong_Check(v_scope));
650 scope = PyLong_AS_LONG(v_scope);
651 flags |= (scope << SCOPE_OFFSET);
652 v_new = PyLong_FromLong(flags);
653 if (!v_new)
654 return 0;
655 if (PyDict_SetItem(symbols, name, v_new) < 0) {
656 Py_DECREF(v_new);
657 return 0;
658 }
659 Py_DECREF(v_new);
660 }
661
662 /* Record not yet resolved free variables from children (if any) */
663 v_free = PyLong_FromLong(FREE << SCOPE_OFFSET);
664 if (!v_free)
665 return 0;
666
667 itr = PyObject_GetIter(free);
668 if (itr == NULL) {
669 Py_DECREF(v_free);
670 return 0;
671 }
672
673 while ((name = PyIter_Next(itr))) {
674 v = PyDict_GetItemWithError(symbols, name);
675
676 /* Handle symbol that already exists in this scope */
677 if (v) {
678 /* Handle a free variable in a method of
679 the class that has the same name as a local
680 or global in the class scope.
681 */
682 if (classflag &&
683 PyLong_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) {
684 long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS;
685 v_new = PyLong_FromLong(flags);
686 if (!v_new) {
687 goto error;
688 }
689 if (PyDict_SetItem(symbols, name, v_new) < 0) {
690 Py_DECREF(v_new);
691 goto error;
692 }
693 Py_DECREF(v_new);
694 }
695 /* It's a cell, or already free in this scope */
696 Py_DECREF(name);
697 continue;
698 }
699 else if (PyErr_Occurred()) {
700 goto error;
701 }
702 /* Handle global symbol */
703 if (bound && !PySet_Contains(bound, name)) {
704 Py_DECREF(name);
705 continue; /* it's a global */
706 }
707 /* Propagate new free symbol up the lexical stack */
708 if (PyDict_SetItem(symbols, name, v_free) < 0) {
709 goto error;
710 }
711 Py_DECREF(name);
712 }
713 Py_DECREF(itr);
714 Py_DECREF(v_free);
715 return 1;
716 error:
717 Py_XDECREF(v_free);
718 Py_XDECREF(itr);
719 Py_XDECREF(name);
720 return 0;
721 }
722
723 /* Make final symbol table decisions for block of ste.
724
725 Arguments:
726 ste -- current symtable entry (input/output)
727 bound -- set of variables bound in enclosing scopes (input). bound
728 is NULL for module blocks.
729 free -- set of free variables in enclosed scopes (output)
730 globals -- set of declared global variables in enclosing scopes (input)
731
732 The implementation uses two mutually recursive functions,
733 analyze_block() and analyze_child_block(). analyze_block() is
734 responsible for analyzing the individual names defined in a block.
735 analyze_child_block() prepares temporary namespace dictionaries
736 used to evaluated nested blocks.
737
738 The two functions exist because a child block should see the name
739 bindings of its enclosing blocks, but those bindings should not
740 propagate back to a parent block.
741 */
742
743 static int
744 analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
745 PyObject *global, PyObject* child_free);
746
747 static int
analyze_block(PySTEntryObject * ste,PyObject * bound,PyObject * free,PyObject * global)748 analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
749 PyObject *global)
750 {
751 PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL;
752 PyObject *newglobal = NULL, *newfree = NULL, *allfree = NULL;
753 PyObject *temp;
754 int i, success = 0;
755 Py_ssize_t pos = 0;
756
757 local = PySet_New(NULL); /* collect new names bound in block */
758 if (!local)
759 goto error;
760 scopes = PyDict_New(); /* collect scopes defined for each name */
761 if (!scopes)
762 goto error;
763
764 /* Allocate new global and bound variable dictionaries. These
765 dictionaries hold the names visible in nested blocks. For
766 ClassBlocks, the bound and global names are initialized
767 before analyzing names, because class bindings aren't
768 visible in methods. For other blocks, they are initialized
769 after names are analyzed.
770 */
771
772 /* TODO(jhylton): Package these dicts in a struct so that we
773 can write reasonable helper functions?
774 */
775 newglobal = PySet_New(NULL);
776 if (!newglobal)
777 goto error;
778 newfree = PySet_New(NULL);
779 if (!newfree)
780 goto error;
781 newbound = PySet_New(NULL);
782 if (!newbound)
783 goto error;
784
785 /* Class namespace has no effect on names visible in
786 nested functions, so populate the global and bound
787 sets to be passed to child blocks before analyzing
788 this one.
789 */
790 if (ste->ste_type == ClassBlock) {
791 /* Pass down known globals */
792 temp = PyNumber_InPlaceOr(newglobal, global);
793 if (!temp)
794 goto error;
795 Py_DECREF(temp);
796 /* Pass down previously bound symbols */
797 if (bound) {
798 temp = PyNumber_InPlaceOr(newbound, bound);
799 if (!temp)
800 goto error;
801 Py_DECREF(temp);
802 }
803 }
804
805 while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) {
806 long flags = PyLong_AS_LONG(v);
807 if (!analyze_name(ste, scopes, name, flags,
808 bound, local, free, global))
809 goto error;
810 }
811
812 /* Populate global and bound sets to be passed to children. */
813 if (ste->ste_type != ClassBlock) {
814 /* Add function locals to bound set */
815 if (ste->ste_type == FunctionBlock) {
816 temp = PyNumber_InPlaceOr(newbound, local);
817 if (!temp)
818 goto error;
819 Py_DECREF(temp);
820 }
821 /* Pass down previously bound symbols */
822 if (bound) {
823 temp = PyNumber_InPlaceOr(newbound, bound);
824 if (!temp)
825 goto error;
826 Py_DECREF(temp);
827 }
828 /* Pass down known globals */
829 temp = PyNumber_InPlaceOr(newglobal, global);
830 if (!temp)
831 goto error;
832 Py_DECREF(temp);
833 }
834 else {
835 /* Special-case __class__ */
836 if (!GET_IDENTIFIER(__class__))
837 goto error;
838 if (PySet_Add(newbound, __class__) < 0)
839 goto error;
840 }
841
842 /* Recursively call analyze_child_block() on each child block.
843
844 newbound, newglobal now contain the names visible in
845 nested blocks. The free variables in the children will
846 be collected in allfree.
847 */
848 allfree = PySet_New(NULL);
849 if (!allfree)
850 goto error;
851 for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) {
852 PyObject *c = PyList_GET_ITEM(ste->ste_children, i);
853 PySTEntryObject* entry;
854 assert(c && PySTEntry_Check(c));
855 entry = (PySTEntryObject*)c;
856 if (!analyze_child_block(entry, newbound, newfree, newglobal,
857 allfree))
858 goto error;
859 /* Check if any children have free variables */
860 if (entry->ste_free || entry->ste_child_free)
861 ste->ste_child_free = 1;
862 }
863
864 temp = PyNumber_InPlaceOr(newfree, allfree);
865 if (!temp)
866 goto error;
867 Py_DECREF(temp);
868
869 /* Check if any local variables must be converted to cell variables */
870 if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree))
871 goto error;
872 else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree))
873 goto error;
874 /* Records the results of the analysis in the symbol table entry */
875 if (!update_symbols(ste->ste_symbols, scopes, bound, newfree,
876 ste->ste_type == ClassBlock))
877 goto error;
878
879 temp = PyNumber_InPlaceOr(free, newfree);
880 if (!temp)
881 goto error;
882 Py_DECREF(temp);
883 success = 1;
884 error:
885 Py_XDECREF(scopes);
886 Py_XDECREF(local);
887 Py_XDECREF(newbound);
888 Py_XDECREF(newglobal);
889 Py_XDECREF(newfree);
890 Py_XDECREF(allfree);
891 if (!success)
892 assert(PyErr_Occurred());
893 return success;
894 }
895
896 static int
analyze_child_block(PySTEntryObject * entry,PyObject * bound,PyObject * free,PyObject * global,PyObject * child_free)897 analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
898 PyObject *global, PyObject* child_free)
899 {
900 PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL;
901 PyObject *temp;
902
903 /* Copy the bound and global dictionaries.
904
905 These dictionaries are used by all blocks enclosed by the
906 current block. The analyze_block() call modifies these
907 dictionaries.
908
909 */
910 temp_bound = PySet_New(bound);
911 if (!temp_bound)
912 goto error;
913 temp_free = PySet_New(free);
914 if (!temp_free)
915 goto error;
916 temp_global = PySet_New(global);
917 if (!temp_global)
918 goto error;
919
920 if (!analyze_block(entry, temp_bound, temp_free, temp_global))
921 goto error;
922 temp = PyNumber_InPlaceOr(child_free, temp_free);
923 if (!temp)
924 goto error;
925 Py_DECREF(temp);
926 Py_DECREF(temp_bound);
927 Py_DECREF(temp_free);
928 Py_DECREF(temp_global);
929 return 1;
930 error:
931 Py_XDECREF(temp_bound);
932 Py_XDECREF(temp_free);
933 Py_XDECREF(temp_global);
934 return 0;
935 }
936
937 static int
symtable_analyze(struct symtable * st)938 symtable_analyze(struct symtable *st)
939 {
940 PyObject *free, *global;
941 int r;
942
943 free = PySet_New(NULL);
944 if (!free)
945 return 0;
946 global = PySet_New(NULL);
947 if (!global) {
948 Py_DECREF(free);
949 return 0;
950 }
951 r = analyze_block(st->st_top, NULL, free, global);
952 Py_DECREF(free);
953 Py_DECREF(global);
954 return r;
955 }
956
957 /* symtable_enter_block() gets a reference via ste_new.
958 This reference is released when the block is exited, via the DECREF
959 in symtable_exit_block().
960 */
961
962 static int
symtable_exit_block(struct symtable * st)963 symtable_exit_block(struct symtable *st)
964 {
965 Py_ssize_t size;
966
967 st->st_cur = NULL;
968 size = PyList_GET_SIZE(st->st_stack);
969 if (size) {
970 if (PyList_SetSlice(st->st_stack, size - 1, size, NULL) < 0)
971 return 0;
972 if (--size)
973 st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, size - 1);
974 }
975 return 1;
976 }
977
978 static int
symtable_enter_block(struct symtable * st,identifier name,_Py_block_ty block,void * ast,int lineno,int col_offset,int end_lineno,int end_col_offset)979 symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
980 void *ast, int lineno, int col_offset,
981 int end_lineno, int end_col_offset)
982 {
983 PySTEntryObject *prev = NULL, *ste;
984
985 ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset);
986 if (ste == NULL)
987 return 0;
988 if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) {
989 Py_DECREF(ste);
990 return 0;
991 }
992 prev = st->st_cur;
993 /* bpo-37757: For now, disallow *all* assignment expressions in the
994 * outermost iterator expression of a comprehension, even those inside
995 * a nested comprehension or a lambda expression.
996 */
997 if (prev) {
998 ste->ste_comp_iter_expr = prev->ste_comp_iter_expr;
999 }
1000 /* The entry is owned by the stack. Borrow it for st_cur. */
1001 Py_DECREF(ste);
1002 st->st_cur = ste;
1003
1004 /* Annotation blocks shouldn't have any affect on the symbol table since in
1005 * the compilation stage, they will all be transformed to strings. They are
1006 * only created if future 'annotations' feature is activated. */
1007 if (block == AnnotationBlock) {
1008 return 1;
1009 }
1010
1011 if (block == ModuleBlock)
1012 st->st_global = st->st_cur->ste_symbols;
1013
1014 if (prev) {
1015 if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) {
1016 return 0;
1017 }
1018 }
1019 return 1;
1020 }
1021
1022 static long
symtable_lookup(struct symtable * st,PyObject * name)1023 symtable_lookup(struct symtable *st, PyObject *name)
1024 {
1025 PyObject *mangled = _Py_Mangle(st->st_private, name);
1026 if (!mangled)
1027 return 0;
1028 long ret = _PyST_GetSymbol(st->st_cur, mangled);
1029 Py_DECREF(mangled);
1030 return ret;
1031 }
1032
1033 static int
symtable_add_def_helper(struct symtable * st,PyObject * name,int flag,struct _symtable_entry * ste,int lineno,int col_offset,int end_lineno,int end_col_offset)1034 symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste,
1035 int lineno, int col_offset, int end_lineno, int end_col_offset)
1036 {
1037 PyObject *o;
1038 PyObject *dict;
1039 long val;
1040 PyObject *mangled = _Py_Mangle(st->st_private, name);
1041
1042
1043 if (!mangled)
1044 return 0;
1045 dict = ste->ste_symbols;
1046 if ((o = PyDict_GetItemWithError(dict, mangled))) {
1047 val = PyLong_AS_LONG(o);
1048 if ((flag & DEF_PARAM) && (val & DEF_PARAM)) {
1049 /* Is it better to use 'mangled' or 'name' here? */
1050 PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name);
1051 PyErr_RangedSyntaxLocationObject(st->st_filename,
1052 lineno, col_offset + 1,
1053 end_lineno, end_col_offset + 1);
1054 goto error;
1055 }
1056 val |= flag;
1057 }
1058 else if (PyErr_Occurred()) {
1059 goto error;
1060 }
1061 else {
1062 val = flag;
1063 }
1064 if (ste->ste_comp_iter_target) {
1065 /* This name is an iteration variable in a comprehension,
1066 * so check for a binding conflict with any named expressions.
1067 * Otherwise, mark it as an iteration variable so subsequent
1068 * named expressions can check for conflicts.
1069 */
1070 if (val & (DEF_GLOBAL | DEF_NONLOCAL)) {
1071 PyErr_Format(PyExc_SyntaxError,
1072 NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name);
1073 PyErr_RangedSyntaxLocationObject(st->st_filename,
1074 lineno, col_offset + 1,
1075 end_lineno, end_col_offset + 1);
1076 goto error;
1077 }
1078 val |= DEF_COMP_ITER;
1079 }
1080 o = PyLong_FromLong(val);
1081 if (o == NULL)
1082 goto error;
1083 if (PyDict_SetItem(dict, mangled, o) < 0) {
1084 Py_DECREF(o);
1085 goto error;
1086 }
1087 Py_DECREF(o);
1088
1089 if (flag & DEF_PARAM) {
1090 if (PyList_Append(ste->ste_varnames, mangled) < 0)
1091 goto error;
1092 } else if (flag & DEF_GLOBAL) {
1093 /* XXX need to update DEF_GLOBAL for other flags too;
1094 perhaps only DEF_FREE_GLOBAL */
1095 val = flag;
1096 if ((o = PyDict_GetItemWithError(st->st_global, mangled))) {
1097 val |= PyLong_AS_LONG(o);
1098 }
1099 else if (PyErr_Occurred()) {
1100 goto error;
1101 }
1102 o = PyLong_FromLong(val);
1103 if (o == NULL)
1104 goto error;
1105 if (PyDict_SetItem(st->st_global, mangled, o) < 0) {
1106 Py_DECREF(o);
1107 goto error;
1108 }
1109 Py_DECREF(o);
1110 }
1111 Py_DECREF(mangled);
1112 return 1;
1113
1114 error:
1115 Py_DECREF(mangled);
1116 return 0;
1117 }
1118
1119 static int
symtable_add_def(struct symtable * st,PyObject * name,int flag,int lineno,int col_offset,int end_lineno,int end_col_offset)1120 symtable_add_def(struct symtable *st, PyObject *name, int flag,
1121 int lineno, int col_offset, int end_lineno, int end_col_offset)
1122 {
1123 return symtable_add_def_helper(st, name, flag, st->st_cur,
1124 lineno, col_offset, end_lineno, end_col_offset);
1125 }
1126
1127 /* VISIT, VISIT_SEQ and VIST_SEQ_TAIL take an ASDL type as their second argument.
1128 They use the ASDL name to synthesize the name of the C type and the visit
1129 function.
1130
1131 VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is
1132 useful if the first node in the sequence requires special treatment.
1133
1134 VISIT_QUIT macro returns the specified value exiting from the function but
1135 first adjusts current recursion counter depth.
1136 */
1137
1138 #define VISIT_QUIT(ST, X) \
1139 return --(ST)->recursion_depth,(X)
1140
1141 #define VISIT(ST, TYPE, V) \
1142 if (!symtable_visit_ ## TYPE((ST), (V))) \
1143 VISIT_QUIT((ST), 0);
1144
1145 #define VISIT_SEQ(ST, TYPE, SEQ) { \
1146 int i; \
1147 asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1148 for (i = 0; i < asdl_seq_LEN(seq); i++) { \
1149 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1150 if (!symtable_visit_ ## TYPE((ST), elt)) \
1151 VISIT_QUIT((ST), 0); \
1152 } \
1153 }
1154
1155 #define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \
1156 int i; \
1157 asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1158 for (i = (START); i < asdl_seq_LEN(seq); i++) { \
1159 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1160 if (!symtable_visit_ ## TYPE((ST), elt)) \
1161 VISIT_QUIT((ST), 0); \
1162 } \
1163 }
1164
1165 #define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) { \
1166 int i = 0; \
1167 asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1168 for (i = 0; i < asdl_seq_LEN(seq); i++) { \
1169 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1170 if (!elt) continue; /* can be NULL */ \
1171 if (!symtable_visit_ ## TYPE((ST), elt)) \
1172 VISIT_QUIT((ST), 0); \
1173 } \
1174 }
1175
1176 static int
symtable_record_directive(struct symtable * st,identifier name,int lineno,int col_offset,int end_lineno,int end_col_offset)1177 symtable_record_directive(struct symtable *st, identifier name, int lineno,
1178 int col_offset, int end_lineno, int end_col_offset)
1179 {
1180 PyObject *data, *mangled;
1181 int res;
1182 if (!st->st_cur->ste_directives) {
1183 st->st_cur->ste_directives = PyList_New(0);
1184 if (!st->st_cur->ste_directives)
1185 return 0;
1186 }
1187 mangled = _Py_Mangle(st->st_private, name);
1188 if (!mangled)
1189 return 0;
1190 data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset);
1191 if (!data)
1192 return 0;
1193 res = PyList_Append(st->st_cur->ste_directives, data);
1194 Py_DECREF(data);
1195 return res == 0;
1196 }
1197
1198
1199 static int
symtable_visit_stmt(struct symtable * st,stmt_ty s)1200 symtable_visit_stmt(struct symtable *st, stmt_ty s)
1201 {
1202 if (++st->recursion_depth > st->recursion_limit) {
1203 PyErr_SetString(PyExc_RecursionError,
1204 "maximum recursion depth exceeded during compilation");
1205 VISIT_QUIT(st, 0);
1206 }
1207 switch (s->kind) {
1208 case FunctionDef_kind:
1209 if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s)))
1210 VISIT_QUIT(st, 0);
1211 if (s->v.FunctionDef.args->defaults)
1212 VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
1213 if (s->v.FunctionDef.args->kw_defaults)
1214 VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults);
1215 if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
1216 s->v.FunctionDef.returns))
1217 VISIT_QUIT(st, 0);
1218 if (s->v.FunctionDef.decorator_list)
1219 VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list);
1220 if (!symtable_enter_block(st, s->v.FunctionDef.name,
1221 FunctionBlock, (void *)s,
1222 LOCATION(s)))
1223 VISIT_QUIT(st, 0);
1224 VISIT(st, arguments, s->v.FunctionDef.args);
1225 VISIT_SEQ(st, stmt, s->v.FunctionDef.body);
1226 if (!symtable_exit_block(st))
1227 VISIT_QUIT(st, 0);
1228 break;
1229 case ClassDef_kind: {
1230 PyObject *tmp;
1231 if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s)))
1232 VISIT_QUIT(st, 0);
1233 VISIT_SEQ(st, expr, s->v.ClassDef.bases);
1234 VISIT_SEQ(st, keyword, s->v.ClassDef.keywords);
1235 if (s->v.ClassDef.decorator_list)
1236 VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
1237 if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock,
1238 (void *)s, s->lineno, s->col_offset,
1239 s->end_lineno, s->end_col_offset))
1240 VISIT_QUIT(st, 0);
1241 tmp = st->st_private;
1242 st->st_private = s->v.ClassDef.name;
1243 VISIT_SEQ(st, stmt, s->v.ClassDef.body);
1244 st->st_private = tmp;
1245 if (!symtable_exit_block(st))
1246 VISIT_QUIT(st, 0);
1247 break;
1248 }
1249 case Return_kind:
1250 if (s->v.Return.value) {
1251 VISIT(st, expr, s->v.Return.value);
1252 st->st_cur->ste_returns_value = 1;
1253 }
1254 break;
1255 case Delete_kind:
1256 VISIT_SEQ(st, expr, s->v.Delete.targets);
1257 break;
1258 case Assign_kind:
1259 VISIT_SEQ(st, expr, s->v.Assign.targets);
1260 VISIT(st, expr, s->v.Assign.value);
1261 break;
1262 case AnnAssign_kind:
1263 if (s->v.AnnAssign.target->kind == Name_kind) {
1264 expr_ty e_name = s->v.AnnAssign.target;
1265 long cur = symtable_lookup(st, e_name->v.Name.id);
1266 if (cur < 0) {
1267 VISIT_QUIT(st, 0);
1268 }
1269 if ((cur & (DEF_GLOBAL | DEF_NONLOCAL))
1270 && (st->st_cur->ste_symbols != st->st_global)
1271 && s->v.AnnAssign.simple) {
1272 PyErr_Format(PyExc_SyntaxError,
1273 cur & DEF_GLOBAL ? GLOBAL_ANNOT : NONLOCAL_ANNOT,
1274 e_name->v.Name.id);
1275 PyErr_RangedSyntaxLocationObject(st->st_filename,
1276 s->lineno,
1277 s->col_offset + 1,
1278 s->end_lineno,
1279 s->end_col_offset + 1);
1280 VISIT_QUIT(st, 0);
1281 }
1282 if (s->v.AnnAssign.simple &&
1283 !symtable_add_def(st, e_name->v.Name.id,
1284 DEF_ANNOT | DEF_LOCAL, LOCATION(e_name))) {
1285 VISIT_QUIT(st, 0);
1286 }
1287 else {
1288 if (s->v.AnnAssign.value
1289 && !symtable_add_def(st, e_name->v.Name.id, DEF_LOCAL, LOCATION(e_name))) {
1290 VISIT_QUIT(st, 0);
1291 }
1292 }
1293 }
1294 else {
1295 VISIT(st, expr, s->v.AnnAssign.target);
1296 }
1297 if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) {
1298 VISIT_QUIT(st, 0);
1299 }
1300
1301 if (s->v.AnnAssign.value) {
1302 VISIT(st, expr, s->v.AnnAssign.value);
1303 }
1304 break;
1305 case AugAssign_kind:
1306 VISIT(st, expr, s->v.AugAssign.target);
1307 VISIT(st, expr, s->v.AugAssign.value);
1308 break;
1309 case For_kind:
1310 VISIT(st, expr, s->v.For.target);
1311 VISIT(st, expr, s->v.For.iter);
1312 VISIT_SEQ(st, stmt, s->v.For.body);
1313 if (s->v.For.orelse)
1314 VISIT_SEQ(st, stmt, s->v.For.orelse);
1315 break;
1316 case While_kind:
1317 VISIT(st, expr, s->v.While.test);
1318 VISIT_SEQ(st, stmt, s->v.While.body);
1319 if (s->v.While.orelse)
1320 VISIT_SEQ(st, stmt, s->v.While.orelse);
1321 break;
1322 case If_kind:
1323 /* XXX if 0: and lookup_yield() hacks */
1324 VISIT(st, expr, s->v.If.test);
1325 VISIT_SEQ(st, stmt, s->v.If.body);
1326 if (s->v.If.orelse)
1327 VISIT_SEQ(st, stmt, s->v.If.orelse);
1328 break;
1329 case Match_kind:
1330 VISIT(st, expr, s->v.Match.subject);
1331 VISIT_SEQ(st, match_case, s->v.Match.cases);
1332 break;
1333 case Raise_kind:
1334 if (s->v.Raise.exc) {
1335 VISIT(st, expr, s->v.Raise.exc);
1336 if (s->v.Raise.cause) {
1337 VISIT(st, expr, s->v.Raise.cause);
1338 }
1339 }
1340 break;
1341 case Try_kind:
1342 VISIT_SEQ(st, stmt, s->v.Try.body);
1343 VISIT_SEQ(st, stmt, s->v.Try.orelse);
1344 VISIT_SEQ(st, excepthandler, s->v.Try.handlers);
1345 VISIT_SEQ(st, stmt, s->v.Try.finalbody);
1346 break;
1347 case Assert_kind:
1348 VISIT(st, expr, s->v.Assert.test);
1349 if (s->v.Assert.msg)
1350 VISIT(st, expr, s->v.Assert.msg);
1351 break;
1352 case Import_kind:
1353 VISIT_SEQ(st, alias, s->v.Import.names);
1354 break;
1355 case ImportFrom_kind:
1356 VISIT_SEQ(st, alias, s->v.ImportFrom.names);
1357 break;
1358 case Global_kind: {
1359 int i;
1360 asdl_identifier_seq *seq = s->v.Global.names;
1361 for (i = 0; i < asdl_seq_LEN(seq); i++) {
1362 identifier name = (identifier)asdl_seq_GET(seq, i);
1363 long cur = symtable_lookup(st, name);
1364 if (cur < 0)
1365 VISIT_QUIT(st, 0);
1366 if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) {
1367 const char* msg;
1368 if (cur & DEF_PARAM) {
1369 msg = GLOBAL_PARAM;
1370 } else if (cur & USE) {
1371 msg = GLOBAL_AFTER_USE;
1372 } else if (cur & DEF_ANNOT) {
1373 msg = GLOBAL_ANNOT;
1374 } else { /* DEF_LOCAL */
1375 msg = GLOBAL_AFTER_ASSIGN;
1376 }
1377 PyErr_Format(PyExc_SyntaxError,
1378 msg, name);
1379 PyErr_RangedSyntaxLocationObject(st->st_filename,
1380 s->lineno,
1381 s->col_offset + 1,
1382 s->end_lineno,
1383 s->end_col_offset + 1);
1384 VISIT_QUIT(st, 0);
1385 }
1386 if (!symtable_add_def(st, name, DEF_GLOBAL, LOCATION(s)))
1387 VISIT_QUIT(st, 0);
1388 if (!symtable_record_directive(st, name, s->lineno, s->col_offset,
1389 s->end_lineno, s->end_col_offset))
1390 VISIT_QUIT(st, 0);
1391 }
1392 break;
1393 }
1394 case Nonlocal_kind: {
1395 int i;
1396 asdl_identifier_seq *seq = s->v.Nonlocal.names;
1397 for (i = 0; i < asdl_seq_LEN(seq); i++) {
1398 identifier name = (identifier)asdl_seq_GET(seq, i);
1399 long cur = symtable_lookup(st, name);
1400 if (cur < 0)
1401 VISIT_QUIT(st, 0);
1402 if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) {
1403 const char* msg;
1404 if (cur & DEF_PARAM) {
1405 msg = NONLOCAL_PARAM;
1406 } else if (cur & USE) {
1407 msg = NONLOCAL_AFTER_USE;
1408 } else if (cur & DEF_ANNOT) {
1409 msg = NONLOCAL_ANNOT;
1410 } else { /* DEF_LOCAL */
1411 msg = NONLOCAL_AFTER_ASSIGN;
1412 }
1413 PyErr_Format(PyExc_SyntaxError, msg, name);
1414 PyErr_RangedSyntaxLocationObject(st->st_filename,
1415 s->lineno,
1416 s->col_offset + 1,
1417 s->end_lineno,
1418 s->end_col_offset + 1);
1419 VISIT_QUIT(st, 0);
1420 }
1421 if (!symtable_add_def(st, name, DEF_NONLOCAL, LOCATION(s)))
1422 VISIT_QUIT(st, 0);
1423 if (!symtable_record_directive(st, name, s->lineno, s->col_offset,
1424 s->end_lineno, s->end_col_offset))
1425 VISIT_QUIT(st, 0);
1426 }
1427 break;
1428 }
1429 case Expr_kind:
1430 VISIT(st, expr, s->v.Expr.value);
1431 break;
1432 case Pass_kind:
1433 case Break_kind:
1434 case Continue_kind:
1435 /* nothing to do here */
1436 break;
1437 case With_kind:
1438 VISIT_SEQ(st, withitem, s->v.With.items);
1439 VISIT_SEQ(st, stmt, s->v.With.body);
1440 break;
1441 case AsyncFunctionDef_kind:
1442 if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s)))
1443 VISIT_QUIT(st, 0);
1444 if (s->v.AsyncFunctionDef.args->defaults)
1445 VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults);
1446 if (s->v.AsyncFunctionDef.args->kw_defaults)
1447 VISIT_SEQ_WITH_NULL(st, expr,
1448 s->v.AsyncFunctionDef.args->kw_defaults);
1449 if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
1450 s->v.AsyncFunctionDef.returns))
1451 VISIT_QUIT(st, 0);
1452 if (s->v.AsyncFunctionDef.decorator_list)
1453 VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list);
1454 if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name,
1455 FunctionBlock, (void *)s,
1456 s->lineno, s->col_offset,
1457 s->end_lineno, s->end_col_offset))
1458 VISIT_QUIT(st, 0);
1459 st->st_cur->ste_coroutine = 1;
1460 VISIT(st, arguments, s->v.AsyncFunctionDef.args);
1461 VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
1462 if (!symtable_exit_block(st))
1463 VISIT_QUIT(st, 0);
1464 break;
1465 case AsyncWith_kind:
1466 VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
1467 VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
1468 break;
1469 case AsyncFor_kind:
1470 VISIT(st, expr, s->v.AsyncFor.target);
1471 VISIT(st, expr, s->v.AsyncFor.iter);
1472 VISIT_SEQ(st, stmt, s->v.AsyncFor.body);
1473 if (s->v.AsyncFor.orelse)
1474 VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse);
1475 break;
1476 }
1477 VISIT_QUIT(st, 1);
1478 }
1479
1480 static int
symtable_extend_namedexpr_scope(struct symtable * st,expr_ty e)1481 symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e)
1482 {
1483 assert(st->st_stack);
1484 assert(e->kind == Name_kind);
1485
1486 PyObject *target_name = e->v.Name.id;
1487 Py_ssize_t i, size;
1488 struct _symtable_entry *ste;
1489 size = PyList_GET_SIZE(st->st_stack);
1490 assert(size);
1491
1492 /* Iterate over the stack in reverse and add to the nearest adequate scope */
1493 for (i = size - 1; i >= 0; i--) {
1494 ste = (struct _symtable_entry *) PyList_GET_ITEM(st->st_stack, i);
1495
1496 /* If we find a comprehension scope, check for a target
1497 * binding conflict with iteration variables, otherwise skip it
1498 */
1499 if (ste->ste_comprehension) {
1500 long target_in_scope = _PyST_GetSymbol(ste, target_name);
1501 if (target_in_scope & DEF_COMP_ITER) {
1502 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name);
1503 PyErr_RangedSyntaxLocationObject(st->st_filename,
1504 e->lineno,
1505 e->col_offset + 1,
1506 e->end_lineno,
1507 e->end_col_offset + 1);
1508 VISIT_QUIT(st, 0);
1509 }
1510 continue;
1511 }
1512
1513 /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */
1514 if (ste->ste_type == FunctionBlock) {
1515 long target_in_scope = _PyST_GetSymbol(ste, target_name);
1516 if (target_in_scope & DEF_GLOBAL) {
1517 if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e)))
1518 VISIT_QUIT(st, 0);
1519 } else {
1520 if (!symtable_add_def(st, target_name, DEF_NONLOCAL, LOCATION(e)))
1521 VISIT_QUIT(st, 0);
1522 }
1523 if (!symtable_record_directive(st, target_name, LOCATION(e)))
1524 VISIT_QUIT(st, 0);
1525
1526 return symtable_add_def_helper(st, target_name, DEF_LOCAL, ste, LOCATION(e));
1527 }
1528 /* If we find a ModuleBlock entry, add as GLOBAL */
1529 if (ste->ste_type == ModuleBlock) {
1530 if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e)))
1531 VISIT_QUIT(st, 0);
1532 if (!symtable_record_directive(st, target_name, LOCATION(e)))
1533 VISIT_QUIT(st, 0);
1534
1535 return symtable_add_def_helper(st, target_name, DEF_GLOBAL, ste, LOCATION(e));
1536 }
1537 /* Disallow usage in ClassBlock */
1538 if (ste->ste_type == ClassBlock) {
1539 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS);
1540 PyErr_RangedSyntaxLocationObject(st->st_filename,
1541 e->lineno,
1542 e->col_offset + 1,
1543 e->end_lineno,
1544 e->end_col_offset + 1);
1545 VISIT_QUIT(st, 0);
1546 }
1547 }
1548
1549 /* We should always find either a FunctionBlock, ModuleBlock or ClassBlock
1550 and should never fall to this case
1551 */
1552 assert(0);
1553 return 0;
1554 }
1555
1556 static int
symtable_handle_namedexpr(struct symtable * st,expr_ty e)1557 symtable_handle_namedexpr(struct symtable *st, expr_ty e)
1558 {
1559 if (st->st_cur->ste_comp_iter_expr > 0) {
1560 /* Assignment isn't allowed in a comprehension iterable expression */
1561 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR);
1562 PyErr_RangedSyntaxLocationObject(st->st_filename,
1563 e->lineno,
1564 e->col_offset + 1,
1565 e->end_lineno,
1566 e->end_col_offset + 1);
1567 return 0;
1568 }
1569 if (st->st_cur->ste_comprehension) {
1570 /* Inside a comprehension body, so find the right target scope */
1571 if (!symtable_extend_namedexpr_scope(st, e->v.NamedExpr.target))
1572 return 0;
1573 }
1574 VISIT(st, expr, e->v.NamedExpr.value);
1575 VISIT(st, expr, e->v.NamedExpr.target);
1576 return 1;
1577 }
1578
1579 static int
symtable_visit_expr(struct symtable * st,expr_ty e)1580 symtable_visit_expr(struct symtable *st, expr_ty e)
1581 {
1582 if (++st->recursion_depth > st->recursion_limit) {
1583 PyErr_SetString(PyExc_RecursionError,
1584 "maximum recursion depth exceeded during compilation");
1585 VISIT_QUIT(st, 0);
1586 }
1587 switch (e->kind) {
1588 case NamedExpr_kind:
1589 if (!symtable_raise_if_annotation_block(st, "named expression", e)) {
1590 VISIT_QUIT(st, 0);
1591 }
1592 if(!symtable_handle_namedexpr(st, e))
1593 VISIT_QUIT(st, 0);
1594 break;
1595 case BoolOp_kind:
1596 VISIT_SEQ(st, expr, e->v.BoolOp.values);
1597 break;
1598 case BinOp_kind:
1599 VISIT(st, expr, e->v.BinOp.left);
1600 VISIT(st, expr, e->v.BinOp.right);
1601 break;
1602 case UnaryOp_kind:
1603 VISIT(st, expr, e->v.UnaryOp.operand);
1604 break;
1605 case Lambda_kind: {
1606 if (!GET_IDENTIFIER(lambda))
1607 VISIT_QUIT(st, 0);
1608 if (e->v.Lambda.args->defaults)
1609 VISIT_SEQ(st, expr, e->v.Lambda.args->defaults);
1610 if (e->v.Lambda.args->kw_defaults)
1611 VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults);
1612 if (!symtable_enter_block(st, lambda,
1613 FunctionBlock, (void *)e,
1614 e->lineno, e->col_offset,
1615 e->end_lineno, e->end_col_offset))
1616 VISIT_QUIT(st, 0);
1617 VISIT(st, arguments, e->v.Lambda.args);
1618 VISIT(st, expr, e->v.Lambda.body);
1619 if (!symtable_exit_block(st))
1620 VISIT_QUIT(st, 0);
1621 break;
1622 }
1623 case IfExp_kind:
1624 VISIT(st, expr, e->v.IfExp.test);
1625 VISIT(st, expr, e->v.IfExp.body);
1626 VISIT(st, expr, e->v.IfExp.orelse);
1627 break;
1628 case Dict_kind:
1629 VISIT_SEQ_WITH_NULL(st, expr, e->v.Dict.keys);
1630 VISIT_SEQ(st, expr, e->v.Dict.values);
1631 break;
1632 case Set_kind:
1633 VISIT_SEQ(st, expr, e->v.Set.elts);
1634 break;
1635 case GeneratorExp_kind:
1636 if (!symtable_visit_genexp(st, e))
1637 VISIT_QUIT(st, 0);
1638 break;
1639 case ListComp_kind:
1640 if (!symtable_visit_listcomp(st, e))
1641 VISIT_QUIT(st, 0);
1642 break;
1643 case SetComp_kind:
1644 if (!symtable_visit_setcomp(st, e))
1645 VISIT_QUIT(st, 0);
1646 break;
1647 case DictComp_kind:
1648 if (!symtable_visit_dictcomp(st, e))
1649 VISIT_QUIT(st, 0);
1650 break;
1651 case Yield_kind:
1652 if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
1653 VISIT_QUIT(st, 0);
1654 }
1655 if (e->v.Yield.value)
1656 VISIT(st, expr, e->v.Yield.value);
1657 st->st_cur->ste_generator = 1;
1658 if (st->st_cur->ste_comprehension) {
1659 return symtable_raise_if_comprehension_block(st, e);
1660 }
1661 break;
1662 case YieldFrom_kind:
1663 if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
1664 VISIT_QUIT(st, 0);
1665 }
1666 VISIT(st, expr, e->v.YieldFrom.value);
1667 st->st_cur->ste_generator = 1;
1668 if (st->st_cur->ste_comprehension) {
1669 return symtable_raise_if_comprehension_block(st, e);
1670 }
1671 break;
1672 case Await_kind:
1673 if (!symtable_raise_if_annotation_block(st, "await expression", e)) {
1674 VISIT_QUIT(st, 0);
1675 }
1676 VISIT(st, expr, e->v.Await.value);
1677 st->st_cur->ste_coroutine = 1;
1678 break;
1679 case Compare_kind:
1680 VISIT(st, expr, e->v.Compare.left);
1681 VISIT_SEQ(st, expr, e->v.Compare.comparators);
1682 break;
1683 case Call_kind:
1684 VISIT(st, expr, e->v.Call.func);
1685 VISIT_SEQ(st, expr, e->v.Call.args);
1686 VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords);
1687 break;
1688 case FormattedValue_kind:
1689 VISIT(st, expr, e->v.FormattedValue.value);
1690 if (e->v.FormattedValue.format_spec)
1691 VISIT(st, expr, e->v.FormattedValue.format_spec);
1692 break;
1693 case JoinedStr_kind:
1694 VISIT_SEQ(st, expr, e->v.JoinedStr.values);
1695 break;
1696 case Constant_kind:
1697 /* Nothing to do here. */
1698 break;
1699 /* The following exprs can be assignment targets. */
1700 case Attribute_kind:
1701 VISIT(st, expr, e->v.Attribute.value);
1702 break;
1703 case Subscript_kind:
1704 VISIT(st, expr, e->v.Subscript.value);
1705 VISIT(st, expr, e->v.Subscript.slice);
1706 break;
1707 case Starred_kind:
1708 VISIT(st, expr, e->v.Starred.value);
1709 break;
1710 case Slice_kind:
1711 if (e->v.Slice.lower)
1712 VISIT(st, expr, e->v.Slice.lower)
1713 if (e->v.Slice.upper)
1714 VISIT(st, expr, e->v.Slice.upper)
1715 if (e->v.Slice.step)
1716 VISIT(st, expr, e->v.Slice.step)
1717 break;
1718 case Name_kind:
1719 if (!symtable_add_def(st, e->v.Name.id,
1720 e->v.Name.ctx == Load ? USE : DEF_LOCAL, LOCATION(e)))
1721 VISIT_QUIT(st, 0);
1722 /* Special-case super: it counts as a use of __class__ */
1723 if (e->v.Name.ctx == Load &&
1724 st->st_cur->ste_type == FunctionBlock &&
1725 _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) {
1726 if (!GET_IDENTIFIER(__class__) ||
1727 !symtable_add_def(st, __class__, USE, LOCATION(e)))
1728 VISIT_QUIT(st, 0);
1729 }
1730 break;
1731 /* child nodes of List and Tuple will have expr_context set */
1732 case List_kind:
1733 VISIT_SEQ(st, expr, e->v.List.elts);
1734 break;
1735 case Tuple_kind:
1736 VISIT_SEQ(st, expr, e->v.Tuple.elts);
1737 break;
1738 }
1739 VISIT_QUIT(st, 1);
1740 }
1741
1742 static int
symtable_visit_pattern(struct symtable * st,pattern_ty p)1743 symtable_visit_pattern(struct symtable *st, pattern_ty p)
1744 {
1745 if (++st->recursion_depth > st->recursion_limit) {
1746 PyErr_SetString(PyExc_RecursionError,
1747 "maximum recursion depth exceeded during compilation");
1748 VISIT_QUIT(st, 0);
1749 }
1750 switch (p->kind) {
1751 case MatchValue_kind:
1752 VISIT(st, expr, p->v.MatchValue.value);
1753 break;
1754 case MatchSingleton_kind:
1755 /* Nothing to do here. */
1756 break;
1757 case MatchSequence_kind:
1758 VISIT_SEQ(st, pattern, p->v.MatchSequence.patterns);
1759 break;
1760 case MatchStar_kind:
1761 if (p->v.MatchStar.name) {
1762 symtable_add_def(st, p->v.MatchStar.name, DEF_LOCAL, LOCATION(p));
1763 }
1764 break;
1765 case MatchMapping_kind:
1766 VISIT_SEQ(st, expr, p->v.MatchMapping.keys);
1767 VISIT_SEQ(st, pattern, p->v.MatchMapping.patterns);
1768 if (p->v.MatchMapping.rest) {
1769 symtable_add_def(st, p->v.MatchMapping.rest, DEF_LOCAL, LOCATION(p));
1770 }
1771 break;
1772 case MatchClass_kind:
1773 VISIT(st, expr, p->v.MatchClass.cls);
1774 VISIT_SEQ(st, pattern, p->v.MatchClass.patterns);
1775 VISIT_SEQ(st, pattern, p->v.MatchClass.kwd_patterns);
1776 break;
1777 case MatchAs_kind:
1778 if (p->v.MatchAs.pattern) {
1779 VISIT(st, pattern, p->v.MatchAs.pattern);
1780 }
1781 if (p->v.MatchAs.name) {
1782 symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL, LOCATION(p));
1783 }
1784 break;
1785 case MatchOr_kind:
1786 VISIT_SEQ(st, pattern, p->v.MatchOr.patterns);
1787 break;
1788 }
1789 VISIT_QUIT(st, 1);
1790 }
1791
1792 static int
symtable_implicit_arg(struct symtable * st,int pos)1793 symtable_implicit_arg(struct symtable *st, int pos)
1794 {
1795 PyObject *id = PyUnicode_FromFormat(".%d", pos);
1796 if (id == NULL)
1797 return 0;
1798 if (!symtable_add_def(st, id, DEF_PARAM, ST_LOCATION(st->st_cur))) {
1799 Py_DECREF(id);
1800 return 0;
1801 }
1802 Py_DECREF(id);
1803 return 1;
1804 }
1805
1806 static int
symtable_visit_params(struct symtable * st,asdl_arg_seq * args)1807 symtable_visit_params(struct symtable *st, asdl_arg_seq *args)
1808 {
1809 int i;
1810
1811 if (!args)
1812 return -1;
1813
1814 for (i = 0; i < asdl_seq_LEN(args); i++) {
1815 arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
1816 if (!symtable_add_def(st, arg->arg, DEF_PARAM, LOCATION(arg)))
1817 return 0;
1818 }
1819
1820 return 1;
1821 }
1822
1823 static int
symtable_visit_annotation(struct symtable * st,expr_ty annotation)1824 symtable_visit_annotation(struct symtable *st, expr_ty annotation)
1825 {
1826 int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
1827 if (future_annotations &&
1828 !symtable_enter_block(st, GET_IDENTIFIER(_annotation), AnnotationBlock,
1829 (void *)annotation, annotation->lineno,
1830 annotation->col_offset, annotation->end_lineno,
1831 annotation->end_col_offset)) {
1832 VISIT_QUIT(st, 0);
1833 }
1834 VISIT(st, expr, annotation);
1835 if (future_annotations && !symtable_exit_block(st)) {
1836 VISIT_QUIT(st, 0);
1837 }
1838 return 1;
1839 }
1840
1841 static int
symtable_visit_argannotations(struct symtable * st,asdl_arg_seq * args)1842 symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
1843 {
1844 int i;
1845
1846 if (!args)
1847 return -1;
1848
1849 for (i = 0; i < asdl_seq_LEN(args); i++) {
1850 arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
1851 if (arg->annotation)
1852 VISIT(st, expr, arg->annotation);
1853 }
1854
1855 return 1;
1856 }
1857
1858 static int
symtable_visit_annotations(struct symtable * st,stmt_ty o,arguments_ty a,expr_ty returns)1859 symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns)
1860 {
1861 int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
1862 if (future_annotations &&
1863 !symtable_enter_block(st, GET_IDENTIFIER(_annotation), AnnotationBlock,
1864 (void *)o, o->lineno, o->col_offset, o->end_lineno,
1865 o->end_col_offset)) {
1866 VISIT_QUIT(st, 0);
1867 }
1868 if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs))
1869 return 0;
1870 if (a->args && !symtable_visit_argannotations(st, a->args))
1871 return 0;
1872 if (a->vararg && a->vararg->annotation)
1873 VISIT(st, expr, a->vararg->annotation);
1874 if (a->kwarg && a->kwarg->annotation)
1875 VISIT(st, expr, a->kwarg->annotation);
1876 if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
1877 return 0;
1878 if (future_annotations && !symtable_exit_block(st)) {
1879 VISIT_QUIT(st, 0);
1880 }
1881 if (returns && !symtable_visit_annotation(st, returns)) {
1882 VISIT_QUIT(st, 0);
1883 }
1884 return 1;
1885 }
1886
1887 static int
symtable_visit_arguments(struct symtable * st,arguments_ty a)1888 symtable_visit_arguments(struct symtable *st, arguments_ty a)
1889 {
1890 /* skip default arguments inside function block
1891 XXX should ast be different?
1892 */
1893 if (a->posonlyargs && !symtable_visit_params(st, a->posonlyargs))
1894 return 0;
1895 if (a->args && !symtable_visit_params(st, a->args))
1896 return 0;
1897 if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs))
1898 return 0;
1899 if (a->vararg) {
1900 if (!symtable_add_def(st, a->vararg->arg, DEF_PARAM, LOCATION(a->vararg)))
1901 return 0;
1902 st->st_cur->ste_varargs = 1;
1903 }
1904 if (a->kwarg) {
1905 if (!symtable_add_def(st, a->kwarg->arg, DEF_PARAM, LOCATION(a->kwarg)))
1906 return 0;
1907 st->st_cur->ste_varkeywords = 1;
1908 }
1909 return 1;
1910 }
1911
1912
1913 static int
symtable_visit_excepthandler(struct symtable * st,excepthandler_ty eh)1914 symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh)
1915 {
1916 if (eh->v.ExceptHandler.type)
1917 VISIT(st, expr, eh->v.ExceptHandler.type);
1918 if (eh->v.ExceptHandler.name)
1919 if (!symtable_add_def(st, eh->v.ExceptHandler.name, DEF_LOCAL, LOCATION(eh)))
1920 return 0;
1921 VISIT_SEQ(st, stmt, eh->v.ExceptHandler.body);
1922 return 1;
1923 }
1924
1925 static int
symtable_visit_withitem(struct symtable * st,withitem_ty item)1926 symtable_visit_withitem(struct symtable *st, withitem_ty item)
1927 {
1928 VISIT(st, expr, item->context_expr);
1929 if (item->optional_vars) {
1930 VISIT(st, expr, item->optional_vars);
1931 }
1932 return 1;
1933 }
1934
1935 static int
symtable_visit_match_case(struct symtable * st,match_case_ty m)1936 symtable_visit_match_case(struct symtable *st, match_case_ty m)
1937 {
1938 VISIT(st, pattern, m->pattern);
1939 if (m->guard) {
1940 VISIT(st, expr, m->guard);
1941 }
1942 VISIT_SEQ(st, stmt, m->body);
1943 return 1;
1944 }
1945
1946 static int
symtable_visit_alias(struct symtable * st,alias_ty a)1947 symtable_visit_alias(struct symtable *st, alias_ty a)
1948 {
1949 /* Compute store_name, the name actually bound by the import
1950 operation. It is different than a->name when a->name is a
1951 dotted package name (e.g. spam.eggs)
1952 */
1953 PyObject *store_name;
1954 PyObject *name = (a->asname == NULL) ? a->name : a->asname;
1955 Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0,
1956 PyUnicode_GET_LENGTH(name), 1);
1957 if (dot != -1) {
1958 store_name = PyUnicode_Substring(name, 0, dot);
1959 if (!store_name)
1960 return 0;
1961 }
1962 else {
1963 store_name = name;
1964 Py_INCREF(store_name);
1965 }
1966 if (!_PyUnicode_EqualToASCIIString(name, "*")) {
1967 int r = symtable_add_def(st, store_name, DEF_IMPORT, LOCATION(a));
1968 Py_DECREF(store_name);
1969 return r;
1970 }
1971 else {
1972 if (st->st_cur->ste_type != ModuleBlock) {
1973 int lineno = a->lineno;
1974 int col_offset = a->col_offset;
1975 int end_lineno = a->end_lineno;
1976 int end_col_offset = a->end_col_offset;
1977 PyErr_SetString(PyExc_SyntaxError, IMPORT_STAR_WARNING);
1978 PyErr_RangedSyntaxLocationObject(st->st_filename,
1979 lineno, col_offset + 1,
1980 end_lineno, end_col_offset + 1);
1981 Py_DECREF(store_name);
1982 return 0;
1983 }
1984 Py_DECREF(store_name);
1985 return 1;
1986 }
1987 }
1988
1989
1990 static int
symtable_visit_comprehension(struct symtable * st,comprehension_ty lc)1991 symtable_visit_comprehension(struct symtable *st, comprehension_ty lc)
1992 {
1993 st->st_cur->ste_comp_iter_target = 1;
1994 VISIT(st, expr, lc->target);
1995 st->st_cur->ste_comp_iter_target = 0;
1996 st->st_cur->ste_comp_iter_expr++;
1997 VISIT(st, expr, lc->iter);
1998 st->st_cur->ste_comp_iter_expr--;
1999 VISIT_SEQ(st, expr, lc->ifs);
2000 if (lc->is_async) {
2001 st->st_cur->ste_coroutine = 1;
2002 }
2003 return 1;
2004 }
2005
2006
2007 static int
symtable_visit_keyword(struct symtable * st,keyword_ty k)2008 symtable_visit_keyword(struct symtable *st, keyword_ty k)
2009 {
2010 VISIT(st, expr, k->value);
2011 return 1;
2012 }
2013
2014
2015 static int
symtable_handle_comprehension(struct symtable * st,expr_ty e,identifier scope_name,asdl_comprehension_seq * generators,expr_ty elt,expr_ty value)2016 symtable_handle_comprehension(struct symtable *st, expr_ty e,
2017 identifier scope_name, asdl_comprehension_seq *generators,
2018 expr_ty elt, expr_ty value)
2019 {
2020 int is_generator = (e->kind == GeneratorExp_kind);
2021 comprehension_ty outermost = ((comprehension_ty)
2022 asdl_seq_GET(generators, 0));
2023 /* Outermost iterator is evaluated in current scope */
2024 st->st_cur->ste_comp_iter_expr++;
2025 VISIT(st, expr, outermost->iter);
2026 st->st_cur->ste_comp_iter_expr--;
2027 /* Create comprehension scope for the rest */
2028 if (!scope_name ||
2029 !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e,
2030 e->lineno, e->col_offset,
2031 e->end_lineno, e->end_col_offset)) {
2032 return 0;
2033 }
2034 switch(e->kind) {
2035 case ListComp_kind:
2036 st->st_cur->ste_comprehension = ListComprehension;
2037 break;
2038 case SetComp_kind:
2039 st->st_cur->ste_comprehension = SetComprehension;
2040 break;
2041 case DictComp_kind:
2042 st->st_cur->ste_comprehension = DictComprehension;
2043 break;
2044 default:
2045 st->st_cur->ste_comprehension = GeneratorExpression;
2046 break;
2047 }
2048 if (outermost->is_async) {
2049 st->st_cur->ste_coroutine = 1;
2050 }
2051
2052 /* Outermost iter is received as an argument */
2053 if (!symtable_implicit_arg(st, 0)) {
2054 symtable_exit_block(st);
2055 return 0;
2056 }
2057 /* Visit iteration variable target, and mark them as such */
2058 st->st_cur->ste_comp_iter_target = 1;
2059 VISIT(st, expr, outermost->target);
2060 st->st_cur->ste_comp_iter_target = 0;
2061 /* Visit the rest of the comprehension body */
2062 VISIT_SEQ(st, expr, outermost->ifs);
2063 VISIT_SEQ_TAIL(st, comprehension, generators, 1);
2064 if (value)
2065 VISIT(st, expr, value);
2066 VISIT(st, expr, elt);
2067 st->st_cur->ste_generator = is_generator;
2068 return symtable_exit_block(st);
2069 }
2070
2071 static int
symtable_visit_genexp(struct symtable * st,expr_ty e)2072 symtable_visit_genexp(struct symtable *st, expr_ty e)
2073 {
2074 return symtable_handle_comprehension(st, e, GET_IDENTIFIER(genexpr),
2075 e->v.GeneratorExp.generators,
2076 e->v.GeneratorExp.elt, NULL);
2077 }
2078
2079 static int
symtable_visit_listcomp(struct symtable * st,expr_ty e)2080 symtable_visit_listcomp(struct symtable *st, expr_ty e)
2081 {
2082 return symtable_handle_comprehension(st, e, GET_IDENTIFIER(listcomp),
2083 e->v.ListComp.generators,
2084 e->v.ListComp.elt, NULL);
2085 }
2086
2087 static int
symtable_visit_setcomp(struct symtable * st,expr_ty e)2088 symtable_visit_setcomp(struct symtable *st, expr_ty e)
2089 {
2090 return symtable_handle_comprehension(st, e, GET_IDENTIFIER(setcomp),
2091 e->v.SetComp.generators,
2092 e->v.SetComp.elt, NULL);
2093 }
2094
2095 static int
symtable_visit_dictcomp(struct symtable * st,expr_ty e)2096 symtable_visit_dictcomp(struct symtable *st, expr_ty e)
2097 {
2098 return symtable_handle_comprehension(st, e, GET_IDENTIFIER(dictcomp),
2099 e->v.DictComp.generators,
2100 e->v.DictComp.key,
2101 e->v.DictComp.value);
2102 }
2103
2104 static int
symtable_raise_if_annotation_block(struct symtable * st,const char * name,expr_ty e)2105 symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e)
2106 {
2107 if (st->st_cur->ste_type != AnnotationBlock) {
2108 return 1;
2109 }
2110
2111 PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name);
2112 PyErr_RangedSyntaxLocationObject(st->st_filename,
2113 e->lineno,
2114 e->col_offset + 1,
2115 e->end_lineno,
2116 e->end_col_offset + 1);
2117 return 0;
2118 }
2119
2120 static int
symtable_raise_if_comprehension_block(struct symtable * st,expr_ty e)2121 symtable_raise_if_comprehension_block(struct symtable *st, expr_ty e) {
2122 _Py_comprehension_ty type = st->st_cur->ste_comprehension;
2123 PyErr_SetString(PyExc_SyntaxError,
2124 (type == ListComprehension) ? "'yield' inside list comprehension" :
2125 (type == SetComprehension) ? "'yield' inside set comprehension" :
2126 (type == DictComprehension) ? "'yield' inside dict comprehension" :
2127 "'yield' inside generator expression");
2128 PyErr_RangedSyntaxLocationObject(st->st_filename,
2129 e->lineno, e->col_offset + 1,
2130 e->end_lineno, e->end_col_offset + 1);
2131 VISIT_QUIT(st, 0);
2132 }
2133
2134 struct symtable *
_Py_SymtableStringObjectFlags(const char * str,PyObject * filename,int start,PyCompilerFlags * flags)2135 _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
2136 int start, PyCompilerFlags *flags)
2137 {
2138 struct symtable *st;
2139 mod_ty mod;
2140 PyArena *arena;
2141
2142 arena = _PyArena_New();
2143 if (arena == NULL)
2144 return NULL;
2145
2146 mod = _PyParser_ASTFromString(str, filename, start, flags, arena);
2147 if (mod == NULL) {
2148 _PyArena_Free(arena);
2149 return NULL;
2150 }
2151 PyFutureFeatures *future = _PyFuture_FromAST(mod, filename);
2152 if (future == NULL) {
2153 _PyArena_Free(arena);
2154 return NULL;
2155 }
2156 future->ff_features |= flags->cf_flags;
2157 st = _PySymtable_Build(mod, filename, future);
2158 PyObject_Free((void *)future);
2159 _PyArena_Free(arena);
2160 return st;
2161 }
2162