1 #include "Python.h"
2 #include "pycore_ast.h" // stmt_ty
3 #include "pycore_parser.h" // _PyParser_ASTFromString()
4 #include "pycore_pystate.h" // _PyThreadState_GET()
5 #include "pycore_symtable.h" // PySTEntryObject
6
7 // Set this to 1 to dump all symtables to stdout for debugging
8 #define _PY_DUMP_SYMTABLE 0
9
10 /* error strings used for warnings */
11 #define GLOBAL_PARAM \
12 "name '%U' is parameter and global"
13
14 #define NONLOCAL_PARAM \
15 "name '%U' is parameter and nonlocal"
16
17 #define GLOBAL_AFTER_ASSIGN \
18 "name '%U' is assigned to before global declaration"
19
20 #define NONLOCAL_AFTER_ASSIGN \
21 "name '%U' is assigned to before nonlocal declaration"
22
23 #define GLOBAL_AFTER_USE \
24 "name '%U' is used prior to global declaration"
25
26 #define NONLOCAL_AFTER_USE \
27 "name '%U' is used prior to nonlocal declaration"
28
29 #define GLOBAL_ANNOT \
30 "annotated name '%U' can't be global"
31
32 #define NONLOCAL_ANNOT \
33 "annotated name '%U' can't be nonlocal"
34
35 #define IMPORT_STAR_WARNING "import * only allowed at module level"
36
37 #define NAMED_EXPR_COMP_IN_CLASS \
38 "assignment expression within a comprehension cannot be used in a class body"
39
40 #define NAMED_EXPR_COMP_IN_TYPEVAR_BOUND \
41 "assignment expression within a comprehension cannot be used in a TypeVar bound"
42
43 #define NAMED_EXPR_COMP_IN_TYPEALIAS \
44 "assignment expression within a comprehension cannot be used in a type alias"
45
46 #define NAMED_EXPR_COMP_IN_TYPEPARAM \
47 "assignment expression within a comprehension cannot be used within the definition of a generic"
48
49 #define NAMED_EXPR_COMP_CONFLICT \
50 "assignment expression cannot rebind comprehension iteration variable '%U'"
51
52 #define NAMED_EXPR_COMP_INNER_LOOP_CONFLICT \
53 "comprehension inner loop cannot rebind assignment expression target '%U'"
54
55 #define NAMED_EXPR_COMP_ITER_EXPR \
56 "assignment expression cannot be used in a comprehension iterable expression"
57
58 #define ANNOTATION_NOT_ALLOWED \
59 "%s cannot be used within an annotation"
60
61 #define EXPR_NOT_ALLOWED_IN_TYPE_VARIABLE \
62 "%s cannot be used within %s"
63
64 #define EXPR_NOT_ALLOWED_IN_TYPE_ALIAS \
65 "%s cannot be used within a type alias"
66
67 #define EXPR_NOT_ALLOWED_IN_TYPE_PARAMETERS \
68 "%s cannot be used within the definition of a generic"
69
70 #define DUPLICATE_TYPE_PARAM \
71 "duplicate type parameter '%U'"
72
73
74 #define LOCATION(x) \
75 (x)->lineno, (x)->col_offset, (x)->end_lineno, (x)->end_col_offset
76
77 #define ST_LOCATION(x) \
78 (x)->ste_lineno, (x)->ste_col_offset, (x)->ste_end_lineno, (x)->ste_end_col_offset
79
80 static PySTEntryObject *
ste_new(struct symtable * st,identifier name,_Py_block_ty block,void * key,int lineno,int col_offset,int end_lineno,int end_col_offset)81 ste_new(struct symtable *st, identifier name, _Py_block_ty block,
82 void *key, int lineno, int col_offset,
83 int end_lineno, int end_col_offset)
84 {
85 PySTEntryObject *ste = NULL;
86 PyObject *k = NULL;
87
88 k = PyLong_FromVoidPtr(key);
89 if (k == NULL)
90 goto fail;
91 ste = PyObject_New(PySTEntryObject, &PySTEntry_Type);
92 if (ste == NULL) {
93 Py_DECREF(k);
94 goto fail;
95 }
96 ste->ste_table = st;
97 ste->ste_id = k; /* ste owns reference to k */
98
99 ste->ste_name = Py_NewRef(name);
100
101 ste->ste_symbols = NULL;
102 ste->ste_varnames = NULL;
103 ste->ste_children = NULL;
104
105 ste->ste_directives = NULL;
106 ste->ste_mangled_names = NULL;
107
108 ste->ste_type = block;
109 ste->ste_scope_info = NULL;
110
111 ste->ste_nested = 0;
112 ste->ste_free = 0;
113 ste->ste_varargs = 0;
114 ste->ste_varkeywords = 0;
115 ste->ste_opt_lineno = 0;
116 ste->ste_opt_col_offset = 0;
117 ste->ste_lineno = lineno;
118 ste->ste_col_offset = col_offset;
119 ste->ste_end_lineno = end_lineno;
120 ste->ste_end_col_offset = end_col_offset;
121
122 if (st->st_cur != NULL &&
123 (st->st_cur->ste_nested ||
124 _PyST_IsFunctionLike(st->st_cur)))
125 ste->ste_nested = 1;
126 ste->ste_child_free = 0;
127 ste->ste_generator = 0;
128 ste->ste_coroutine = 0;
129 ste->ste_comprehension = NoComprehension;
130 ste->ste_returns_value = 0;
131 ste->ste_needs_class_closure = 0;
132 ste->ste_comp_inlined = 0;
133 ste->ste_comp_iter_target = 0;
134 ste->ste_can_see_class_scope = 0;
135 ste->ste_comp_iter_expr = 0;
136 ste->ste_needs_classdict = 0;
137
138 ste->ste_symbols = PyDict_New();
139 ste->ste_varnames = PyList_New(0);
140 ste->ste_children = PyList_New(0);
141 if (ste->ste_symbols == NULL
142 || ste->ste_varnames == NULL
143 || ste->ste_children == NULL)
144 goto fail;
145
146 if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0)
147 goto fail;
148
149 return ste;
150 fail:
151 Py_XDECREF(ste);
152 return NULL;
153 }
154
155 static PyObject *
ste_repr(PySTEntryObject * ste)156 ste_repr(PySTEntryObject *ste)
157 {
158 return PyUnicode_FromFormat("<symtable entry %U(%R), line %d>",
159 ste->ste_name, ste->ste_id, ste->ste_lineno);
160 }
161
162 static void
ste_dealloc(PySTEntryObject * ste)163 ste_dealloc(PySTEntryObject *ste)
164 {
165 ste->ste_table = NULL;
166 Py_XDECREF(ste->ste_id);
167 Py_XDECREF(ste->ste_name);
168 Py_XDECREF(ste->ste_symbols);
169 Py_XDECREF(ste->ste_varnames);
170 Py_XDECREF(ste->ste_children);
171 Py_XDECREF(ste->ste_directives);
172 Py_XDECREF(ste->ste_mangled_names);
173 PyObject_Free(ste);
174 }
175
176 #define OFF(x) offsetof(PySTEntryObject, x)
177
178 static PyMemberDef ste_memberlist[] = {
179 {"id", _Py_T_OBJECT, OFF(ste_id), Py_READONLY},
180 {"name", _Py_T_OBJECT, OFF(ste_name), Py_READONLY},
181 {"symbols", _Py_T_OBJECT, OFF(ste_symbols), Py_READONLY},
182 {"varnames", _Py_T_OBJECT, OFF(ste_varnames), Py_READONLY},
183 {"children", _Py_T_OBJECT, OFF(ste_children), Py_READONLY},
184 {"nested", Py_T_INT, OFF(ste_nested), Py_READONLY},
185 {"type", Py_T_INT, OFF(ste_type), Py_READONLY},
186 {"lineno", Py_T_INT, OFF(ste_lineno), Py_READONLY},
187 {NULL}
188 };
189
190 PyTypeObject PySTEntry_Type = {
191 PyVarObject_HEAD_INIT(&PyType_Type, 0)
192 "symtable entry",
193 sizeof(PySTEntryObject),
194 0,
195 (destructor)ste_dealloc, /* tp_dealloc */
196 0, /* tp_vectorcall_offset */
197 0, /* tp_getattr */
198 0, /* tp_setattr */
199 0, /* tp_as_async */
200 (reprfunc)ste_repr, /* tp_repr */
201 0, /* tp_as_number */
202 0, /* tp_as_sequence */
203 0, /* tp_as_mapping */
204 0, /* tp_hash */
205 0, /* tp_call */
206 0, /* tp_str */
207 PyObject_GenericGetAttr, /* tp_getattro */
208 0, /* tp_setattro */
209 0, /* tp_as_buffer */
210 Py_TPFLAGS_DEFAULT, /* tp_flags */
211 0, /* tp_doc */
212 0, /* tp_traverse */
213 0, /* tp_clear */
214 0, /* tp_richcompare */
215 0, /* tp_weaklistoffset */
216 0, /* tp_iter */
217 0, /* tp_iternext */
218 0, /* tp_methods */
219 ste_memberlist, /* tp_members */
220 0, /* tp_getset */
221 0, /* tp_base */
222 0, /* tp_dict */
223 0, /* tp_descr_get */
224 0, /* tp_descr_set */
225 0, /* tp_dictoffset */
226 0, /* tp_init */
227 0, /* tp_alloc */
228 0, /* tp_new */
229 };
230
231 static int symtable_analyze(struct symtable *st);
232 static int symtable_enter_block(struct symtable *st, identifier name,
233 _Py_block_ty block, void *ast,
234 int lineno, int col_offset,
235 int end_lineno, int end_col_offset);
236 static int symtable_exit_block(struct symtable *st);
237 static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
238 static int symtable_visit_expr(struct symtable *st, expr_ty s);
239 static int symtable_visit_type_param(struct symtable *st, type_param_ty s);
240 static int symtable_visit_genexp(struct symtable *st, expr_ty s);
241 static int symtable_visit_listcomp(struct symtable *st, expr_ty s);
242 static int symtable_visit_setcomp(struct symtable *st, expr_ty s);
243 static int symtable_visit_dictcomp(struct symtable *st, expr_ty s);
244 static int symtable_visit_arguments(struct symtable *st, arguments_ty);
245 static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty);
246 static int symtable_visit_alias(struct symtable *st, alias_ty);
247 static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
248 static int symtable_visit_keyword(struct symtable *st, keyword_ty);
249 static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args);
250 static int symtable_visit_annotation(struct symtable *st, expr_ty annotation);
251 static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args);
252 static int symtable_implicit_arg(struct symtable *st, int pos);
253 static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty);
254 static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
255 static int symtable_visit_match_case(struct symtable *st, match_case_ty m);
256 static int symtable_visit_pattern(struct symtable *st, pattern_ty s);
257 static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty);
258 static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty);
259
260 /* For debugging purposes only */
261 #if _PY_DUMP_SYMTABLE
_dump_symtable(PySTEntryObject * ste,PyObject * prefix)262 static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix)
263 {
264 const char *blocktype = "";
265 switch (ste->ste_type) {
266 case FunctionBlock: blocktype = "FunctionBlock"; break;
267 case ClassBlock: blocktype = "ClassBlock"; break;
268 case ModuleBlock: blocktype = "ModuleBlock"; break;
269 case AnnotationBlock: blocktype = "AnnotationBlock"; break;
270 case TypeVariableBlock: blocktype = "TypeVariableBlock"; break;
271 case TypeAliasBlock: blocktype = "TypeAliasBlock"; break;
272 case TypeParametersBlock: blocktype = "TypeParametersBlock"; break;
273 }
274 const char *comptype = "";
275 switch (ste->ste_comprehension) {
276 case ListComprehension: comptype = " ListComprehension"; break;
277 case DictComprehension: comptype = " DictComprehension"; break;
278 case SetComprehension: comptype = " SetComprehension"; break;
279 case GeneratorExpression: comptype = " GeneratorExpression"; break;
280 case NoComprehension: break;
281 }
282 PyObject* msg = PyUnicode_FromFormat(
283 (
284 "%U=== Symtable for %U ===\n"
285 "%U%s%s\n"
286 "%U%s%s%s%s%s%s%s%s%s%s%s%s%s\n"
287 "%Ulineno: %d col_offset: %d\n"
288 "%U--- Symbols ---\n"
289 ),
290 prefix,
291 ste->ste_name,
292 prefix,
293 blocktype,
294 comptype,
295 prefix,
296 ste->ste_nested ? " nested" : "",
297 ste->ste_free ? " free" : "",
298 ste->ste_child_free ? " child_free" : "",
299 ste->ste_generator ? " generator" : "",
300 ste->ste_coroutine ? " coroutine" : "",
301 ste->ste_varargs ? " varargs" : "",
302 ste->ste_varkeywords ? " varkeywords" : "",
303 ste->ste_returns_value ? " returns_value" : "",
304 ste->ste_needs_class_closure ? " needs_class_closure" : "",
305 ste->ste_needs_classdict ? " needs_classdict" : "",
306 ste->ste_comp_inlined ? " comp_inlined" : "",
307 ste->ste_comp_iter_target ? " comp_iter_target" : "",
308 ste->ste_can_see_class_scope ? " can_see_class_scope" : "",
309 prefix,
310 ste->ste_lineno,
311 ste->ste_col_offset,
312 prefix
313 );
314 assert(msg != NULL);
315 printf("%s", PyUnicode_AsUTF8(msg));
316 Py_DECREF(msg);
317 PyObject *name, *value;
318 Py_ssize_t pos = 0;
319 while (PyDict_Next(ste->ste_symbols, &pos, &name, &value)) {
320 int scope = _PyST_GetScope(ste, name);
321 long flags = _PyST_GetSymbol(ste, name);
322 printf("%s %s: ", PyUnicode_AsUTF8(prefix), PyUnicode_AsUTF8(name));
323 if (flags & DEF_GLOBAL) printf(" DEF_GLOBAL");
324 if (flags & DEF_LOCAL) printf(" DEF_LOCAL");
325 if (flags & DEF_PARAM) printf(" DEF_PARAM");
326 if (flags & DEF_NONLOCAL) printf(" DEF_NONLOCAL");
327 if (flags & USE) printf(" USE");
328 if (flags & DEF_FREE) printf(" DEF_FREE");
329 if (flags & DEF_FREE_CLASS) printf(" DEF_FREE_CLASS");
330 if (flags & DEF_IMPORT) printf(" DEF_IMPORT");
331 if (flags & DEF_ANNOT) printf(" DEF_ANNOT");
332 if (flags & DEF_COMP_ITER) printf(" DEF_COMP_ITER");
333 if (flags & DEF_TYPE_PARAM) printf(" DEF_TYPE_PARAM");
334 if (flags & DEF_COMP_CELL) printf(" DEF_COMP_CELL");
335 switch (scope) {
336 case LOCAL: printf(" LOCAL"); break;
337 case GLOBAL_EXPLICIT: printf(" GLOBAL_EXPLICIT"); break;
338 case GLOBAL_IMPLICIT: printf(" GLOBAL_IMPLICIT"); break;
339 case FREE: printf(" FREE"); break;
340 case CELL: printf(" CELL"); break;
341 }
342 printf("\n");
343 }
344 printf("%s--- Children ---\n", PyUnicode_AsUTF8(prefix));
345 PyObject *new_prefix = PyUnicode_FromFormat(" %U", prefix);
346 assert(new_prefix != NULL);
347 for (Py_ssize_t i = 0; i < PyList_GET_SIZE(ste->ste_children); i++) {
348 PyObject *child = PyList_GetItem(ste->ste_children, i);
349 assert(child != NULL && PySTEntry_Check(child));
350 _dump_symtable((PySTEntryObject *)child, new_prefix);
351 }
352 Py_DECREF(new_prefix);
353 }
354
dump_symtable(PySTEntryObject * ste)355 static void dump_symtable(PySTEntryObject* ste)
356 {
357 PyObject *empty = PyUnicode_FromString("");
358 assert(empty != NULL);
359 _dump_symtable(ste, empty);
360 Py_DECREF(empty);
361 }
362 #endif
363
364 #define DUPLICATE_ARGUMENT \
365 "duplicate argument '%U' in function definition"
366
367 static struct symtable *
symtable_new(void)368 symtable_new(void)
369 {
370 struct symtable *st;
371
372 st = (struct symtable *)PyMem_Malloc(sizeof(struct symtable));
373 if (st == NULL) {
374 PyErr_NoMemory();
375 return NULL;
376 }
377
378 st->st_filename = NULL;
379 st->st_blocks = NULL;
380
381 if ((st->st_stack = PyList_New(0)) == NULL)
382 goto fail;
383 if ((st->st_blocks = PyDict_New()) == NULL)
384 goto fail;
385 st->st_cur = NULL;
386 st->st_private = NULL;
387 return st;
388 fail:
389 _PySymtable_Free(st);
390 return NULL;
391 }
392
393 struct symtable *
_PySymtable_Build(mod_ty mod,PyObject * filename,_PyFutureFeatures * future)394 _PySymtable_Build(mod_ty mod, PyObject *filename, _PyFutureFeatures *future)
395 {
396 struct symtable *st = symtable_new();
397 asdl_stmt_seq *seq;
398 int i;
399 PyThreadState *tstate;
400 int starting_recursion_depth;
401
402 if (st == NULL)
403 return NULL;
404 if (filename == NULL) {
405 _PySymtable_Free(st);
406 return NULL;
407 }
408 st->st_filename = Py_NewRef(filename);
409 st->st_future = future;
410
411 /* Setup recursion depth check counters */
412 tstate = _PyThreadState_GET();
413 if (!tstate) {
414 _PySymtable_Free(st);
415 return NULL;
416 }
417 /* Be careful here to prevent overflow. */
418 int recursion_depth = Py_C_RECURSION_LIMIT - tstate->c_recursion_remaining;
419 starting_recursion_depth = recursion_depth;
420 st->recursion_depth = starting_recursion_depth;
421 st->recursion_limit = Py_C_RECURSION_LIMIT;
422
423 /* Make the initial symbol information gathering pass */
424 if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) {
425 _PySymtable_Free(st);
426 return NULL;
427 }
428
429 st->st_top = st->st_cur;
430 switch (mod->kind) {
431 case Module_kind:
432 seq = mod->v.Module.body;
433 for (i = 0; i < asdl_seq_LEN(seq); i++)
434 if (!symtable_visit_stmt(st,
435 (stmt_ty)asdl_seq_GET(seq, i)))
436 goto error;
437 break;
438 case Expression_kind:
439 if (!symtable_visit_expr(st, mod->v.Expression.body))
440 goto error;
441 break;
442 case Interactive_kind:
443 seq = mod->v.Interactive.body;
444 for (i = 0; i < asdl_seq_LEN(seq); i++)
445 if (!symtable_visit_stmt(st,
446 (stmt_ty)asdl_seq_GET(seq, i)))
447 goto error;
448 break;
449 case FunctionType_kind:
450 PyErr_SetString(PyExc_RuntimeError,
451 "this compiler does not handle FunctionTypes");
452 goto error;
453 }
454 if (!symtable_exit_block(st)) {
455 _PySymtable_Free(st);
456 return NULL;
457 }
458 /* Check that the recursion depth counting balanced correctly */
459 if (st->recursion_depth != starting_recursion_depth) {
460 PyErr_Format(PyExc_SystemError,
461 "symtable analysis recursion depth mismatch (before=%d, after=%d)",
462 starting_recursion_depth, st->recursion_depth);
463 _PySymtable_Free(st);
464 return NULL;
465 }
466 /* Make the second symbol analysis pass */
467 if (symtable_analyze(st)) {
468 #if _PY_DUMP_SYMTABLE
469 dump_symtable(st->st_top);
470 #endif
471 return st;
472 }
473 _PySymtable_Free(st);
474 return NULL;
475 error:
476 (void) symtable_exit_block(st);
477 _PySymtable_Free(st);
478 return NULL;
479 }
480
481
482 void
_PySymtable_Free(struct symtable * st)483 _PySymtable_Free(struct symtable *st)
484 {
485 Py_XDECREF(st->st_filename);
486 Py_XDECREF(st->st_blocks);
487 Py_XDECREF(st->st_stack);
488 PyMem_Free((void *)st);
489 }
490
491 PySTEntryObject *
_PySymtable_Lookup(struct symtable * st,void * key)492 _PySymtable_Lookup(struct symtable *st, void *key)
493 {
494 PyObject *k, *v;
495
496 k = PyLong_FromVoidPtr(key);
497 if (k == NULL)
498 return NULL;
499 if (PyDict_GetItemRef(st->st_blocks, k, &v) == 0) {
500 PyErr_SetString(PyExc_KeyError,
501 "unknown symbol table entry");
502 }
503 Py_DECREF(k);
504
505 assert(v == NULL || PySTEntry_Check(v));
506 return (PySTEntryObject *)v;
507 }
508
509 long
_PyST_GetSymbol(PySTEntryObject * ste,PyObject * name)510 _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name)
511 {
512 PyObject *v = PyDict_GetItemWithError(ste->ste_symbols, name);
513 if (!v)
514 return 0;
515 assert(PyLong_Check(v));
516 return PyLong_AS_LONG(v);
517 }
518
519 int
_PyST_GetScope(PySTEntryObject * ste,PyObject * name)520 _PyST_GetScope(PySTEntryObject *ste, PyObject *name)
521 {
522 long symbol = _PyST_GetSymbol(ste, name);
523 return (symbol >> SCOPE_OFFSET) & SCOPE_MASK;
524 }
525
526 int
_PyST_IsFunctionLike(PySTEntryObject * ste)527 _PyST_IsFunctionLike(PySTEntryObject *ste)
528 {
529 return ste->ste_type == FunctionBlock
530 || ste->ste_type == TypeVariableBlock
531 || ste->ste_type == TypeAliasBlock
532 || ste->ste_type == TypeParametersBlock;
533 }
534
535 static int
error_at_directive(PySTEntryObject * ste,PyObject * name)536 error_at_directive(PySTEntryObject *ste, PyObject *name)
537 {
538 Py_ssize_t i;
539 PyObject *data;
540 assert(ste->ste_directives);
541 for (i = 0; i < PyList_GET_SIZE(ste->ste_directives); i++) {
542 data = PyList_GET_ITEM(ste->ste_directives, i);
543 assert(PyTuple_CheckExact(data));
544 assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(data, 0)));
545 if (PyUnicode_Compare(PyTuple_GET_ITEM(data, 0), name) == 0) {
546 PyErr_RangedSyntaxLocationObject(ste->ste_table->st_filename,
547 PyLong_AsLong(PyTuple_GET_ITEM(data, 1)),
548 PyLong_AsLong(PyTuple_GET_ITEM(data, 2)) + 1,
549 PyLong_AsLong(PyTuple_GET_ITEM(data, 3)),
550 PyLong_AsLong(PyTuple_GET_ITEM(data, 4)) + 1);
551
552 return 0;
553 }
554 }
555 PyErr_SetString(PyExc_RuntimeError,
556 "BUG: internal directive bookkeeping broken");
557 return 0;
558 }
559
560
561 /* Analyze raw symbol information to determine scope of each name.
562
563 The next several functions are helpers for symtable_analyze(),
564 which determines whether a name is local, global, or free. In addition,
565 it determines which local variables are cell variables; they provide
566 bindings that are used for free variables in enclosed blocks.
567
568 There are also two kinds of global variables, implicit and explicit. An
569 explicit global is declared with the global statement. An implicit
570 global is a free variable for which the compiler has found no binding
571 in an enclosing function scope. The implicit global is either a global
572 or a builtin. Python's module and class blocks use the xxx_NAME opcodes
573 to handle these names to implement slightly odd semantics. In such a
574 block, the name is treated as global until it is assigned to; then it
575 is treated as a local.
576
577 The symbol table requires two passes to determine the scope of each name.
578 The first pass collects raw facts from the AST via the symtable_visit_*
579 functions: the name is a parameter here, the name is used but not defined
580 here, etc. The second pass analyzes these facts during a pass over the
581 PySTEntryObjects created during pass 1.
582
583 When a function is entered during the second pass, the parent passes
584 the set of all name bindings visible to its children. These bindings
585 are used to determine if non-local variables are free or implicit globals.
586 Names which are explicitly declared nonlocal must exist in this set of
587 visible names - if they do not, a syntax error is raised. After doing
588 the local analysis, it analyzes each of its child blocks using an
589 updated set of name bindings.
590
591 The children update the free variable set. If a local variable is added to
592 the free variable set by the child, the variable is marked as a cell. The
593 function object being defined must provide runtime storage for the variable
594 that may outlive the function's frame. Cell variables are removed from the
595 free set before the analyze function returns to its parent.
596
597 During analysis, the names are:
598 symbols: dict mapping from symbol names to flag values (including offset scope values)
599 scopes: dict mapping from symbol names to scope values (no offset)
600 local: set of all symbol names local to the current scope
601 bound: set of all symbol names local to a containing function scope
602 free: set of all symbol names referenced but not bound in child scopes
603 global: set of all symbol names explicitly declared as global
604 */
605
606 #define SET_SCOPE(DICT, NAME, I) { \
607 PyObject *o = PyLong_FromLong(I); \
608 if (!o) \
609 return 0; \
610 if (PyDict_SetItem((DICT), (NAME), o) < 0) { \
611 Py_DECREF(o); \
612 return 0; \
613 } \
614 Py_DECREF(o); \
615 }
616
617 /* Decide on scope of name, given flags.
618
619 The namespace dictionaries may be modified to record information
620 about the new name. For example, a new global will add an entry to
621 global. A name that was global can be changed to local.
622 */
623
624 static int
analyze_name(PySTEntryObject * ste,PyObject * scopes,PyObject * name,long flags,PyObject * bound,PyObject * local,PyObject * free,PyObject * global,PyObject * type_params,PySTEntryObject * class_entry)625 analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
626 PyObject *bound, PyObject *local, PyObject *free,
627 PyObject *global, PyObject *type_params, PySTEntryObject *class_entry)
628 {
629 int contains;
630 if (flags & DEF_GLOBAL) {
631 if (flags & DEF_NONLOCAL) {
632 PyErr_Format(PyExc_SyntaxError,
633 "name '%U' is nonlocal and global",
634 name);
635 return error_at_directive(ste, name);
636 }
637 SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
638 if (PySet_Add(global, name) < 0)
639 return 0;
640 if (bound && (PySet_Discard(bound, name) < 0))
641 return 0;
642 return 1;
643 }
644 if (flags & DEF_NONLOCAL) {
645 if (!bound) {
646 PyErr_Format(PyExc_SyntaxError,
647 "nonlocal declaration not allowed at module level");
648 return error_at_directive(ste, name);
649 }
650 contains = PySet_Contains(bound, name);
651 if (contains < 0) {
652 return 0;
653 }
654 if (!contains) {
655 PyErr_Format(PyExc_SyntaxError,
656 "no binding for nonlocal '%U' found",
657 name);
658
659 return error_at_directive(ste, name);
660 }
661 contains = PySet_Contains(type_params, name);
662 if (contains < 0) {
663 return 0;
664 }
665 if (contains) {
666 PyErr_Format(PyExc_SyntaxError,
667 "nonlocal binding not allowed for type parameter '%U'",
668 name);
669 return error_at_directive(ste, name);
670 }
671 SET_SCOPE(scopes, name, FREE);
672 ste->ste_free = 1;
673 return PySet_Add(free, name) >= 0;
674 }
675 if (flags & DEF_BOUND) {
676 SET_SCOPE(scopes, name, LOCAL);
677 if (PySet_Add(local, name) < 0)
678 return 0;
679 if (PySet_Discard(global, name) < 0)
680 return 0;
681 if (flags & DEF_TYPE_PARAM) {
682 if (PySet_Add(type_params, name) < 0)
683 return 0;
684 }
685 else {
686 if (PySet_Discard(type_params, name) < 0)
687 return 0;
688 }
689 return 1;
690 }
691 // If we were passed class_entry (i.e., we're in an ste_can_see_class_scope scope)
692 // and the bound name is in that set, then the name is potentially bound both by
693 // the immediately enclosing class namespace, and also by an outer function namespace.
694 // In that case, we want the runtime name resolution to look at only the class
695 // namespace and the globals (not the namespace providing the bound).
696 // Similarly, if the name is explicitly global in the class namespace (through the
697 // global statement), we want to also treat it as a global in this scope.
698 if (class_entry != NULL) {
699 long class_flags = _PyST_GetSymbol(class_entry, name);
700 if (class_flags & DEF_GLOBAL) {
701 SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
702 return 1;
703 }
704 else if (class_flags & DEF_BOUND && !(class_flags & DEF_NONLOCAL)) {
705 SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
706 return 1;
707 }
708 }
709 /* If an enclosing block has a binding for this name, it
710 is a free variable rather than a global variable.
711 Note that having a non-NULL bound implies that the block
712 is nested.
713 */
714 if (bound) {
715 contains = PySet_Contains(bound, name);
716 if (contains < 0) {
717 return 0;
718 }
719 if (contains) {
720 SET_SCOPE(scopes, name, FREE);
721 ste->ste_free = 1;
722 return PySet_Add(free, name) >= 0;
723 }
724 }
725 /* If a parent has a global statement, then call it global
726 explicit? It could also be global implicit.
727 */
728 if (global) {
729 contains = PySet_Contains(global, name);
730 if (contains < 0) {
731 return 0;
732 }
733 if (contains) {
734 SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
735 return 1;
736 }
737 }
738 if (ste->ste_nested)
739 ste->ste_free = 1;
740 SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
741 return 1;
742 }
743
744 static int
is_free_in_any_child(PySTEntryObject * entry,PyObject * key)745 is_free_in_any_child(PySTEntryObject *entry, PyObject *key)
746 {
747 for (Py_ssize_t i = 0; i < PyList_GET_SIZE(entry->ste_children); i++) {
748 PySTEntryObject *child_ste = (PySTEntryObject *)PyList_GET_ITEM(
749 entry->ste_children, i);
750 long scope = _PyST_GetScope(child_ste, key);
751 if (scope == FREE) {
752 return 1;
753 }
754 }
755 return 0;
756 }
757
758 static int
inline_comprehension(PySTEntryObject * ste,PySTEntryObject * comp,PyObject * scopes,PyObject * comp_free,PyObject * inlined_cells)759 inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp,
760 PyObject *scopes, PyObject *comp_free,
761 PyObject *inlined_cells)
762 {
763 PyObject *k, *v;
764 Py_ssize_t pos = 0;
765 int remove_dunder_class = 0;
766
767 while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) {
768 // skip comprehension parameter
769 long comp_flags = PyLong_AS_LONG(v);
770 if (comp_flags & DEF_PARAM) {
771 assert(_PyUnicode_EqualToASCIIString(k, ".0"));
772 continue;
773 }
774 int scope = (comp_flags >> SCOPE_OFFSET) & SCOPE_MASK;
775 int only_flags = comp_flags & ((1 << SCOPE_OFFSET) - 1);
776 if (scope == CELL || only_flags & DEF_COMP_CELL) {
777 if (PySet_Add(inlined_cells, k) < 0) {
778 return 0;
779 }
780 }
781 PyObject *existing = PyDict_GetItemWithError(ste->ste_symbols, k);
782 if (existing == NULL && PyErr_Occurred()) {
783 return 0;
784 }
785 // __class__ is never allowed to be free through a class scope (see
786 // drop_class_free)
787 if (scope == FREE && ste->ste_type == ClassBlock &&
788 _PyUnicode_EqualToASCIIString(k, "__class__")) {
789 scope = GLOBAL_IMPLICIT;
790 if (PySet_Discard(comp_free, k) < 0) {
791 return 0;
792 }
793 remove_dunder_class = 1;
794 }
795 if (!existing) {
796 // name does not exist in scope, copy from comprehension
797 assert(scope != FREE || PySet_Contains(comp_free, k) == 1);
798 PyObject *v_flags = PyLong_FromLong(only_flags);
799 if (v_flags == NULL) {
800 return 0;
801 }
802 int ok = PyDict_SetItem(ste->ste_symbols, k, v_flags);
803 Py_DECREF(v_flags);
804 if (ok < 0) {
805 return 0;
806 }
807 SET_SCOPE(scopes, k, scope);
808 }
809 else {
810 if (PyLong_AsLong(existing) & DEF_BOUND) {
811 // free vars in comprehension that are locals in outer scope can
812 // now simply be locals, unless they are free in comp children,
813 // or if the outer scope is a class block
814 if (!is_free_in_any_child(comp, k) && ste->ste_type != ClassBlock) {
815 if (PySet_Discard(comp_free, k) < 0) {
816 return 0;
817 }
818 }
819 }
820 }
821 }
822 comp->ste_free = PySet_Size(comp_free) > 0;
823 if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) {
824 return 0;
825 }
826 return 1;
827 }
828
829 #undef SET_SCOPE
830
831 /* If a name is defined in free and also in locals, then this block
832 provides the binding for the free variable. The name should be
833 marked CELL in this block and removed from the free list.
834
835 Note that the current block's free variables are included in free.
836 That's safe because no name can be free and local in the same scope.
837 */
838
839 static int
analyze_cells(PyObject * scopes,PyObject * free,PyObject * inlined_cells)840 analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells)
841 {
842 PyObject *name, *v, *v_cell;
843 int success = 0;
844 Py_ssize_t pos = 0;
845
846 v_cell = PyLong_FromLong(CELL);
847 if (!v_cell)
848 return 0;
849 while (PyDict_Next(scopes, &pos, &name, &v)) {
850 long scope;
851 assert(PyLong_Check(v));
852 scope = PyLong_AS_LONG(v);
853 if (scope != LOCAL)
854 continue;
855 int contains = PySet_Contains(free, name);
856 if (contains < 0) {
857 goto error;
858 }
859 if (!contains) {
860 contains = PySet_Contains(inlined_cells, name);
861 if (contains < 0) {
862 goto error;
863 }
864 if (!contains) {
865 continue;
866 }
867 }
868 /* Replace LOCAL with CELL for this name, and remove
869 from free. It is safe to replace the value of name
870 in the dict, because it will not cause a resize.
871 */
872 if (PyDict_SetItem(scopes, name, v_cell) < 0)
873 goto error;
874 if (PySet_Discard(free, name) < 0)
875 goto error;
876 }
877 success = 1;
878 error:
879 Py_DECREF(v_cell);
880 return success;
881 }
882
883 static int
drop_class_free(PySTEntryObject * ste,PyObject * free)884 drop_class_free(PySTEntryObject *ste, PyObject *free)
885 {
886 int res;
887 res = PySet_Discard(free, &_Py_ID(__class__));
888 if (res < 0)
889 return 0;
890 if (res)
891 ste->ste_needs_class_closure = 1;
892 res = PySet_Discard(free, &_Py_ID(__classdict__));
893 if (res < 0)
894 return 0;
895 if (res)
896 ste->ste_needs_classdict = 1;
897 return 1;
898 }
899
900 /* Enter the final scope information into the ste_symbols dict.
901 *
902 * All arguments are dicts. Modifies symbols, others are read-only.
903 */
904 static int
update_symbols(PyObject * symbols,PyObject * scopes,PyObject * bound,PyObject * free,PyObject * inlined_cells,int classflag)905 update_symbols(PyObject *symbols, PyObject *scopes,
906 PyObject *bound, PyObject *free,
907 PyObject *inlined_cells, int classflag)
908 {
909 PyObject *name = NULL, *itr = NULL;
910 PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL;
911 Py_ssize_t pos = 0;
912
913 /* Update scope information for all symbols in this scope */
914 while (PyDict_Next(symbols, &pos, &name, &v)) {
915 long scope, flags;
916 assert(PyLong_Check(v));
917 flags = PyLong_AS_LONG(v);
918 int contains = PySet_Contains(inlined_cells, name);
919 if (contains < 0) {
920 return 0;
921 }
922 if (contains) {
923 flags |= DEF_COMP_CELL;
924 }
925 v_scope = PyDict_GetItemWithError(scopes, name);
926 assert(v_scope && PyLong_Check(v_scope));
927 scope = PyLong_AS_LONG(v_scope);
928 flags |= (scope << SCOPE_OFFSET);
929 v_new = PyLong_FromLong(flags);
930 if (!v_new)
931 return 0;
932 if (PyDict_SetItem(symbols, name, v_new) < 0) {
933 Py_DECREF(v_new);
934 return 0;
935 }
936 Py_DECREF(v_new);
937 }
938
939 /* Record not yet resolved free variables from children (if any) */
940 v_free = PyLong_FromLong(FREE << SCOPE_OFFSET);
941 if (!v_free)
942 return 0;
943
944 itr = PyObject_GetIter(free);
945 if (itr == NULL) {
946 Py_DECREF(v_free);
947 return 0;
948 }
949
950 while ((name = PyIter_Next(itr))) {
951 v = PyDict_GetItemWithError(symbols, name);
952
953 /* Handle symbol that already exists in this scope */
954 if (v) {
955 /* Handle a free variable in a method of
956 the class that has the same name as a local
957 or global in the class scope.
958 */
959 if (classflag) {
960 long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS;
961 v_new = PyLong_FromLong(flags);
962 if (!v_new) {
963 goto error;
964 }
965 if (PyDict_SetItem(symbols, name, v_new) < 0) {
966 Py_DECREF(v_new);
967 goto error;
968 }
969 Py_DECREF(v_new);
970 }
971 /* It's a cell, or already free in this scope */
972 Py_DECREF(name);
973 continue;
974 }
975 else if (PyErr_Occurred()) {
976 goto error;
977 }
978 /* Handle global symbol */
979 if (bound) {
980 int contains = PySet_Contains(bound, name);
981 if (contains < 0) {
982 goto error;
983 }
984 if (!contains) {
985 Py_DECREF(name);
986 continue; /* it's a global */
987 }
988 }
989 /* Propagate new free symbol up the lexical stack */
990 if (PyDict_SetItem(symbols, name, v_free) < 0) {
991 goto error;
992 }
993 Py_DECREF(name);
994 }
995
996 /* Check if loop ended because of exception in PyIter_Next */
997 if (PyErr_Occurred()) {
998 goto error;
999 }
1000
1001 Py_DECREF(itr);
1002 Py_DECREF(v_free);
1003 return 1;
1004 error:
1005 Py_XDECREF(v_free);
1006 Py_XDECREF(itr);
1007 Py_XDECREF(name);
1008 return 0;
1009 }
1010
1011 /* Make final symbol table decisions for block of ste.
1012
1013 Arguments:
1014 ste -- current symtable entry (input/output)
1015 bound -- set of variables bound in enclosing scopes (input). bound
1016 is NULL for module blocks.
1017 free -- set of free variables in enclosed scopes (output)
1018 globals -- set of declared global variables in enclosing scopes (input)
1019
1020 The implementation uses two mutually recursive functions,
1021 analyze_block() and analyze_child_block(). analyze_block() is
1022 responsible for analyzing the individual names defined in a block.
1023 analyze_child_block() prepares temporary namespace dictionaries
1024 used to evaluated nested blocks.
1025
1026 The two functions exist because a child block should see the name
1027 bindings of its enclosing blocks, but those bindings should not
1028 propagate back to a parent block.
1029 */
1030
1031 static int
1032 analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
1033 PyObject *global, PyObject *type_params,
1034 PySTEntryObject *class_entry, PyObject **child_free);
1035
1036 static int
analyze_block(PySTEntryObject * ste,PyObject * bound,PyObject * free,PyObject * global,PyObject * type_params,PySTEntryObject * class_entry)1037 analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
1038 PyObject *global, PyObject *type_params,
1039 PySTEntryObject *class_entry)
1040 {
1041 PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL;
1042 PyObject *newglobal = NULL, *newfree = NULL, *inlined_cells = NULL;
1043 PyObject *temp;
1044 int success = 0;
1045 Py_ssize_t i, pos = 0;
1046
1047 local = PySet_New(NULL); /* collect new names bound in block */
1048 if (!local)
1049 goto error;
1050 scopes = PyDict_New(); /* collect scopes defined for each name */
1051 if (!scopes)
1052 goto error;
1053
1054 /* Allocate new global, bound and free variable sets. These
1055 sets hold the names visible in nested blocks. For
1056 ClassBlocks, the bound and global names are initialized
1057 before analyzing names, because class bindings aren't
1058 visible in methods. For other blocks, they are initialized
1059 after names are analyzed.
1060 */
1061
1062 /* TODO(jhylton): Package these dicts in a struct so that we
1063 can write reasonable helper functions?
1064 */
1065 newglobal = PySet_New(NULL);
1066 if (!newglobal)
1067 goto error;
1068 newfree = PySet_New(NULL);
1069 if (!newfree)
1070 goto error;
1071 newbound = PySet_New(NULL);
1072 if (!newbound)
1073 goto error;
1074 inlined_cells = PySet_New(NULL);
1075 if (!inlined_cells)
1076 goto error;
1077
1078 /* Class namespace has no effect on names visible in
1079 nested functions, so populate the global and bound
1080 sets to be passed to child blocks before analyzing
1081 this one.
1082 */
1083 if (ste->ste_type == ClassBlock) {
1084 /* Pass down known globals */
1085 temp = PyNumber_InPlaceOr(newglobal, global);
1086 if (!temp)
1087 goto error;
1088 Py_DECREF(temp);
1089 /* Pass down previously bound symbols */
1090 if (bound) {
1091 temp = PyNumber_InPlaceOr(newbound, bound);
1092 if (!temp)
1093 goto error;
1094 Py_DECREF(temp);
1095 }
1096 }
1097
1098 while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) {
1099 long flags = PyLong_AS_LONG(v);
1100 if (!analyze_name(ste, scopes, name, flags,
1101 bound, local, free, global, type_params, class_entry))
1102 goto error;
1103 }
1104
1105 /* Populate global and bound sets to be passed to children. */
1106 if (ste->ste_type != ClassBlock) {
1107 /* Add function locals to bound set */
1108 if (_PyST_IsFunctionLike(ste)) {
1109 temp = PyNumber_InPlaceOr(newbound, local);
1110 if (!temp)
1111 goto error;
1112 Py_DECREF(temp);
1113 }
1114 /* Pass down previously bound symbols */
1115 if (bound) {
1116 temp = PyNumber_InPlaceOr(newbound, bound);
1117 if (!temp)
1118 goto error;
1119 Py_DECREF(temp);
1120 }
1121 /* Pass down known globals */
1122 temp = PyNumber_InPlaceOr(newglobal, global);
1123 if (!temp)
1124 goto error;
1125 Py_DECREF(temp);
1126 }
1127 else {
1128 /* Special-case __class__ and __classdict__ */
1129 if (PySet_Add(newbound, &_Py_ID(__class__)) < 0)
1130 goto error;
1131 if (PySet_Add(newbound, &_Py_ID(__classdict__)) < 0)
1132 goto error;
1133 }
1134
1135 /* Recursively call analyze_child_block() on each child block.
1136
1137 newbound, newglobal now contain the names visible in
1138 nested blocks. The free variables in the children will
1139 be added to newfree.
1140 */
1141 for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) {
1142 PyObject *child_free = NULL;
1143 PyObject *c = PyList_GET_ITEM(ste->ste_children, i);
1144 PySTEntryObject* entry;
1145 assert(c && PySTEntry_Check(c));
1146 entry = (PySTEntryObject*)c;
1147
1148 PySTEntryObject *new_class_entry = NULL;
1149 if (entry->ste_can_see_class_scope) {
1150 if (ste->ste_type == ClassBlock) {
1151 new_class_entry = ste;
1152 }
1153 else if (class_entry) {
1154 new_class_entry = class_entry;
1155 }
1156 }
1157
1158 // we inline all non-generator-expression comprehensions,
1159 // except those in annotation scopes that are nested in classes
1160 int inline_comp =
1161 entry->ste_comprehension &&
1162 !entry->ste_generator &&
1163 !ste->ste_can_see_class_scope;
1164
1165 if (!analyze_child_block(entry, newbound, newfree, newglobal,
1166 type_params, new_class_entry, &child_free))
1167 {
1168 goto error;
1169 }
1170 if (inline_comp) {
1171 if (!inline_comprehension(ste, entry, scopes, child_free, inlined_cells)) {
1172 Py_DECREF(child_free);
1173 goto error;
1174 }
1175 entry->ste_comp_inlined = 1;
1176 }
1177 temp = PyNumber_InPlaceOr(newfree, child_free);
1178 Py_DECREF(child_free);
1179 if (!temp)
1180 goto error;
1181 Py_DECREF(temp);
1182 /* Check if any children have free variables */
1183 if (entry->ste_free || entry->ste_child_free)
1184 ste->ste_child_free = 1;
1185 }
1186
1187 /* Splice children of inlined comprehensions into our children list */
1188 for (i = PyList_GET_SIZE(ste->ste_children) - 1; i >= 0; --i) {
1189 PyObject* c = PyList_GET_ITEM(ste->ste_children, i);
1190 PySTEntryObject* entry;
1191 assert(c && PySTEntry_Check(c));
1192 entry = (PySTEntryObject*)c;
1193 if (entry->ste_comp_inlined &&
1194 PyList_SetSlice(ste->ste_children, i, i + 1,
1195 entry->ste_children) < 0)
1196 {
1197 goto error;
1198 }
1199 }
1200
1201 /* Check if any local variables must be converted to cell variables */
1202 if (_PyST_IsFunctionLike(ste) && !analyze_cells(scopes, newfree, inlined_cells))
1203 goto error;
1204 else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree))
1205 goto error;
1206 /* Records the results of the analysis in the symbol table entry */
1207 if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, inlined_cells,
1208 (ste->ste_type == ClassBlock) || ste->ste_can_see_class_scope))
1209 goto error;
1210
1211 temp = PyNumber_InPlaceOr(free, newfree);
1212 if (!temp)
1213 goto error;
1214 Py_DECREF(temp);
1215 success = 1;
1216 error:
1217 Py_XDECREF(scopes);
1218 Py_XDECREF(local);
1219 Py_XDECREF(newbound);
1220 Py_XDECREF(newglobal);
1221 Py_XDECREF(newfree);
1222 Py_XDECREF(inlined_cells);
1223 if (!success)
1224 assert(PyErr_Occurred());
1225 return success;
1226 }
1227
1228 static int
analyze_child_block(PySTEntryObject * entry,PyObject * bound,PyObject * free,PyObject * global,PyObject * type_params,PySTEntryObject * class_entry,PyObject ** child_free)1229 analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
1230 PyObject *global, PyObject *type_params,
1231 PySTEntryObject *class_entry, PyObject** child_free)
1232 {
1233 PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL;
1234 PyObject *temp_type_params = NULL;
1235
1236 /* Copy the bound/global/free sets.
1237
1238 These sets are used by all blocks enclosed by the
1239 current block. The analyze_block() call modifies these
1240 sets.
1241
1242 */
1243 temp_bound = PySet_New(bound);
1244 if (!temp_bound)
1245 goto error;
1246 temp_free = PySet_New(free);
1247 if (!temp_free)
1248 goto error;
1249 temp_global = PySet_New(global);
1250 if (!temp_global)
1251 goto error;
1252 temp_type_params = PySet_New(type_params);
1253 if (!temp_type_params)
1254 goto error;
1255
1256 if (!analyze_block(entry, temp_bound, temp_free, temp_global,
1257 temp_type_params, class_entry))
1258 goto error;
1259 *child_free = temp_free;
1260 Py_DECREF(temp_bound);
1261 Py_DECREF(temp_global);
1262 Py_DECREF(temp_type_params);
1263 return 1;
1264 error:
1265 Py_XDECREF(temp_bound);
1266 Py_XDECREF(temp_free);
1267 Py_XDECREF(temp_global);
1268 Py_XDECREF(temp_type_params);
1269 return 0;
1270 }
1271
1272 static int
symtable_analyze(struct symtable * st)1273 symtable_analyze(struct symtable *st)
1274 {
1275 PyObject *free, *global, *type_params;
1276 int r;
1277
1278 free = PySet_New(NULL);
1279 if (!free)
1280 return 0;
1281 global = PySet_New(NULL);
1282 if (!global) {
1283 Py_DECREF(free);
1284 return 0;
1285 }
1286 type_params = PySet_New(NULL);
1287 if (!type_params) {
1288 Py_DECREF(free);
1289 Py_DECREF(global);
1290 return 0;
1291 }
1292 r = analyze_block(st->st_top, NULL, free, global, type_params, NULL);
1293 Py_DECREF(free);
1294 Py_DECREF(global);
1295 Py_DECREF(type_params);
1296 return r;
1297 }
1298
1299 /* symtable_enter_block() gets a reference via ste_new.
1300 This reference is released when the block is exited, via the DECREF
1301 in symtable_exit_block().
1302 */
1303
1304 static int
symtable_exit_block(struct symtable * st)1305 symtable_exit_block(struct symtable *st)
1306 {
1307 Py_ssize_t size;
1308
1309 st->st_cur = NULL;
1310 size = PyList_GET_SIZE(st->st_stack);
1311 if (size) {
1312 if (PyList_SetSlice(st->st_stack, size - 1, size, NULL) < 0)
1313 return 0;
1314 if (--size)
1315 st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, size - 1);
1316 }
1317 return 1;
1318 }
1319
1320 static int
symtable_enter_block(struct symtable * st,identifier name,_Py_block_ty block,void * ast,int lineno,int col_offset,int end_lineno,int end_col_offset)1321 symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
1322 void *ast, int lineno, int col_offset,
1323 int end_lineno, int end_col_offset)
1324 {
1325 PySTEntryObject *prev = NULL, *ste;
1326
1327 ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset);
1328 if (ste == NULL)
1329 return 0;
1330 if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) {
1331 Py_DECREF(ste);
1332 return 0;
1333 }
1334 prev = st->st_cur;
1335 /* bpo-37757: For now, disallow *all* assignment expressions in the
1336 * outermost iterator expression of a comprehension, even those inside
1337 * a nested comprehension or a lambda expression.
1338 */
1339 if (prev) {
1340 ste->ste_comp_iter_expr = prev->ste_comp_iter_expr;
1341 }
1342 /* No need to inherit ste_mangled_names in classes, where all names
1343 * are mangled. */
1344 if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) {
1345 ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names);
1346 }
1347 /* The entry is owned by the stack. Borrow it for st_cur. */
1348 Py_DECREF(ste);
1349 st->st_cur = ste;
1350
1351 /* Annotation blocks shouldn't have any affect on the symbol table since in
1352 * the compilation stage, they will all be transformed to strings. They are
1353 * only created if future 'annotations' feature is activated. */
1354 if (block == AnnotationBlock) {
1355 return 1;
1356 }
1357
1358 if (block == ModuleBlock)
1359 st->st_global = st->st_cur->ste_symbols;
1360
1361 if (prev) {
1362 if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) {
1363 return 0;
1364 }
1365 }
1366 return 1;
1367 }
1368
1369 static long
symtable_lookup_entry(struct symtable * st,PySTEntryObject * ste,PyObject * name)1370 symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name)
1371 {
1372 PyObject *mangled = _Py_MaybeMangle(st->st_private, ste, name);
1373 if (!mangled)
1374 return 0;
1375 long ret = _PyST_GetSymbol(ste, mangled);
1376 Py_DECREF(mangled);
1377 return ret;
1378 }
1379
1380 static long
symtable_lookup(struct symtable * st,PyObject * name)1381 symtable_lookup(struct symtable *st, PyObject *name)
1382 {
1383 return symtable_lookup_entry(st, st->st_cur, name);
1384 }
1385
1386 static int
symtable_add_def_helper(struct symtable * st,PyObject * name,int flag,struct _symtable_entry * ste,int lineno,int col_offset,int end_lineno,int end_col_offset)1387 symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste,
1388 int lineno, int col_offset, int end_lineno, int end_col_offset)
1389 {
1390 PyObject *o;
1391 PyObject *dict;
1392 long val;
1393 PyObject *mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name);
1394
1395 if (!mangled)
1396 return 0;
1397 dict = ste->ste_symbols;
1398 if ((o = PyDict_GetItemWithError(dict, mangled))) {
1399 val = PyLong_AS_LONG(o);
1400 if ((flag & DEF_PARAM) && (val & DEF_PARAM)) {
1401 /* Is it better to use 'mangled' or 'name' here? */
1402 PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name);
1403 PyErr_RangedSyntaxLocationObject(st->st_filename,
1404 lineno, col_offset + 1,
1405 end_lineno, end_col_offset + 1);
1406 goto error;
1407 }
1408 if ((flag & DEF_TYPE_PARAM) && (val & DEF_TYPE_PARAM)) {
1409 PyErr_Format(PyExc_SyntaxError, DUPLICATE_TYPE_PARAM, name);
1410 PyErr_RangedSyntaxLocationObject(st->st_filename,
1411 lineno, col_offset + 1,
1412 end_lineno, end_col_offset + 1);
1413 goto error;
1414 }
1415 val |= flag;
1416 }
1417 else if (PyErr_Occurred()) {
1418 goto error;
1419 }
1420 else {
1421 val = flag;
1422 }
1423 if (ste->ste_comp_iter_target) {
1424 /* This name is an iteration variable in a comprehension,
1425 * so check for a binding conflict with any named expressions.
1426 * Otherwise, mark it as an iteration variable so subsequent
1427 * named expressions can check for conflicts.
1428 */
1429 if (val & (DEF_GLOBAL | DEF_NONLOCAL)) {
1430 PyErr_Format(PyExc_SyntaxError,
1431 NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name);
1432 PyErr_RangedSyntaxLocationObject(st->st_filename,
1433 lineno, col_offset + 1,
1434 end_lineno, end_col_offset + 1);
1435 goto error;
1436 }
1437 val |= DEF_COMP_ITER;
1438 }
1439 o = PyLong_FromLong(val);
1440 if (o == NULL)
1441 goto error;
1442 if (PyDict_SetItem(dict, mangled, o) < 0) {
1443 Py_DECREF(o);
1444 goto error;
1445 }
1446 Py_DECREF(o);
1447
1448 if (flag & DEF_PARAM) {
1449 if (PyList_Append(ste->ste_varnames, mangled) < 0)
1450 goto error;
1451 } else if (flag & DEF_GLOBAL) {
1452 /* XXX need to update DEF_GLOBAL for other flags too;
1453 perhaps only DEF_FREE_GLOBAL */
1454 val = flag;
1455 if ((o = PyDict_GetItemWithError(st->st_global, mangled))) {
1456 val |= PyLong_AS_LONG(o);
1457 }
1458 else if (PyErr_Occurred()) {
1459 goto error;
1460 }
1461 o = PyLong_FromLong(val);
1462 if (o == NULL)
1463 goto error;
1464 if (PyDict_SetItem(st->st_global, mangled, o) < 0) {
1465 Py_DECREF(o);
1466 goto error;
1467 }
1468 Py_DECREF(o);
1469 }
1470 Py_DECREF(mangled);
1471 return 1;
1472
1473 error:
1474 Py_DECREF(mangled);
1475 return 0;
1476 }
1477
1478 static int
symtable_add_def(struct symtable * st,PyObject * name,int flag,int lineno,int col_offset,int end_lineno,int end_col_offset)1479 symtable_add_def(struct symtable *st, PyObject *name, int flag,
1480 int lineno, int col_offset, int end_lineno, int end_col_offset)
1481 {
1482 if ((flag & DEF_TYPE_PARAM) && st->st_cur->ste_mangled_names != NULL) {
1483 if(PySet_Add(st->st_cur->ste_mangled_names, name) < 0) {
1484 return 0;
1485 }
1486 }
1487 return symtable_add_def_helper(st, name, flag, st->st_cur,
1488 lineno, col_offset, end_lineno, end_col_offset);
1489 }
1490
1491 static int
symtable_enter_type_param_block(struct symtable * st,identifier name,void * ast,int has_defaults,int has_kwdefaults,enum _stmt_kind kind,int lineno,int col_offset,int end_lineno,int end_col_offset)1492 symtable_enter_type_param_block(struct symtable *st, identifier name,
1493 void *ast, int has_defaults, int has_kwdefaults,
1494 enum _stmt_kind kind,
1495 int lineno, int col_offset,
1496 int end_lineno, int end_col_offset)
1497 {
1498 _Py_block_ty current_type = st->st_cur->ste_type;
1499 if(!symtable_enter_block(st, name, TypeParametersBlock, ast, lineno,
1500 col_offset, end_lineno, end_col_offset)) {
1501 return 0;
1502 }
1503 if (current_type == ClassBlock) {
1504 st->st_cur->ste_can_see_class_scope = 1;
1505 if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, lineno, col_offset, end_lineno, end_col_offset)) {
1506 return 0;
1507 }
1508 }
1509 if (kind == ClassDef_kind) {
1510 _Py_DECLARE_STR(type_params, ".type_params");
1511 // It gets "set" when we create the type params tuple and
1512 // "used" when we build up the bases.
1513 if (!symtable_add_def(st, &_Py_STR(type_params), DEF_LOCAL,
1514 lineno, col_offset, end_lineno, end_col_offset)) {
1515 return 0;
1516 }
1517 if (!symtable_add_def(st, &_Py_STR(type_params), USE,
1518 lineno, col_offset, end_lineno, end_col_offset)) {
1519 return 0;
1520 }
1521 // This is used for setting the generic base
1522 _Py_DECLARE_STR(generic_base, ".generic_base");
1523 if (!symtable_add_def(st, &_Py_STR(generic_base), DEF_LOCAL,
1524 lineno, col_offset, end_lineno, end_col_offset)) {
1525 return 0;
1526 }
1527 if (!symtable_add_def(st, &_Py_STR(generic_base), USE,
1528 lineno, col_offset, end_lineno, end_col_offset)) {
1529 return 0;
1530 }
1531 }
1532 if (has_defaults) {
1533 _Py_DECLARE_STR(defaults, ".defaults");
1534 if (!symtable_add_def(st, &_Py_STR(defaults), DEF_PARAM,
1535 lineno, col_offset, end_lineno, end_col_offset)) {
1536 return 0;
1537 }
1538 }
1539 if (has_kwdefaults) {
1540 _Py_DECLARE_STR(kwdefaults, ".kwdefaults");
1541 if (!symtable_add_def(st, &_Py_STR(kwdefaults), DEF_PARAM,
1542 lineno, col_offset, end_lineno, end_col_offset)) {
1543 return 0;
1544 }
1545 }
1546 return 1;
1547 }
1548
1549 /* VISIT, VISIT_SEQ and VIST_SEQ_TAIL take an ASDL type as their second argument.
1550 They use the ASDL name to synthesize the name of the C type and the visit
1551 function.
1552
1553 VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is
1554 useful if the first node in the sequence requires special treatment.
1555
1556 VISIT_QUIT macro returns the specified value exiting from the function but
1557 first adjusts current recursion counter depth.
1558 */
1559
1560 #define VISIT_QUIT(ST, X) \
1561 return --(ST)->recursion_depth,(X)
1562
1563 #define VISIT(ST, TYPE, V) \
1564 if (!symtable_visit_ ## TYPE((ST), (V))) \
1565 VISIT_QUIT((ST), 0);
1566
1567 #define VISIT_SEQ(ST, TYPE, SEQ) { \
1568 int i; \
1569 asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1570 for (i = 0; i < asdl_seq_LEN(seq); i++) { \
1571 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1572 if (!symtable_visit_ ## TYPE((ST), elt)) \
1573 VISIT_QUIT((ST), 0); \
1574 } \
1575 }
1576
1577 #define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \
1578 int i; \
1579 asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1580 for (i = (START); i < asdl_seq_LEN(seq); i++) { \
1581 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1582 if (!symtable_visit_ ## TYPE((ST), elt)) \
1583 VISIT_QUIT((ST), 0); \
1584 } \
1585 }
1586
1587 #define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) { \
1588 int i = 0; \
1589 asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1590 for (i = 0; i < asdl_seq_LEN(seq); i++) { \
1591 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1592 if (!elt) continue; /* can be NULL */ \
1593 if (!symtable_visit_ ## TYPE((ST), elt)) \
1594 VISIT_QUIT((ST), 0); \
1595 } \
1596 }
1597
1598 static int
symtable_record_directive(struct symtable * st,identifier name,int lineno,int col_offset,int end_lineno,int end_col_offset)1599 symtable_record_directive(struct symtable *st, identifier name, int lineno,
1600 int col_offset, int end_lineno, int end_col_offset)
1601 {
1602 PyObject *data, *mangled;
1603 int res;
1604 if (!st->st_cur->ste_directives) {
1605 st->st_cur->ste_directives = PyList_New(0);
1606 if (!st->st_cur->ste_directives)
1607 return 0;
1608 }
1609 mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name);
1610 if (!mangled)
1611 return 0;
1612 data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset);
1613 if (!data)
1614 return 0;
1615 res = PyList_Append(st->st_cur->ste_directives, data);
1616 Py_DECREF(data);
1617 return res == 0;
1618 }
1619
1620 static int
has_kwonlydefaults(asdl_arg_seq * kwonlyargs,asdl_expr_seq * kw_defaults)1621 has_kwonlydefaults(asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults)
1622 {
1623 for (int i = 0; i < asdl_seq_LEN(kwonlyargs); i++) {
1624 expr_ty default_ = asdl_seq_GET(kw_defaults, i);
1625 if (default_) {
1626 return 1;
1627 }
1628 }
1629 return 0;
1630 }
1631
1632 static int
symtable_visit_stmt(struct symtable * st,stmt_ty s)1633 symtable_visit_stmt(struct symtable *st, stmt_ty s)
1634 {
1635 if (++st->recursion_depth > st->recursion_limit) {
1636 PyErr_SetString(PyExc_RecursionError,
1637 "maximum recursion depth exceeded during compilation");
1638 VISIT_QUIT(st, 0);
1639 }
1640 switch (s->kind) {
1641 case FunctionDef_kind:
1642 if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s)))
1643 VISIT_QUIT(st, 0);
1644 if (s->v.FunctionDef.args->defaults)
1645 VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
1646 if (s->v.FunctionDef.args->kw_defaults)
1647 VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults);
1648 if (s->v.FunctionDef.decorator_list)
1649 VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list);
1650 if (asdl_seq_LEN(s->v.FunctionDef.type_params) > 0) {
1651 if (!symtable_enter_type_param_block(
1652 st, s->v.FunctionDef.name,
1653 (void *)s->v.FunctionDef.type_params,
1654 s->v.FunctionDef.args->defaults != NULL,
1655 has_kwonlydefaults(s->v.FunctionDef.args->kwonlyargs,
1656 s->v.FunctionDef.args->kw_defaults),
1657 s->kind,
1658 LOCATION(s))) {
1659 VISIT_QUIT(st, 0);
1660 }
1661 VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params);
1662 }
1663 if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
1664 s->v.FunctionDef.returns))
1665 VISIT_QUIT(st, 0);
1666 if (!symtable_enter_block(st, s->v.FunctionDef.name,
1667 FunctionBlock, (void *)s,
1668 LOCATION(s)))
1669 VISIT_QUIT(st, 0);
1670 VISIT(st, arguments, s->v.FunctionDef.args);
1671 VISIT_SEQ(st, stmt, s->v.FunctionDef.body);
1672 if (!symtable_exit_block(st))
1673 VISIT_QUIT(st, 0);
1674 if (asdl_seq_LEN(s->v.FunctionDef.type_params) > 0) {
1675 if (!symtable_exit_block(st))
1676 VISIT_QUIT(st, 0);
1677 }
1678 break;
1679 case ClassDef_kind: {
1680 PyObject *tmp;
1681 if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s)))
1682 VISIT_QUIT(st, 0);
1683 if (s->v.ClassDef.decorator_list)
1684 VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
1685 tmp = st->st_private;
1686 if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
1687 if (!symtable_enter_type_param_block(st, s->v.ClassDef.name,
1688 (void *)s->v.ClassDef.type_params,
1689 false, false, s->kind,
1690 LOCATION(s))) {
1691 VISIT_QUIT(st, 0);
1692 }
1693 st->st_private = s->v.ClassDef.name;
1694 st->st_cur->ste_mangled_names = PySet_New(NULL);
1695 if (!st->st_cur->ste_mangled_names) {
1696 VISIT_QUIT(st, 0);
1697 }
1698 VISIT_SEQ(st, type_param, s->v.ClassDef.type_params);
1699 }
1700 VISIT_SEQ(st, expr, s->v.ClassDef.bases);
1701 VISIT_SEQ(st, keyword, s->v.ClassDef.keywords);
1702 if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock,
1703 (void *)s, s->lineno, s->col_offset,
1704 s->end_lineno, s->end_col_offset))
1705 VISIT_QUIT(st, 0);
1706 st->st_private = s->v.ClassDef.name;
1707 if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
1708 if (!symtable_add_def(st, &_Py_ID(__type_params__),
1709 DEF_LOCAL, LOCATION(s))) {
1710 VISIT_QUIT(st, 0);
1711 }
1712 _Py_DECLARE_STR(type_params, ".type_params");
1713 if (!symtable_add_def(st, &_Py_STR(type_params),
1714 USE, LOCATION(s))) {
1715 VISIT_QUIT(st, 0);
1716 }
1717 }
1718 VISIT_SEQ(st, stmt, s->v.ClassDef.body);
1719 if (!symtable_exit_block(st))
1720 VISIT_QUIT(st, 0);
1721 if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
1722 if (!symtable_exit_block(st))
1723 VISIT_QUIT(st, 0);
1724 }
1725 st->st_private = tmp;
1726 break;
1727 }
1728 case TypeAlias_kind: {
1729 VISIT(st, expr, s->v.TypeAlias.name);
1730 assert(s->v.TypeAlias.name->kind == Name_kind);
1731 PyObject *name = s->v.TypeAlias.name->v.Name.id;
1732 int is_in_class = st->st_cur->ste_type == ClassBlock;
1733 int is_generic = asdl_seq_LEN(s->v.TypeAlias.type_params) > 0;
1734 if (is_generic) {
1735 if (!symtable_enter_type_param_block(
1736 st, name,
1737 (void *)s->v.TypeAlias.type_params,
1738 false, false, s->kind,
1739 LOCATION(s))) {
1740 VISIT_QUIT(st, 0);
1741 }
1742 VISIT_SEQ(st, type_param, s->v.TypeAlias.type_params);
1743 }
1744 if (!symtable_enter_block(st, name, TypeAliasBlock,
1745 (void *)s, LOCATION(s)))
1746 VISIT_QUIT(st, 0);
1747 st->st_cur->ste_can_see_class_scope = is_in_class;
1748 if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(s->v.TypeAlias.value))) {
1749 VISIT_QUIT(st, 0);
1750 }
1751 VISIT(st, expr, s->v.TypeAlias.value);
1752 if (!symtable_exit_block(st))
1753 VISIT_QUIT(st, 0);
1754 if (is_generic) {
1755 if (!symtable_exit_block(st))
1756 VISIT_QUIT(st, 0);
1757 }
1758 break;
1759 }
1760 case Return_kind:
1761 if (s->v.Return.value) {
1762 VISIT(st, expr, s->v.Return.value);
1763 st->st_cur->ste_returns_value = 1;
1764 }
1765 break;
1766 case Delete_kind:
1767 VISIT_SEQ(st, expr, s->v.Delete.targets);
1768 break;
1769 case Assign_kind:
1770 VISIT_SEQ(st, expr, s->v.Assign.targets);
1771 VISIT(st, expr, s->v.Assign.value);
1772 break;
1773 case AnnAssign_kind:
1774 if (s->v.AnnAssign.target->kind == Name_kind) {
1775 expr_ty e_name = s->v.AnnAssign.target;
1776 long cur = symtable_lookup(st, e_name->v.Name.id);
1777 if (cur < 0) {
1778 VISIT_QUIT(st, 0);
1779 }
1780 if ((cur & (DEF_GLOBAL | DEF_NONLOCAL))
1781 && (st->st_cur->ste_symbols != st->st_global)
1782 && s->v.AnnAssign.simple) {
1783 PyErr_Format(PyExc_SyntaxError,
1784 cur & DEF_GLOBAL ? GLOBAL_ANNOT : NONLOCAL_ANNOT,
1785 e_name->v.Name.id);
1786 PyErr_RangedSyntaxLocationObject(st->st_filename,
1787 s->lineno,
1788 s->col_offset + 1,
1789 s->end_lineno,
1790 s->end_col_offset + 1);
1791 VISIT_QUIT(st, 0);
1792 }
1793 if (s->v.AnnAssign.simple &&
1794 !symtable_add_def(st, e_name->v.Name.id,
1795 DEF_ANNOT | DEF_LOCAL, LOCATION(e_name))) {
1796 VISIT_QUIT(st, 0);
1797 }
1798 else {
1799 if (s->v.AnnAssign.value
1800 && !symtable_add_def(st, e_name->v.Name.id, DEF_LOCAL, LOCATION(e_name))) {
1801 VISIT_QUIT(st, 0);
1802 }
1803 }
1804 }
1805 else {
1806 VISIT(st, expr, s->v.AnnAssign.target);
1807 }
1808 if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) {
1809 VISIT_QUIT(st, 0);
1810 }
1811
1812 if (s->v.AnnAssign.value) {
1813 VISIT(st, expr, s->v.AnnAssign.value);
1814 }
1815 break;
1816 case AugAssign_kind:
1817 VISIT(st, expr, s->v.AugAssign.target);
1818 VISIT(st, expr, s->v.AugAssign.value);
1819 break;
1820 case For_kind:
1821 VISIT(st, expr, s->v.For.target);
1822 VISIT(st, expr, s->v.For.iter);
1823 VISIT_SEQ(st, stmt, s->v.For.body);
1824 if (s->v.For.orelse)
1825 VISIT_SEQ(st, stmt, s->v.For.orelse);
1826 break;
1827 case While_kind:
1828 VISIT(st, expr, s->v.While.test);
1829 VISIT_SEQ(st, stmt, s->v.While.body);
1830 if (s->v.While.orelse)
1831 VISIT_SEQ(st, stmt, s->v.While.orelse);
1832 break;
1833 case If_kind:
1834 /* XXX if 0: and lookup_yield() hacks */
1835 VISIT(st, expr, s->v.If.test);
1836 VISIT_SEQ(st, stmt, s->v.If.body);
1837 if (s->v.If.orelse)
1838 VISIT_SEQ(st, stmt, s->v.If.orelse);
1839 break;
1840 case Match_kind:
1841 VISIT(st, expr, s->v.Match.subject);
1842 VISIT_SEQ(st, match_case, s->v.Match.cases);
1843 break;
1844 case Raise_kind:
1845 if (s->v.Raise.exc) {
1846 VISIT(st, expr, s->v.Raise.exc);
1847 if (s->v.Raise.cause) {
1848 VISIT(st, expr, s->v.Raise.cause);
1849 }
1850 }
1851 break;
1852 case Try_kind:
1853 VISIT_SEQ(st, stmt, s->v.Try.body);
1854 VISIT_SEQ(st, excepthandler, s->v.Try.handlers);
1855 VISIT_SEQ(st, stmt, s->v.Try.orelse);
1856 VISIT_SEQ(st, stmt, s->v.Try.finalbody);
1857 break;
1858 case TryStar_kind:
1859 VISIT_SEQ(st, stmt, s->v.TryStar.body);
1860 VISIT_SEQ(st, excepthandler, s->v.TryStar.handlers);
1861 VISIT_SEQ(st, stmt, s->v.TryStar.orelse);
1862 VISIT_SEQ(st, stmt, s->v.TryStar.finalbody);
1863 break;
1864 case Assert_kind:
1865 VISIT(st, expr, s->v.Assert.test);
1866 if (s->v.Assert.msg)
1867 VISIT(st, expr, s->v.Assert.msg);
1868 break;
1869 case Import_kind:
1870 VISIT_SEQ(st, alias, s->v.Import.names);
1871 break;
1872 case ImportFrom_kind:
1873 VISIT_SEQ(st, alias, s->v.ImportFrom.names);
1874 break;
1875 case Global_kind: {
1876 int i;
1877 asdl_identifier_seq *seq = s->v.Global.names;
1878 for (i = 0; i < asdl_seq_LEN(seq); i++) {
1879 identifier name = (identifier)asdl_seq_GET(seq, i);
1880 long cur = symtable_lookup(st, name);
1881 if (cur < 0)
1882 VISIT_QUIT(st, 0);
1883 if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) {
1884 const char* msg;
1885 if (cur & DEF_PARAM) {
1886 msg = GLOBAL_PARAM;
1887 } else if (cur & USE) {
1888 msg = GLOBAL_AFTER_USE;
1889 } else if (cur & DEF_ANNOT) {
1890 msg = GLOBAL_ANNOT;
1891 } else { /* DEF_LOCAL */
1892 msg = GLOBAL_AFTER_ASSIGN;
1893 }
1894 PyErr_Format(PyExc_SyntaxError,
1895 msg, name);
1896 PyErr_RangedSyntaxLocationObject(st->st_filename,
1897 s->lineno,
1898 s->col_offset + 1,
1899 s->end_lineno,
1900 s->end_col_offset + 1);
1901 VISIT_QUIT(st, 0);
1902 }
1903 if (!symtable_add_def(st, name, DEF_GLOBAL, LOCATION(s)))
1904 VISIT_QUIT(st, 0);
1905 if (!symtable_record_directive(st, name, s->lineno, s->col_offset,
1906 s->end_lineno, s->end_col_offset))
1907 VISIT_QUIT(st, 0);
1908 }
1909 break;
1910 }
1911 case Nonlocal_kind: {
1912 int i;
1913 asdl_identifier_seq *seq = s->v.Nonlocal.names;
1914 for (i = 0; i < asdl_seq_LEN(seq); i++) {
1915 identifier name = (identifier)asdl_seq_GET(seq, i);
1916 long cur = symtable_lookup(st, name);
1917 if (cur < 0)
1918 VISIT_QUIT(st, 0);
1919 if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) {
1920 const char* msg;
1921 if (cur & DEF_PARAM) {
1922 msg = NONLOCAL_PARAM;
1923 } else if (cur & USE) {
1924 msg = NONLOCAL_AFTER_USE;
1925 } else if (cur & DEF_ANNOT) {
1926 msg = NONLOCAL_ANNOT;
1927 } else { /* DEF_LOCAL */
1928 msg = NONLOCAL_AFTER_ASSIGN;
1929 }
1930 PyErr_Format(PyExc_SyntaxError, msg, name);
1931 PyErr_RangedSyntaxLocationObject(st->st_filename,
1932 s->lineno,
1933 s->col_offset + 1,
1934 s->end_lineno,
1935 s->end_col_offset + 1);
1936 VISIT_QUIT(st, 0);
1937 }
1938 if (!symtable_add_def(st, name, DEF_NONLOCAL, LOCATION(s)))
1939 VISIT_QUIT(st, 0);
1940 if (!symtable_record_directive(st, name, s->lineno, s->col_offset,
1941 s->end_lineno, s->end_col_offset))
1942 VISIT_QUIT(st, 0);
1943 }
1944 break;
1945 }
1946 case Expr_kind:
1947 VISIT(st, expr, s->v.Expr.value);
1948 break;
1949 case Pass_kind:
1950 case Break_kind:
1951 case Continue_kind:
1952 /* nothing to do here */
1953 break;
1954 case With_kind:
1955 VISIT_SEQ(st, withitem, s->v.With.items);
1956 VISIT_SEQ(st, stmt, s->v.With.body);
1957 break;
1958 case AsyncFunctionDef_kind:
1959 if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s)))
1960 VISIT_QUIT(st, 0);
1961 if (s->v.AsyncFunctionDef.args->defaults)
1962 VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults);
1963 if (s->v.AsyncFunctionDef.args->kw_defaults)
1964 VISIT_SEQ_WITH_NULL(st, expr,
1965 s->v.AsyncFunctionDef.args->kw_defaults);
1966 if (s->v.AsyncFunctionDef.decorator_list)
1967 VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list);
1968 if (asdl_seq_LEN(s->v.AsyncFunctionDef.type_params) > 0) {
1969 if (!symtable_enter_type_param_block(
1970 st, s->v.AsyncFunctionDef.name,
1971 (void *)s->v.AsyncFunctionDef.type_params,
1972 s->v.AsyncFunctionDef.args->defaults != NULL,
1973 has_kwonlydefaults(s->v.AsyncFunctionDef.args->kwonlyargs,
1974 s->v.AsyncFunctionDef.args->kw_defaults),
1975 s->kind,
1976 LOCATION(s))) {
1977 VISIT_QUIT(st, 0);
1978 }
1979 VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params);
1980 }
1981 if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
1982 s->v.AsyncFunctionDef.returns))
1983 VISIT_QUIT(st, 0);
1984 if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name,
1985 FunctionBlock, (void *)s,
1986 s->lineno, s->col_offset,
1987 s->end_lineno, s->end_col_offset))
1988 VISIT_QUIT(st, 0);
1989 st->st_cur->ste_coroutine = 1;
1990 VISIT(st, arguments, s->v.AsyncFunctionDef.args);
1991 VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
1992 if (!symtable_exit_block(st))
1993 VISIT_QUIT(st, 0);
1994 if (asdl_seq_LEN(s->v.AsyncFunctionDef.type_params) > 0) {
1995 if (!symtable_exit_block(st))
1996 VISIT_QUIT(st, 0);
1997 }
1998 break;
1999 case AsyncWith_kind:
2000 VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
2001 VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
2002 break;
2003 case AsyncFor_kind:
2004 VISIT(st, expr, s->v.AsyncFor.target);
2005 VISIT(st, expr, s->v.AsyncFor.iter);
2006 VISIT_SEQ(st, stmt, s->v.AsyncFor.body);
2007 if (s->v.AsyncFor.orelse)
2008 VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse);
2009 break;
2010 }
2011 VISIT_QUIT(st, 1);
2012 }
2013
2014 static int
symtable_extend_namedexpr_scope(struct symtable * st,expr_ty e)2015 symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e)
2016 {
2017 assert(st->st_stack);
2018 assert(e->kind == Name_kind);
2019
2020 PyObject *target_name = e->v.Name.id;
2021 Py_ssize_t i, size;
2022 struct _symtable_entry *ste;
2023 size = PyList_GET_SIZE(st->st_stack);
2024 assert(size);
2025
2026 /* Iterate over the stack in reverse and add to the nearest adequate scope */
2027 for (i = size - 1; i >= 0; i--) {
2028 ste = (struct _symtable_entry *) PyList_GET_ITEM(st->st_stack, i);
2029
2030 /* If we find a comprehension scope, check for a target
2031 * binding conflict with iteration variables, otherwise skip it
2032 */
2033 if (ste->ste_comprehension) {
2034 long target_in_scope = symtable_lookup_entry(st, ste, target_name);
2035 if ((target_in_scope & DEF_COMP_ITER) &&
2036 (target_in_scope & DEF_LOCAL)) {
2037 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name);
2038 PyErr_RangedSyntaxLocationObject(st->st_filename,
2039 e->lineno,
2040 e->col_offset + 1,
2041 e->end_lineno,
2042 e->end_col_offset + 1);
2043 VISIT_QUIT(st, 0);
2044 }
2045 continue;
2046 }
2047
2048 /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */
2049 if (ste->ste_type == FunctionBlock) {
2050 long target_in_scope = symtable_lookup_entry(st, ste, target_name);
2051 if (target_in_scope & DEF_GLOBAL) {
2052 if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e)))
2053 VISIT_QUIT(st, 0);
2054 } else {
2055 if (!symtable_add_def(st, target_name, DEF_NONLOCAL, LOCATION(e)))
2056 VISIT_QUIT(st, 0);
2057 }
2058 if (!symtable_record_directive(st, target_name, LOCATION(e)))
2059 VISIT_QUIT(st, 0);
2060
2061 return symtable_add_def_helper(st, target_name, DEF_LOCAL, ste, LOCATION(e));
2062 }
2063 /* If we find a ModuleBlock entry, add as GLOBAL */
2064 if (ste->ste_type == ModuleBlock) {
2065 if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e)))
2066 VISIT_QUIT(st, 0);
2067 if (!symtable_record_directive(st, target_name, LOCATION(e)))
2068 VISIT_QUIT(st, 0);
2069
2070 return symtable_add_def_helper(st, target_name, DEF_GLOBAL, ste, LOCATION(e));
2071 }
2072 /* Disallow usage in ClassBlock and type scopes */
2073 if (ste->ste_type == ClassBlock ||
2074 ste->ste_type == TypeParametersBlock ||
2075 ste->ste_type == TypeAliasBlock ||
2076 ste->ste_type == TypeVariableBlock) {
2077 switch (ste->ste_type) {
2078 case ClassBlock:
2079 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS);
2080 break;
2081 case TypeParametersBlock:
2082 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEPARAM);
2083 break;
2084 case TypeAliasBlock:
2085 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEALIAS);
2086 break;
2087 case TypeVariableBlock:
2088 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_TYPEVAR_BOUND);
2089 break;
2090 default:
2091 Py_UNREACHABLE();
2092 }
2093 PyErr_RangedSyntaxLocationObject(st->st_filename,
2094 e->lineno,
2095 e->col_offset + 1,
2096 e->end_lineno,
2097 e->end_col_offset + 1);
2098 VISIT_QUIT(st, 0);
2099 }
2100 }
2101
2102 /* We should always find either a function-like block, ModuleBlock or ClassBlock
2103 and should never fall to this case
2104 */
2105 Py_UNREACHABLE();
2106 return 0;
2107 }
2108
2109 static int
symtable_handle_namedexpr(struct symtable * st,expr_ty e)2110 symtable_handle_namedexpr(struct symtable *st, expr_ty e)
2111 {
2112 if (st->st_cur->ste_comp_iter_expr > 0) {
2113 /* Assignment isn't allowed in a comprehension iterable expression */
2114 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR);
2115 PyErr_RangedSyntaxLocationObject(st->st_filename,
2116 e->lineno,
2117 e->col_offset + 1,
2118 e->end_lineno,
2119 e->end_col_offset + 1);
2120 return 0;
2121 }
2122 if (st->st_cur->ste_comprehension) {
2123 /* Inside a comprehension body, so find the right target scope */
2124 if (!symtable_extend_namedexpr_scope(st, e->v.NamedExpr.target))
2125 return 0;
2126 }
2127 VISIT(st, expr, e->v.NamedExpr.value);
2128 VISIT(st, expr, e->v.NamedExpr.target);
2129 return 1;
2130 }
2131
2132 static int
symtable_visit_expr(struct symtable * st,expr_ty e)2133 symtable_visit_expr(struct symtable *st, expr_ty e)
2134 {
2135 if (++st->recursion_depth > st->recursion_limit) {
2136 PyErr_SetString(PyExc_RecursionError,
2137 "maximum recursion depth exceeded during compilation");
2138 VISIT_QUIT(st, 0);
2139 }
2140 switch (e->kind) {
2141 case NamedExpr_kind:
2142 if (!symtable_raise_if_annotation_block(st, "named expression", e)) {
2143 VISIT_QUIT(st, 0);
2144 }
2145 if(!symtable_handle_namedexpr(st, e))
2146 VISIT_QUIT(st, 0);
2147 break;
2148 case BoolOp_kind:
2149 VISIT_SEQ(st, expr, e->v.BoolOp.values);
2150 break;
2151 case BinOp_kind:
2152 VISIT(st, expr, e->v.BinOp.left);
2153 VISIT(st, expr, e->v.BinOp.right);
2154 break;
2155 case UnaryOp_kind:
2156 VISIT(st, expr, e->v.UnaryOp.operand);
2157 break;
2158 case Lambda_kind: {
2159 if (e->v.Lambda.args->defaults)
2160 VISIT_SEQ(st, expr, e->v.Lambda.args->defaults);
2161 if (e->v.Lambda.args->kw_defaults)
2162 VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults);
2163 if (!symtable_enter_block(st, &_Py_ID(lambda),
2164 FunctionBlock, (void *)e,
2165 e->lineno, e->col_offset,
2166 e->end_lineno, e->end_col_offset))
2167 VISIT_QUIT(st, 0);
2168 VISIT(st, arguments, e->v.Lambda.args);
2169 VISIT(st, expr, e->v.Lambda.body);
2170 if (!symtable_exit_block(st))
2171 VISIT_QUIT(st, 0);
2172 break;
2173 }
2174 case IfExp_kind:
2175 VISIT(st, expr, e->v.IfExp.test);
2176 VISIT(st, expr, e->v.IfExp.body);
2177 VISIT(st, expr, e->v.IfExp.orelse);
2178 break;
2179 case Dict_kind:
2180 VISIT_SEQ_WITH_NULL(st, expr, e->v.Dict.keys);
2181 VISIT_SEQ(st, expr, e->v.Dict.values);
2182 break;
2183 case Set_kind:
2184 VISIT_SEQ(st, expr, e->v.Set.elts);
2185 break;
2186 case GeneratorExp_kind:
2187 if (!symtable_visit_genexp(st, e))
2188 VISIT_QUIT(st, 0);
2189 break;
2190 case ListComp_kind:
2191 if (!symtable_visit_listcomp(st, e))
2192 VISIT_QUIT(st, 0);
2193 break;
2194 case SetComp_kind:
2195 if (!symtable_visit_setcomp(st, e))
2196 VISIT_QUIT(st, 0);
2197 break;
2198 case DictComp_kind:
2199 if (!symtable_visit_dictcomp(st, e))
2200 VISIT_QUIT(st, 0);
2201 break;
2202 case Yield_kind:
2203 if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
2204 VISIT_QUIT(st, 0);
2205 }
2206 if (e->v.Yield.value)
2207 VISIT(st, expr, e->v.Yield.value);
2208 st->st_cur->ste_generator = 1;
2209 if (st->st_cur->ste_comprehension) {
2210 return symtable_raise_if_comprehension_block(st, e);
2211 }
2212 break;
2213 case YieldFrom_kind:
2214 if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
2215 VISIT_QUIT(st, 0);
2216 }
2217 VISIT(st, expr, e->v.YieldFrom.value);
2218 st->st_cur->ste_generator = 1;
2219 if (st->st_cur->ste_comprehension) {
2220 return symtable_raise_if_comprehension_block(st, e);
2221 }
2222 break;
2223 case Await_kind:
2224 if (!symtable_raise_if_annotation_block(st, "await expression", e)) {
2225 VISIT_QUIT(st, 0);
2226 }
2227 VISIT(st, expr, e->v.Await.value);
2228 st->st_cur->ste_coroutine = 1;
2229 break;
2230 case Compare_kind:
2231 VISIT(st, expr, e->v.Compare.left);
2232 VISIT_SEQ(st, expr, e->v.Compare.comparators);
2233 break;
2234 case Call_kind:
2235 VISIT(st, expr, e->v.Call.func);
2236 VISIT_SEQ(st, expr, e->v.Call.args);
2237 VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords);
2238 break;
2239 case FormattedValue_kind:
2240 VISIT(st, expr, e->v.FormattedValue.value);
2241 if (e->v.FormattedValue.format_spec)
2242 VISIT(st, expr, e->v.FormattedValue.format_spec);
2243 break;
2244 case JoinedStr_kind:
2245 VISIT_SEQ(st, expr, e->v.JoinedStr.values);
2246 break;
2247 case Constant_kind:
2248 /* Nothing to do here. */
2249 break;
2250 /* The following exprs can be assignment targets. */
2251 case Attribute_kind:
2252 VISIT(st, expr, e->v.Attribute.value);
2253 break;
2254 case Subscript_kind:
2255 VISIT(st, expr, e->v.Subscript.value);
2256 VISIT(st, expr, e->v.Subscript.slice);
2257 break;
2258 case Starred_kind:
2259 VISIT(st, expr, e->v.Starred.value);
2260 break;
2261 case Slice_kind:
2262 if (e->v.Slice.lower)
2263 VISIT(st, expr, e->v.Slice.lower)
2264 if (e->v.Slice.upper)
2265 VISIT(st, expr, e->v.Slice.upper)
2266 if (e->v.Slice.step)
2267 VISIT(st, expr, e->v.Slice.step)
2268 break;
2269 case Name_kind:
2270 if (!symtable_add_def(st, e->v.Name.id,
2271 e->v.Name.ctx == Load ? USE : DEF_LOCAL, LOCATION(e)))
2272 VISIT_QUIT(st, 0);
2273 /* Special-case super: it counts as a use of __class__ */
2274 if (e->v.Name.ctx == Load &&
2275 _PyST_IsFunctionLike(st->st_cur) &&
2276 _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) {
2277 if (!symtable_add_def(st, &_Py_ID(__class__), USE, LOCATION(e)))
2278 VISIT_QUIT(st, 0);
2279 }
2280 break;
2281 /* child nodes of List and Tuple will have expr_context set */
2282 case List_kind:
2283 VISIT_SEQ(st, expr, e->v.List.elts);
2284 break;
2285 case Tuple_kind:
2286 VISIT_SEQ(st, expr, e->v.Tuple.elts);
2287 break;
2288 }
2289 VISIT_QUIT(st, 1);
2290 }
2291
2292 static int
symtable_visit_type_param_bound_or_default(struct symtable * st,expr_ty e,identifier name,void * key,const char * ste_scope_info)2293 symtable_visit_type_param_bound_or_default(
2294 struct symtable *st, expr_ty e, identifier name,
2295 void *key, const char *ste_scope_info)
2296 {
2297 if (e) {
2298 int is_in_class = st->st_cur->ste_can_see_class_scope;
2299 if (!symtable_enter_block(st, name, TypeVariableBlock, key, LOCATION(e)))
2300 return 0;
2301
2302 st->st_cur->ste_can_see_class_scope = is_in_class;
2303 if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(e))) {
2304 VISIT_QUIT(st, 0);
2305 }
2306
2307 assert(ste_scope_info != NULL);
2308 st->st_cur->ste_scope_info = ste_scope_info;
2309 VISIT(st, expr, e);
2310
2311 if (!symtable_exit_block(st)) {
2312 return 0;
2313 }
2314 }
2315 return 1;
2316 }
2317
2318 static int
symtable_visit_type_param(struct symtable * st,type_param_ty tp)2319 symtable_visit_type_param(struct symtable *st, type_param_ty tp)
2320 {
2321 if (++st->recursion_depth > st->recursion_limit) {
2322 PyErr_SetString(PyExc_RecursionError,
2323 "maximum recursion depth exceeded during compilation");
2324 VISIT_QUIT(st, 0);
2325 }
2326 switch(tp->kind) {
2327 case TypeVar_kind:
2328 if (!symtable_add_def(st, tp->v.TypeVar.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp)))
2329 VISIT_QUIT(st, 0);
2330
2331 const char *ste_scope_info = NULL;
2332 const expr_ty bound = tp->v.TypeVar.bound;
2333 if (bound != NULL) {
2334 ste_scope_info = bound->kind == Tuple_kind ? "a TypeVar constraint" : "a TypeVar bound";
2335 }
2336
2337 // We must use a different key for the bound and default. The obvious choice would be to
2338 // use the .bound and .default_value pointers, but that fails when the expression immediately
2339 // inside the bound or default is a comprehension: we would reuse the same key for
2340 // the comprehension scope. Therefore, use the address + 1 as the second key.
2341 // The only requirement for the key is that it is unique and it matches the logic in
2342 // compile.c where the scope is retrieved.
2343 if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.bound, tp->v.TypeVar.name,
2344 (void *)tp, ste_scope_info)) {
2345 VISIT_QUIT(st, 0);
2346 }
2347
2348 if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.default_value, tp->v.TypeVar.name,
2349 (void *)((uintptr_t)tp + 1), "a TypeVar default")) {
2350 VISIT_QUIT(st, 0);
2351 }
2352 break;
2353 case TypeVarTuple_kind:
2354 if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) {
2355 VISIT_QUIT(st, 0);
2356 }
2357
2358 if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVarTuple.default_value, tp->v.TypeVarTuple.name,
2359 (void *)tp, "a TypeVarTuple default")) {
2360 VISIT_QUIT(st, 0);
2361 }
2362 break;
2363 case ParamSpec_kind:
2364 if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) {
2365 VISIT_QUIT(st, 0);
2366 }
2367
2368 if (!symtable_visit_type_param_bound_or_default(st, tp->v.ParamSpec.default_value, tp->v.ParamSpec.name,
2369 (void *)tp, "a ParamSpec default")) {
2370 VISIT_QUIT(st, 0);
2371 }
2372 break;
2373 }
2374 VISIT_QUIT(st, 1);
2375 }
2376
2377 static int
symtable_visit_pattern(struct symtable * st,pattern_ty p)2378 symtable_visit_pattern(struct symtable *st, pattern_ty p)
2379 {
2380 if (++st->recursion_depth > st->recursion_limit) {
2381 PyErr_SetString(PyExc_RecursionError,
2382 "maximum recursion depth exceeded during compilation");
2383 VISIT_QUIT(st, 0);
2384 }
2385 switch (p->kind) {
2386 case MatchValue_kind:
2387 VISIT(st, expr, p->v.MatchValue.value);
2388 break;
2389 case MatchSingleton_kind:
2390 /* Nothing to do here. */
2391 break;
2392 case MatchSequence_kind:
2393 VISIT_SEQ(st, pattern, p->v.MatchSequence.patterns);
2394 break;
2395 case MatchStar_kind:
2396 if (p->v.MatchStar.name) {
2397 symtable_add_def(st, p->v.MatchStar.name, DEF_LOCAL, LOCATION(p));
2398 }
2399 break;
2400 case MatchMapping_kind:
2401 VISIT_SEQ(st, expr, p->v.MatchMapping.keys);
2402 VISIT_SEQ(st, pattern, p->v.MatchMapping.patterns);
2403 if (p->v.MatchMapping.rest) {
2404 symtable_add_def(st, p->v.MatchMapping.rest, DEF_LOCAL, LOCATION(p));
2405 }
2406 break;
2407 case MatchClass_kind:
2408 VISIT(st, expr, p->v.MatchClass.cls);
2409 VISIT_SEQ(st, pattern, p->v.MatchClass.patterns);
2410 VISIT_SEQ(st, pattern, p->v.MatchClass.kwd_patterns);
2411 break;
2412 case MatchAs_kind:
2413 if (p->v.MatchAs.pattern) {
2414 VISIT(st, pattern, p->v.MatchAs.pattern);
2415 }
2416 if (p->v.MatchAs.name) {
2417 symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL, LOCATION(p));
2418 }
2419 break;
2420 case MatchOr_kind:
2421 VISIT_SEQ(st, pattern, p->v.MatchOr.patterns);
2422 break;
2423 }
2424 VISIT_QUIT(st, 1);
2425 }
2426
2427 static int
symtable_implicit_arg(struct symtable * st,int pos)2428 symtable_implicit_arg(struct symtable *st, int pos)
2429 {
2430 PyObject *id = PyUnicode_FromFormat(".%d", pos);
2431 if (id == NULL)
2432 return 0;
2433 if (!symtable_add_def(st, id, DEF_PARAM, ST_LOCATION(st->st_cur))) {
2434 Py_DECREF(id);
2435 return 0;
2436 }
2437 Py_DECREF(id);
2438 return 1;
2439 }
2440
2441 static int
symtable_visit_params(struct symtable * st,asdl_arg_seq * args)2442 symtable_visit_params(struct symtable *st, asdl_arg_seq *args)
2443 {
2444 int i;
2445
2446 if (!args)
2447 return -1;
2448
2449 for (i = 0; i < asdl_seq_LEN(args); i++) {
2450 arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
2451 if (!symtable_add_def(st, arg->arg, DEF_PARAM, LOCATION(arg)))
2452 return 0;
2453 }
2454
2455 return 1;
2456 }
2457
2458 static int
symtable_visit_annotation(struct symtable * st,expr_ty annotation)2459 symtable_visit_annotation(struct symtable *st, expr_ty annotation)
2460 {
2461 int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
2462 if (future_annotations &&
2463 !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
2464 (void *)annotation, annotation->lineno,
2465 annotation->col_offset, annotation->end_lineno,
2466 annotation->end_col_offset)) {
2467 VISIT_QUIT(st, 0);
2468 }
2469 VISIT(st, expr, annotation);
2470 if (future_annotations && !symtable_exit_block(st)) {
2471 VISIT_QUIT(st, 0);
2472 }
2473 return 1;
2474 }
2475
2476 static int
symtable_visit_argannotations(struct symtable * st,asdl_arg_seq * args)2477 symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
2478 {
2479 int i;
2480
2481 if (!args)
2482 return -1;
2483
2484 for (i = 0; i < asdl_seq_LEN(args); i++) {
2485 arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
2486 if (arg->annotation)
2487 VISIT(st, expr, arg->annotation);
2488 }
2489
2490 return 1;
2491 }
2492
2493 static int
symtable_visit_annotations(struct symtable * st,stmt_ty o,arguments_ty a,expr_ty returns)2494 symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns)
2495 {
2496 int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
2497 if (future_annotations &&
2498 !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
2499 (void *)o, o->lineno, o->col_offset, o->end_lineno,
2500 o->end_col_offset)) {
2501 VISIT_QUIT(st, 0);
2502 }
2503 if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs))
2504 return 0;
2505 if (a->args && !symtable_visit_argannotations(st, a->args))
2506 return 0;
2507 if (a->vararg && a->vararg->annotation)
2508 VISIT(st, expr, a->vararg->annotation);
2509 if (a->kwarg && a->kwarg->annotation)
2510 VISIT(st, expr, a->kwarg->annotation);
2511 if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
2512 return 0;
2513 if (future_annotations && !symtable_exit_block(st)) {
2514 VISIT_QUIT(st, 0);
2515 }
2516 if (returns && !symtable_visit_annotation(st, returns)) {
2517 VISIT_QUIT(st, 0);
2518 }
2519 return 1;
2520 }
2521
2522 static int
symtable_visit_arguments(struct symtable * st,arguments_ty a)2523 symtable_visit_arguments(struct symtable *st, arguments_ty a)
2524 {
2525 /* skip default arguments inside function block
2526 XXX should ast be different?
2527 */
2528 if (a->posonlyargs && !symtable_visit_params(st, a->posonlyargs))
2529 return 0;
2530 if (a->args && !symtable_visit_params(st, a->args))
2531 return 0;
2532 if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs))
2533 return 0;
2534 if (a->vararg) {
2535 if (!symtable_add_def(st, a->vararg->arg, DEF_PARAM, LOCATION(a->vararg)))
2536 return 0;
2537 st->st_cur->ste_varargs = 1;
2538 }
2539 if (a->kwarg) {
2540 if (!symtable_add_def(st, a->kwarg->arg, DEF_PARAM, LOCATION(a->kwarg)))
2541 return 0;
2542 st->st_cur->ste_varkeywords = 1;
2543 }
2544 return 1;
2545 }
2546
2547
2548 static int
symtable_visit_excepthandler(struct symtable * st,excepthandler_ty eh)2549 symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh)
2550 {
2551 if (eh->v.ExceptHandler.type)
2552 VISIT(st, expr, eh->v.ExceptHandler.type);
2553 if (eh->v.ExceptHandler.name)
2554 if (!symtable_add_def(st, eh->v.ExceptHandler.name, DEF_LOCAL, LOCATION(eh)))
2555 return 0;
2556 VISIT_SEQ(st, stmt, eh->v.ExceptHandler.body);
2557 return 1;
2558 }
2559
2560 static int
symtable_visit_withitem(struct symtable * st,withitem_ty item)2561 symtable_visit_withitem(struct symtable *st, withitem_ty item)
2562 {
2563 VISIT(st, expr, item->context_expr);
2564 if (item->optional_vars) {
2565 VISIT(st, expr, item->optional_vars);
2566 }
2567 return 1;
2568 }
2569
2570 static int
symtable_visit_match_case(struct symtable * st,match_case_ty m)2571 symtable_visit_match_case(struct symtable *st, match_case_ty m)
2572 {
2573 VISIT(st, pattern, m->pattern);
2574 if (m->guard) {
2575 VISIT(st, expr, m->guard);
2576 }
2577 VISIT_SEQ(st, stmt, m->body);
2578 return 1;
2579 }
2580
2581 static int
symtable_visit_alias(struct symtable * st,alias_ty a)2582 symtable_visit_alias(struct symtable *st, alias_ty a)
2583 {
2584 /* Compute store_name, the name actually bound by the import
2585 operation. It is different than a->name when a->name is a
2586 dotted package name (e.g. spam.eggs)
2587 */
2588 PyObject *store_name;
2589 PyObject *name = (a->asname == NULL) ? a->name : a->asname;
2590 Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0,
2591 PyUnicode_GET_LENGTH(name), 1);
2592 if (dot != -1) {
2593 store_name = PyUnicode_Substring(name, 0, dot);
2594 if (!store_name)
2595 return 0;
2596 }
2597 else {
2598 store_name = Py_NewRef(name);
2599 }
2600 if (!_PyUnicode_EqualToASCIIString(name, "*")) {
2601 int r = symtable_add_def(st, store_name, DEF_IMPORT, LOCATION(a));
2602 Py_DECREF(store_name);
2603 return r;
2604 }
2605 else {
2606 if (st->st_cur->ste_type != ModuleBlock) {
2607 int lineno = a->lineno;
2608 int col_offset = a->col_offset;
2609 int end_lineno = a->end_lineno;
2610 int end_col_offset = a->end_col_offset;
2611 PyErr_SetString(PyExc_SyntaxError, IMPORT_STAR_WARNING);
2612 PyErr_RangedSyntaxLocationObject(st->st_filename,
2613 lineno, col_offset + 1,
2614 end_lineno, end_col_offset + 1);
2615 Py_DECREF(store_name);
2616 return 0;
2617 }
2618 Py_DECREF(store_name);
2619 return 1;
2620 }
2621 }
2622
2623
2624 static int
symtable_visit_comprehension(struct symtable * st,comprehension_ty lc)2625 symtable_visit_comprehension(struct symtable *st, comprehension_ty lc)
2626 {
2627 st->st_cur->ste_comp_iter_target = 1;
2628 VISIT(st, expr, lc->target);
2629 st->st_cur->ste_comp_iter_target = 0;
2630 st->st_cur->ste_comp_iter_expr++;
2631 VISIT(st, expr, lc->iter);
2632 st->st_cur->ste_comp_iter_expr--;
2633 VISIT_SEQ(st, expr, lc->ifs);
2634 if (lc->is_async) {
2635 st->st_cur->ste_coroutine = 1;
2636 }
2637 return 1;
2638 }
2639
2640
2641 static int
symtable_visit_keyword(struct symtable * st,keyword_ty k)2642 symtable_visit_keyword(struct symtable *st, keyword_ty k)
2643 {
2644 VISIT(st, expr, k->value);
2645 return 1;
2646 }
2647
2648
2649 static int
symtable_handle_comprehension(struct symtable * st,expr_ty e,identifier scope_name,asdl_comprehension_seq * generators,expr_ty elt,expr_ty value)2650 symtable_handle_comprehension(struct symtable *st, expr_ty e,
2651 identifier scope_name, asdl_comprehension_seq *generators,
2652 expr_ty elt, expr_ty value)
2653 {
2654 int is_generator = (e->kind == GeneratorExp_kind);
2655 comprehension_ty outermost = ((comprehension_ty)
2656 asdl_seq_GET(generators, 0));
2657 /* Outermost iterator is evaluated in current scope */
2658 st->st_cur->ste_comp_iter_expr++;
2659 VISIT(st, expr, outermost->iter);
2660 st->st_cur->ste_comp_iter_expr--;
2661 /* Create comprehension scope for the rest */
2662 if (!scope_name ||
2663 !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e,
2664 e->lineno, e->col_offset,
2665 e->end_lineno, e->end_col_offset)) {
2666 return 0;
2667 }
2668 switch(e->kind) {
2669 case ListComp_kind:
2670 st->st_cur->ste_comprehension = ListComprehension;
2671 break;
2672 case SetComp_kind:
2673 st->st_cur->ste_comprehension = SetComprehension;
2674 break;
2675 case DictComp_kind:
2676 st->st_cur->ste_comprehension = DictComprehension;
2677 break;
2678 default:
2679 st->st_cur->ste_comprehension = GeneratorExpression;
2680 break;
2681 }
2682 if (outermost->is_async) {
2683 st->st_cur->ste_coroutine = 1;
2684 }
2685
2686 /* Outermost iter is received as an argument */
2687 if (!symtable_implicit_arg(st, 0)) {
2688 symtable_exit_block(st);
2689 return 0;
2690 }
2691 /* Visit iteration variable target, and mark them as such */
2692 st->st_cur->ste_comp_iter_target = 1;
2693 VISIT(st, expr, outermost->target);
2694 st->st_cur->ste_comp_iter_target = 0;
2695 /* Visit the rest of the comprehension body */
2696 VISIT_SEQ(st, expr, outermost->ifs);
2697 VISIT_SEQ_TAIL(st, comprehension, generators, 1);
2698 if (value)
2699 VISIT(st, expr, value);
2700 VISIT(st, expr, elt);
2701 st->st_cur->ste_generator = is_generator;
2702 int is_async = st->st_cur->ste_coroutine && !is_generator;
2703 if (!symtable_exit_block(st)) {
2704 return 0;
2705 }
2706 if (is_async) {
2707 st->st_cur->ste_coroutine = 1;
2708 }
2709 return 1;
2710 }
2711
2712 static int
symtable_visit_genexp(struct symtable * st,expr_ty e)2713 symtable_visit_genexp(struct symtable *st, expr_ty e)
2714 {
2715 return symtable_handle_comprehension(st, e, &_Py_ID(genexpr),
2716 e->v.GeneratorExp.generators,
2717 e->v.GeneratorExp.elt, NULL);
2718 }
2719
2720 static int
symtable_visit_listcomp(struct symtable * st,expr_ty e)2721 symtable_visit_listcomp(struct symtable *st, expr_ty e)
2722 {
2723 return symtable_handle_comprehension(st, e, &_Py_ID(listcomp),
2724 e->v.ListComp.generators,
2725 e->v.ListComp.elt, NULL);
2726 }
2727
2728 static int
symtable_visit_setcomp(struct symtable * st,expr_ty e)2729 symtable_visit_setcomp(struct symtable *st, expr_ty e)
2730 {
2731 return symtable_handle_comprehension(st, e, &_Py_ID(setcomp),
2732 e->v.SetComp.generators,
2733 e->v.SetComp.elt, NULL);
2734 }
2735
2736 static int
symtable_visit_dictcomp(struct symtable * st,expr_ty e)2737 symtable_visit_dictcomp(struct symtable *st, expr_ty e)
2738 {
2739 return symtable_handle_comprehension(st, e, &_Py_ID(dictcomp),
2740 e->v.DictComp.generators,
2741 e->v.DictComp.key,
2742 e->v.DictComp.value);
2743 }
2744
2745 static int
symtable_raise_if_annotation_block(struct symtable * st,const char * name,expr_ty e)2746 symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e)
2747 {
2748 enum _block_type type = st->st_cur->ste_type;
2749 if (type == AnnotationBlock)
2750 PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name);
2751 else if (type == TypeVariableBlock) {
2752 const char *info = st->st_cur->ste_scope_info;
2753 assert(info != NULL); // e.g., info == "a ParamSpec default"
2754 PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_VARIABLE, name, info);
2755 }
2756 else if (type == TypeAliasBlock) {
2757 // for now, we do not have any extra information
2758 assert(st->st_cur->ste_scope_info == NULL);
2759 PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_ALIAS, name);
2760 }
2761 else if (type == TypeParametersBlock) {
2762 // for now, we do not have any extra information
2763 assert(st->st_cur->ste_scope_info == NULL);
2764 PyErr_Format(PyExc_SyntaxError, EXPR_NOT_ALLOWED_IN_TYPE_PARAMETERS, name);
2765 }
2766 else
2767 return 1;
2768
2769 PyErr_RangedSyntaxLocationObject(st->st_filename,
2770 e->lineno,
2771 e->col_offset + 1,
2772 e->end_lineno,
2773 e->end_col_offset + 1);
2774 return 0;
2775 }
2776
2777 static int
symtable_raise_if_comprehension_block(struct symtable * st,expr_ty e)2778 symtable_raise_if_comprehension_block(struct symtable *st, expr_ty e) {
2779 _Py_comprehension_ty type = st->st_cur->ste_comprehension;
2780 PyErr_SetString(PyExc_SyntaxError,
2781 (type == ListComprehension) ? "'yield' inside list comprehension" :
2782 (type == SetComprehension) ? "'yield' inside set comprehension" :
2783 (type == DictComprehension) ? "'yield' inside dict comprehension" :
2784 "'yield' inside generator expression");
2785 PyErr_RangedSyntaxLocationObject(st->st_filename,
2786 e->lineno, e->col_offset + 1,
2787 e->end_lineno, e->end_col_offset + 1);
2788 VISIT_QUIT(st, 0);
2789 }
2790
2791 struct symtable *
_Py_SymtableStringObjectFlags(const char * str,PyObject * filename,int start,PyCompilerFlags * flags)2792 _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
2793 int start, PyCompilerFlags *flags)
2794 {
2795 struct symtable *st;
2796 mod_ty mod;
2797 PyArena *arena;
2798
2799 arena = _PyArena_New();
2800 if (arena == NULL)
2801 return NULL;
2802
2803 mod = _PyParser_ASTFromString(str, filename, start, flags, arena);
2804 if (mod == NULL) {
2805 _PyArena_Free(arena);
2806 return NULL;
2807 }
2808 _PyFutureFeatures future;
2809 if (!_PyFuture_FromAST(mod, filename, &future)) {
2810 _PyArena_Free(arena);
2811 return NULL;
2812 }
2813 future.ff_features |= flags->cf_flags;
2814 st = _PySymtable_Build(mod, filename, &future);
2815 _PyArena_Free(arena);
2816 return st;
2817 }
2818
2819 PyObject *
_Py_MaybeMangle(PyObject * privateobj,PySTEntryObject * ste,PyObject * name)2820 _Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name)
2821 {
2822 /* Special case for type parameter blocks around generic classes:
2823 * we want to mangle type parameter names (so a type param with a private
2824 * name can be used inside the class body), but we don't want to mangle
2825 * any other names that appear within the type parameter scope.
2826 */
2827 if (ste->ste_mangled_names != NULL) {
2828 int result = PySet_Contains(ste->ste_mangled_names, name);
2829 if (result < 0) {
2830 return NULL;
2831 }
2832 if (result == 0) {
2833 return Py_NewRef(name);
2834 }
2835 }
2836 return _Py_Mangle(privateobj, name);
2837 }
2838
2839 PyObject *
_Py_Mangle(PyObject * privateobj,PyObject * ident)2840 _Py_Mangle(PyObject *privateobj, PyObject *ident)
2841 {
2842 /* Name mangling: __private becomes _classname__private.
2843 This is independent from how the name is used. */
2844 if (privateobj == NULL || !PyUnicode_Check(privateobj) ||
2845 PyUnicode_READ_CHAR(ident, 0) != '_' ||
2846 PyUnicode_READ_CHAR(ident, 1) != '_') {
2847 return Py_NewRef(ident);
2848 }
2849 size_t nlen = PyUnicode_GET_LENGTH(ident);
2850 size_t plen = PyUnicode_GET_LENGTH(privateobj);
2851 /* Don't mangle __id__ or names with dots.
2852
2853 The only time a name with a dot can occur is when
2854 we are compiling an import statement that has a
2855 package name.
2856
2857 TODO(jhylton): Decide whether we want to support
2858 mangling of the module name, e.g. __M.X.
2859 */
2860 if ((PyUnicode_READ_CHAR(ident, nlen-1) == '_' &&
2861 PyUnicode_READ_CHAR(ident, nlen-2) == '_') ||
2862 PyUnicode_FindChar(ident, '.', 0, nlen, 1) != -1) {
2863 return Py_NewRef(ident); /* Don't mangle __whatever__ */
2864 }
2865 /* Strip leading underscores from class name */
2866 size_t ipriv = 0;
2867 while (PyUnicode_READ_CHAR(privateobj, ipriv) == '_') {
2868 ipriv++;
2869 }
2870 if (ipriv == plen) {
2871 return Py_NewRef(ident); /* Don't mangle if class is just underscores */
2872 }
2873 plen -= ipriv;
2874
2875 if (plen + nlen >= PY_SSIZE_T_MAX - 1) {
2876 PyErr_SetString(PyExc_OverflowError,
2877 "private identifier too large to be mangled");
2878 return NULL;
2879 }
2880
2881 Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(ident);
2882 if (PyUnicode_MAX_CHAR_VALUE(privateobj) > maxchar) {
2883 maxchar = PyUnicode_MAX_CHAR_VALUE(privateobj);
2884 }
2885
2886 PyObject *result = PyUnicode_New(1 + nlen + plen, maxchar);
2887 if (!result) {
2888 return NULL;
2889 }
2890 /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */
2891 PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_');
2892 if (PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen) < 0) {
2893 Py_DECREF(result);
2894 return NULL;
2895 }
2896 if (PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen) < 0) {
2897 Py_DECREF(result);
2898 return NULL;
2899 }
2900 assert(_PyUnicode_CheckConsistency(result, 1));
2901 return result;
2902 }
2903