• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * This file compiles an abstract syntax tree (AST) into Python bytecode.
3  *
4  * The primary entry point is PyAST_Compile(), which returns a
5  * PyCodeObject.  The compiler makes several passes to build the code
6  * object:
7  *   1. Checks for future statements.  See future.c
8  *   2. Builds a symbol table.  See symtable.c.
9  *   3. Generate code for basic blocks.  See compiler_mod() in this file.
10  *   4. Assemble the basic blocks into final code.  See assemble() in
11  *      this file.
12  *   5. Optimize the byte code (peephole optimizations).  See peephole.c
13  *
14  * Note that compiler_mod() suggests module, but the module ast type
15  * (mod_ty) has cases for expressions and interactive statements.
16  *
17  * CAUTION: The VISIT_* macros abort the current function when they
18  * encounter a problem. So don't invoke them when there is memory
19  * which needs to be released. Code blocks are OK, as the compiler
20  * structure takes care of releasing those.  Use the arena to manage
21  * objects.
22  */
23 
24 #include "Python.h"
25 
26 #include "Python-ast.h"
27 #include "node.h"
28 #include "ast.h"
29 #include "code.h"
30 #include "symtable.h"
31 #include "opcode.h"
32 #include "wordcode_helpers.h"
33 
34 #define DEFAULT_BLOCK_SIZE 16
35 #define DEFAULT_BLOCKS 8
36 #define DEFAULT_CODE_SIZE 128
37 #define DEFAULT_LNOTAB_SIZE 16
38 
39 #define COMP_GENEXP   0
40 #define COMP_LISTCOMP 1
41 #define COMP_SETCOMP  2
42 #define COMP_DICTCOMP 3
43 
44 struct instr {
45     unsigned i_jabs : 1;
46     unsigned i_jrel : 1;
47     unsigned char i_opcode;
48     int i_oparg;
49     struct basicblock_ *i_target; /* target block (if jump instruction) */
50     int i_lineno;
51 };
52 
53 typedef struct basicblock_ {
54     /* Each basicblock in a compilation unit is linked via b_list in the
55        reverse order that the block are allocated.  b_list points to the next
56        block, not to be confused with b_next, which is next by control flow. */
57     struct basicblock_ *b_list;
58     /* number of instructions used */
59     int b_iused;
60     /* length of instruction array (b_instr) */
61     int b_ialloc;
62     /* pointer to an array of instructions, initially NULL */
63     struct instr *b_instr;
64     /* If b_next is non-NULL, it is a pointer to the next
65        block reached by normal control flow. */
66     struct basicblock_ *b_next;
67     /* b_seen is used to perform a DFS of basicblocks. */
68     unsigned b_seen : 1;
69     /* b_return is true if a RETURN_VALUE opcode is inserted. */
70     unsigned b_return : 1;
71     /* depth of stack upon entry of block, computed by stackdepth() */
72     int b_startdepth;
73     /* instruction offset for block, computed by assemble_jump_offsets() */
74     int b_offset;
75 } basicblock;
76 
77 /* fblockinfo tracks the current frame block.
78 
79 A frame block is used to handle loops, try/except, and try/finally.
80 It's called a frame block to distinguish it from a basic block in the
81 compiler IR.
82 */
83 
84 enum fblocktype { LOOP, EXCEPT, FINALLY_TRY, FINALLY_END };
85 
86 struct fblockinfo {
87     enum fblocktype fb_type;
88     basicblock *fb_block;
89 };
90 
91 enum {
92     COMPILER_SCOPE_MODULE,
93     COMPILER_SCOPE_CLASS,
94     COMPILER_SCOPE_FUNCTION,
95     COMPILER_SCOPE_ASYNC_FUNCTION,
96     COMPILER_SCOPE_LAMBDA,
97     COMPILER_SCOPE_COMPREHENSION,
98 };
99 
100 /* The following items change on entry and exit of code blocks.
101    They must be saved and restored when returning to a block.
102 */
103 struct compiler_unit {
104     PySTEntryObject *u_ste;
105 
106     PyObject *u_name;
107     PyObject *u_qualname;  /* dot-separated qualified name (lazy) */
108     int u_scope_type;
109 
110     /* The following fields are dicts that map objects to
111        the index of them in co_XXX.      The index is used as
112        the argument for opcodes that refer to those collections.
113     */
114     PyObject *u_consts;    /* all constants */
115     PyObject *u_names;     /* all names */
116     PyObject *u_varnames;  /* local variables */
117     PyObject *u_cellvars;  /* cell variables */
118     PyObject *u_freevars;  /* free variables */
119 
120     PyObject *u_private;        /* for private name mangling */
121 
122     Py_ssize_t u_argcount;        /* number of arguments for block */
123     Py_ssize_t u_kwonlyargcount; /* number of keyword only arguments for block */
124     /* Pointer to the most recently allocated block.  By following b_list
125        members, you can reach all early allocated blocks. */
126     basicblock *u_blocks;
127     basicblock *u_curblock; /* pointer to current block */
128 
129     int u_nfblocks;
130     struct fblockinfo u_fblock[CO_MAXBLOCKS];
131 
132     int u_firstlineno; /* the first lineno of the block */
133     int u_lineno;          /* the lineno for the current stmt */
134     int u_col_offset;      /* the offset of the current stmt */
135     int u_lineno_set;  /* boolean to indicate whether instr
136                           has been generated with current lineno */
137 };
138 
139 /* This struct captures the global state of a compilation.
140 
141 The u pointer points to the current compilation unit, while units
142 for enclosing blocks are stored in c_stack.     The u and c_stack are
143 managed by compiler_enter_scope() and compiler_exit_scope().
144 
145 Note that we don't track recursion levels during compilation - the
146 task of detecting and rejecting excessive levels of nesting is
147 handled by the symbol analysis pass.
148 
149 */
150 
151 struct compiler {
152     PyObject *c_filename;
153     struct symtable *c_st;
154     PyFutureFeatures *c_future; /* pointer to module's __future__ */
155     PyCompilerFlags *c_flags;
156 
157     int c_optimize;              /* optimization level */
158     int c_interactive;           /* true if in interactive mode */
159     int c_nestlevel;
160 
161     struct compiler_unit *u; /* compiler state for current block */
162     PyObject *c_stack;           /* Python list holding compiler_unit ptrs */
163     PyArena *c_arena;            /* pointer to memory allocation arena */
164 };
165 
166 static int compiler_enter_scope(struct compiler *, identifier, int, void *, int);
167 static void compiler_free(struct compiler *);
168 static basicblock *compiler_new_block(struct compiler *);
169 static int compiler_next_instr(struct compiler *, basicblock *);
170 static int compiler_addop(struct compiler *, int);
171 static int compiler_addop_o(struct compiler *, int, PyObject *, PyObject *);
172 static int compiler_addop_i(struct compiler *, int, Py_ssize_t);
173 static int compiler_addop_j(struct compiler *, int, basicblock *, int);
174 static int compiler_error(struct compiler *, const char *);
175 static int compiler_nameop(struct compiler *, identifier, expr_context_ty);
176 
177 static PyCodeObject *compiler_mod(struct compiler *, mod_ty);
178 static int compiler_visit_stmt(struct compiler *, stmt_ty);
179 static int compiler_visit_keyword(struct compiler *, keyword_ty);
180 static int compiler_visit_expr(struct compiler *, expr_ty);
181 static int compiler_augassign(struct compiler *, stmt_ty);
182 static int compiler_annassign(struct compiler *, stmt_ty);
183 static int compiler_visit_slice(struct compiler *, slice_ty,
184                                 expr_context_ty);
185 
186 static int compiler_push_fblock(struct compiler *, enum fblocktype,
187                                 basicblock *);
188 static void compiler_pop_fblock(struct compiler *, enum fblocktype,
189                                 basicblock *);
190 /* Returns true if there is a loop on the fblock stack. */
191 static int compiler_in_loop(struct compiler *);
192 
193 static int inplace_binop(struct compiler *, operator_ty);
194 static int expr_constant(expr_ty);
195 
196 static int compiler_with(struct compiler *, stmt_ty, int);
197 static int compiler_async_with(struct compiler *, stmt_ty, int);
198 static int compiler_async_for(struct compiler *, stmt_ty);
199 static int compiler_call_helper(struct compiler *c, int n,
200                                 asdl_seq *args,
201                                 asdl_seq *keywords);
202 static int compiler_try_except(struct compiler *, stmt_ty);
203 static int compiler_set_qualname(struct compiler *);
204 
205 static int compiler_sync_comprehension_generator(
206                                       struct compiler *c,
207                                       asdl_seq *generators, int gen_index,
208                                       expr_ty elt, expr_ty val, int type);
209 
210 static int compiler_async_comprehension_generator(
211                                       struct compiler *c,
212                                       asdl_seq *generators, int gen_index,
213                                       expr_ty elt, expr_ty val, int type);
214 
215 static PyCodeObject *assemble(struct compiler *, int addNone);
216 static PyObject *__doc__, *__annotations__;
217 
218 #define CAPSULE_NAME "compile.c compiler unit"
219 
220 PyObject *
_Py_Mangle(PyObject * privateobj,PyObject * ident)221 _Py_Mangle(PyObject *privateobj, PyObject *ident)
222 {
223     /* Name mangling: __private becomes _classname__private.
224        This is independent from how the name is used. */
225     PyObject *result;
226     size_t nlen, plen, ipriv;
227     Py_UCS4 maxchar;
228     if (privateobj == NULL || !PyUnicode_Check(privateobj) ||
229         PyUnicode_READ_CHAR(ident, 0) != '_' ||
230         PyUnicode_READ_CHAR(ident, 1) != '_') {
231         Py_INCREF(ident);
232         return ident;
233     }
234     nlen = PyUnicode_GET_LENGTH(ident);
235     plen = PyUnicode_GET_LENGTH(privateobj);
236     /* Don't mangle __id__ or names with dots.
237 
238        The only time a name with a dot can occur is when
239        we are compiling an import statement that has a
240        package name.
241 
242        TODO(jhylton): Decide whether we want to support
243        mangling of the module name, e.g. __M.X.
244     */
245     if ((PyUnicode_READ_CHAR(ident, nlen-1) == '_' &&
246          PyUnicode_READ_CHAR(ident, nlen-2) == '_') ||
247         PyUnicode_FindChar(ident, '.', 0, nlen, 1) != -1) {
248         Py_INCREF(ident);
249         return ident; /* Don't mangle __whatever__ */
250     }
251     /* Strip leading underscores from class name */
252     ipriv = 0;
253     while (PyUnicode_READ_CHAR(privateobj, ipriv) == '_')
254         ipriv++;
255     if (ipriv == plen) {
256         Py_INCREF(ident);
257         return ident; /* Don't mangle if class is just underscores */
258     }
259     plen -= ipriv;
260 
261     if (plen + nlen >= PY_SSIZE_T_MAX - 1) {
262         PyErr_SetString(PyExc_OverflowError,
263                         "private identifier too large to be mangled");
264         return NULL;
265     }
266 
267     maxchar = PyUnicode_MAX_CHAR_VALUE(ident);
268     if (PyUnicode_MAX_CHAR_VALUE(privateobj) > maxchar)
269         maxchar = PyUnicode_MAX_CHAR_VALUE(privateobj);
270 
271     result = PyUnicode_New(1 + nlen + plen, maxchar);
272     if (!result)
273         return 0;
274     /* ident = "_" + priv[ipriv:] + ident # i.e. 1+plen+nlen bytes */
275     PyUnicode_WRITE(PyUnicode_KIND(result), PyUnicode_DATA(result), 0, '_');
276     if (PyUnicode_CopyCharacters(result, 1, privateobj, ipriv, plen) < 0) {
277         Py_DECREF(result);
278         return NULL;
279     }
280     if (PyUnicode_CopyCharacters(result, plen+1, ident, 0, nlen) < 0) {
281         Py_DECREF(result);
282         return NULL;
283     }
284     assert(_PyUnicode_CheckConsistency(result, 1));
285     return result;
286 }
287 
288 static int
compiler_init(struct compiler * c)289 compiler_init(struct compiler *c)
290 {
291     memset(c, 0, sizeof(struct compiler));
292 
293     c->c_stack = PyList_New(0);
294     if (!c->c_stack)
295         return 0;
296 
297     return 1;
298 }
299 
300 PyCodeObject *
PyAST_CompileObject(mod_ty mod,PyObject * filename,PyCompilerFlags * flags,int optimize,PyArena * arena)301 PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags,
302                    int optimize, PyArena *arena)
303 {
304     struct compiler c;
305     PyCodeObject *co = NULL;
306     PyCompilerFlags local_flags;
307     int merged;
308 
309     if (!__doc__) {
310         __doc__ = PyUnicode_InternFromString("__doc__");
311         if (!__doc__)
312             return NULL;
313     }
314     if (!__annotations__) {
315         __annotations__ = PyUnicode_InternFromString("__annotations__");
316         if (!__annotations__)
317             return NULL;
318     }
319     if (!compiler_init(&c))
320         return NULL;
321     Py_INCREF(filename);
322     c.c_filename = filename;
323     c.c_arena = arena;
324     c.c_future = PyFuture_FromASTObject(mod, filename);
325     if (c.c_future == NULL)
326         goto finally;
327     if (!flags) {
328         local_flags.cf_flags = 0;
329         flags = &local_flags;
330     }
331     merged = c.c_future->ff_features | flags->cf_flags;
332     c.c_future->ff_features = merged;
333     flags->cf_flags = merged;
334     c.c_flags = flags;
335     c.c_optimize = (optimize == -1) ? Py_OptimizeFlag : optimize;
336     c.c_nestlevel = 0;
337 
338     if (!_PyAST_Optimize(mod, arena, c.c_optimize)) {
339         goto finally;
340     }
341 
342     c.c_st = PySymtable_BuildObject(mod, filename, c.c_future);
343     if (c.c_st == NULL) {
344         if (!PyErr_Occurred())
345             PyErr_SetString(PyExc_SystemError, "no symtable");
346         goto finally;
347     }
348 
349     co = compiler_mod(&c, mod);
350 
351  finally:
352     compiler_free(&c);
353     assert(co || PyErr_Occurred());
354     return co;
355 }
356 
357 PyCodeObject *
PyAST_CompileEx(mod_ty mod,const char * filename_str,PyCompilerFlags * flags,int optimize,PyArena * arena)358 PyAST_CompileEx(mod_ty mod, const char *filename_str, PyCompilerFlags *flags,
359                 int optimize, PyArena *arena)
360 {
361     PyObject *filename;
362     PyCodeObject *co;
363     filename = PyUnicode_DecodeFSDefault(filename_str);
364     if (filename == NULL)
365         return NULL;
366     co = PyAST_CompileObject(mod, filename, flags, optimize, arena);
367     Py_DECREF(filename);
368     return co;
369 
370 }
371 
372 PyCodeObject *
PyNode_Compile(struct _node * n,const char * filename)373 PyNode_Compile(struct _node *n, const char *filename)
374 {
375     PyCodeObject *co = NULL;
376     mod_ty mod;
377     PyArena *arena = PyArena_New();
378     if (!arena)
379         return NULL;
380     mod = PyAST_FromNode(n, NULL, filename, arena);
381     if (mod)
382         co = PyAST_Compile(mod, filename, NULL, arena);
383     PyArena_Free(arena);
384     return co;
385 }
386 
387 static void
compiler_free(struct compiler * c)388 compiler_free(struct compiler *c)
389 {
390     if (c->c_st)
391         PySymtable_Free(c->c_st);
392     if (c->c_future)
393         PyObject_Free(c->c_future);
394     Py_XDECREF(c->c_filename);
395     Py_DECREF(c->c_stack);
396 }
397 
398 static PyObject *
list2dict(PyObject * list)399 list2dict(PyObject *list)
400 {
401     Py_ssize_t i, n;
402     PyObject *v, *k;
403     PyObject *dict = PyDict_New();
404     if (!dict) return NULL;
405 
406     n = PyList_Size(list);
407     for (i = 0; i < n; i++) {
408         v = PyLong_FromSsize_t(i);
409         if (!v) {
410             Py_DECREF(dict);
411             return NULL;
412         }
413         k = PyList_GET_ITEM(list, i);
414         k = _PyCode_ConstantKey(k);
415         if (k == NULL || PyDict_SetItem(dict, k, v) < 0) {
416             Py_XDECREF(k);
417             Py_DECREF(v);
418             Py_DECREF(dict);
419             return NULL;
420         }
421         Py_DECREF(k);
422         Py_DECREF(v);
423     }
424     return dict;
425 }
426 
427 /* Return new dict containing names from src that match scope(s).
428 
429 src is a symbol table dictionary.  If the scope of a name matches
430 either scope_type or flag is set, insert it into the new dict.  The
431 values are integers, starting at offset and increasing by one for
432 each key.
433 */
434 
435 static PyObject *
dictbytype(PyObject * src,int scope_type,int flag,Py_ssize_t offset)436 dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset)
437 {
438     Py_ssize_t i = offset, scope, num_keys, key_i;
439     PyObject *k, *v, *dest = PyDict_New();
440     PyObject *sorted_keys;
441 
442     assert(offset >= 0);
443     if (dest == NULL)
444         return NULL;
445 
446     /* Sort the keys so that we have a deterministic order on the indexes
447        saved in the returned dictionary.  These indexes are used as indexes
448        into the free and cell var storage.  Therefore if they aren't
449        deterministic, then the generated bytecode is not deterministic.
450     */
451     sorted_keys = PyDict_Keys(src);
452     if (sorted_keys == NULL)
453         return NULL;
454     if (PyList_Sort(sorted_keys) != 0) {
455         Py_DECREF(sorted_keys);
456         return NULL;
457     }
458     num_keys = PyList_GET_SIZE(sorted_keys);
459 
460     for (key_i = 0; key_i < num_keys; key_i++) {
461         /* XXX this should probably be a macro in symtable.h */
462         long vi;
463         k = PyList_GET_ITEM(sorted_keys, key_i);
464         v = PyDict_GetItem(src, k);
465         assert(PyLong_Check(v));
466         vi = PyLong_AS_LONG(v);
467         scope = (vi >> SCOPE_OFFSET) & SCOPE_MASK;
468 
469         if (scope == scope_type || vi & flag) {
470             PyObject *tuple, *item = PyLong_FromSsize_t(i);
471             if (item == NULL) {
472                 Py_DECREF(sorted_keys);
473                 Py_DECREF(dest);
474                 return NULL;
475             }
476             i++;
477             tuple = _PyCode_ConstantKey(k);
478             if (!tuple || PyDict_SetItem(dest, tuple, item) < 0) {
479                 Py_DECREF(sorted_keys);
480                 Py_DECREF(item);
481                 Py_DECREF(dest);
482                 Py_XDECREF(tuple);
483                 return NULL;
484             }
485             Py_DECREF(item);
486             Py_DECREF(tuple);
487         }
488     }
489     Py_DECREF(sorted_keys);
490     return dest;
491 }
492 
493 static void
compiler_unit_check(struct compiler_unit * u)494 compiler_unit_check(struct compiler_unit *u)
495 {
496     basicblock *block;
497     for (block = u->u_blocks; block != NULL; block = block->b_list) {
498         assert((uintptr_t)block != 0xcbcbcbcbU);
499         assert((uintptr_t)block != 0xfbfbfbfbU);
500         assert((uintptr_t)block != 0xdbdbdbdbU);
501         if (block->b_instr != NULL) {
502             assert(block->b_ialloc > 0);
503             assert(block->b_iused > 0);
504             assert(block->b_ialloc >= block->b_iused);
505         }
506         else {
507             assert (block->b_iused == 0);
508             assert (block->b_ialloc == 0);
509         }
510     }
511 }
512 
513 static void
compiler_unit_free(struct compiler_unit * u)514 compiler_unit_free(struct compiler_unit *u)
515 {
516     basicblock *b, *next;
517 
518     compiler_unit_check(u);
519     b = u->u_blocks;
520     while (b != NULL) {
521         if (b->b_instr)
522             PyObject_Free((void *)b->b_instr);
523         next = b->b_list;
524         PyObject_Free((void *)b);
525         b = next;
526     }
527     Py_CLEAR(u->u_ste);
528     Py_CLEAR(u->u_name);
529     Py_CLEAR(u->u_qualname);
530     Py_CLEAR(u->u_consts);
531     Py_CLEAR(u->u_names);
532     Py_CLEAR(u->u_varnames);
533     Py_CLEAR(u->u_freevars);
534     Py_CLEAR(u->u_cellvars);
535     Py_CLEAR(u->u_private);
536     PyObject_Free(u);
537 }
538 
539 static int
compiler_enter_scope(struct compiler * c,identifier name,int scope_type,void * key,int lineno)540 compiler_enter_scope(struct compiler *c, identifier name,
541                      int scope_type, void *key, int lineno)
542 {
543     struct compiler_unit *u;
544     basicblock *block;
545 
546     u = (struct compiler_unit *)PyObject_Malloc(sizeof(
547                                             struct compiler_unit));
548     if (!u) {
549         PyErr_NoMemory();
550         return 0;
551     }
552     memset(u, 0, sizeof(struct compiler_unit));
553     u->u_scope_type = scope_type;
554     u->u_argcount = 0;
555     u->u_kwonlyargcount = 0;
556     u->u_ste = PySymtable_Lookup(c->c_st, key);
557     if (!u->u_ste) {
558         compiler_unit_free(u);
559         return 0;
560     }
561     Py_INCREF(name);
562     u->u_name = name;
563     u->u_varnames = list2dict(u->u_ste->ste_varnames);
564     u->u_cellvars = dictbytype(u->u_ste->ste_symbols, CELL, 0, 0);
565     if (!u->u_varnames || !u->u_cellvars) {
566         compiler_unit_free(u);
567         return 0;
568     }
569     if (u->u_ste->ste_needs_class_closure) {
570         /* Cook up an implicit __class__ cell. */
571         _Py_IDENTIFIER(__class__);
572         PyObject *tuple, *name;
573         int res;
574         assert(u->u_scope_type == COMPILER_SCOPE_CLASS);
575         assert(PyDict_GET_SIZE(u->u_cellvars) == 0);
576         name = _PyUnicode_FromId(&PyId___class__);
577         if (!name) {
578             compiler_unit_free(u);
579             return 0;
580         }
581         tuple = _PyCode_ConstantKey(name);
582         if (!tuple) {
583             compiler_unit_free(u);
584             return 0;
585         }
586         res = PyDict_SetItem(u->u_cellvars, tuple, _PyLong_Zero);
587         Py_DECREF(tuple);
588         if (res < 0) {
589             compiler_unit_free(u);
590             return 0;
591         }
592     }
593 
594     u->u_freevars = dictbytype(u->u_ste->ste_symbols, FREE, DEF_FREE_CLASS,
595                                PyDict_GET_SIZE(u->u_cellvars));
596     if (!u->u_freevars) {
597         compiler_unit_free(u);
598         return 0;
599     }
600 
601     u->u_blocks = NULL;
602     u->u_nfblocks = 0;
603     u->u_firstlineno = lineno;
604     u->u_lineno = 0;
605     u->u_col_offset = 0;
606     u->u_lineno_set = 0;
607     u->u_consts = PyDict_New();
608     if (!u->u_consts) {
609         compiler_unit_free(u);
610         return 0;
611     }
612     u->u_names = PyDict_New();
613     if (!u->u_names) {
614         compiler_unit_free(u);
615         return 0;
616     }
617 
618     u->u_private = NULL;
619 
620     /* Push the old compiler_unit on the stack. */
621     if (c->u) {
622         PyObject *capsule = PyCapsule_New(c->u, CAPSULE_NAME, NULL);
623         if (!capsule || PyList_Append(c->c_stack, capsule) < 0) {
624             Py_XDECREF(capsule);
625             compiler_unit_free(u);
626             return 0;
627         }
628         Py_DECREF(capsule);
629         u->u_private = c->u->u_private;
630         Py_XINCREF(u->u_private);
631     }
632     c->u = u;
633 
634     c->c_nestlevel++;
635 
636     block = compiler_new_block(c);
637     if (block == NULL)
638         return 0;
639     c->u->u_curblock = block;
640 
641     if (u->u_scope_type != COMPILER_SCOPE_MODULE) {
642         if (!compiler_set_qualname(c))
643             return 0;
644     }
645 
646     return 1;
647 }
648 
649 static void
compiler_exit_scope(struct compiler * c)650 compiler_exit_scope(struct compiler *c)
651 {
652     Py_ssize_t n;
653     PyObject *capsule;
654 
655     c->c_nestlevel--;
656     compiler_unit_free(c->u);
657     /* Restore c->u to the parent unit. */
658     n = PyList_GET_SIZE(c->c_stack) - 1;
659     if (n >= 0) {
660         capsule = PyList_GET_ITEM(c->c_stack, n);
661         c->u = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);
662         assert(c->u);
663         /* we are deleting from a list so this really shouldn't fail */
664         if (PySequence_DelItem(c->c_stack, n) < 0)
665             Py_FatalError("compiler_exit_scope()");
666         compiler_unit_check(c->u);
667     }
668     else
669         c->u = NULL;
670 
671 }
672 
673 static int
compiler_set_qualname(struct compiler * c)674 compiler_set_qualname(struct compiler *c)
675 {
676     _Py_static_string(dot, ".");
677     _Py_static_string(dot_locals, ".<locals>");
678     Py_ssize_t stack_size;
679     struct compiler_unit *u = c->u;
680     PyObject *name, *base, *dot_str, *dot_locals_str;
681 
682     base = NULL;
683     stack_size = PyList_GET_SIZE(c->c_stack);
684     assert(stack_size >= 1);
685     if (stack_size > 1) {
686         int scope, force_global = 0;
687         struct compiler_unit *parent;
688         PyObject *mangled, *capsule;
689 
690         capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1);
691         parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME);
692         assert(parent);
693 
694         if (u->u_scope_type == COMPILER_SCOPE_FUNCTION
695             || u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION
696             || u->u_scope_type == COMPILER_SCOPE_CLASS) {
697             assert(u->u_name);
698             mangled = _Py_Mangle(parent->u_private, u->u_name);
699             if (!mangled)
700                 return 0;
701             scope = PyST_GetScope(parent->u_ste, mangled);
702             Py_DECREF(mangled);
703             assert(scope != GLOBAL_IMPLICIT);
704             if (scope == GLOBAL_EXPLICIT)
705                 force_global = 1;
706         }
707 
708         if (!force_global) {
709             if (parent->u_scope_type == COMPILER_SCOPE_FUNCTION
710                 || parent->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION
711                 || parent->u_scope_type == COMPILER_SCOPE_LAMBDA) {
712                 dot_locals_str = _PyUnicode_FromId(&dot_locals);
713                 if (dot_locals_str == NULL)
714                     return 0;
715                 base = PyUnicode_Concat(parent->u_qualname, dot_locals_str);
716                 if (base == NULL)
717                     return 0;
718             }
719             else {
720                 Py_INCREF(parent->u_qualname);
721                 base = parent->u_qualname;
722             }
723         }
724     }
725 
726     if (base != NULL) {
727         dot_str = _PyUnicode_FromId(&dot);
728         if (dot_str == NULL) {
729             Py_DECREF(base);
730             return 0;
731         }
732         name = PyUnicode_Concat(base, dot_str);
733         Py_DECREF(base);
734         if (name == NULL)
735             return 0;
736         PyUnicode_Append(&name, u->u_name);
737         if (name == NULL)
738             return 0;
739     }
740     else {
741         Py_INCREF(u->u_name);
742         name = u->u_name;
743     }
744     u->u_qualname = name;
745 
746     return 1;
747 }
748 
749 
750 /* Allocate a new block and return a pointer to it.
751    Returns NULL on error.
752 */
753 
754 static basicblock *
compiler_new_block(struct compiler * c)755 compiler_new_block(struct compiler *c)
756 {
757     basicblock *b;
758     struct compiler_unit *u;
759 
760     u = c->u;
761     b = (basicblock *)PyObject_Malloc(sizeof(basicblock));
762     if (b == NULL) {
763         PyErr_NoMemory();
764         return NULL;
765     }
766     memset((void *)b, 0, sizeof(basicblock));
767     /* Extend the singly linked list of blocks with new block. */
768     b->b_list = u->u_blocks;
769     u->u_blocks = b;
770     return b;
771 }
772 
773 static basicblock *
compiler_next_block(struct compiler * c)774 compiler_next_block(struct compiler *c)
775 {
776     basicblock *block = compiler_new_block(c);
777     if (block == NULL)
778         return NULL;
779     c->u->u_curblock->b_next = block;
780     c->u->u_curblock = block;
781     return block;
782 }
783 
784 static basicblock *
compiler_use_next_block(struct compiler * c,basicblock * block)785 compiler_use_next_block(struct compiler *c, basicblock *block)
786 {
787     assert(block != NULL);
788     c->u->u_curblock->b_next = block;
789     c->u->u_curblock = block;
790     return block;
791 }
792 
793 /* Returns the offset of the next instruction in the current block's
794    b_instr array.  Resizes the b_instr as necessary.
795    Returns -1 on failure.
796 */
797 
798 static int
compiler_next_instr(struct compiler * c,basicblock * b)799 compiler_next_instr(struct compiler *c, basicblock *b)
800 {
801     assert(b != NULL);
802     if (b->b_instr == NULL) {
803         b->b_instr = (struct instr *)PyObject_Malloc(
804                          sizeof(struct instr) * DEFAULT_BLOCK_SIZE);
805         if (b->b_instr == NULL) {
806             PyErr_NoMemory();
807             return -1;
808         }
809         b->b_ialloc = DEFAULT_BLOCK_SIZE;
810         memset((char *)b->b_instr, 0,
811                sizeof(struct instr) * DEFAULT_BLOCK_SIZE);
812     }
813     else if (b->b_iused == b->b_ialloc) {
814         struct instr *tmp;
815         size_t oldsize, newsize;
816         oldsize = b->b_ialloc * sizeof(struct instr);
817         newsize = oldsize << 1;
818 
819         if (oldsize > (SIZE_MAX >> 1)) {
820             PyErr_NoMemory();
821             return -1;
822         }
823 
824         if (newsize == 0) {
825             PyErr_NoMemory();
826             return -1;
827         }
828         b->b_ialloc <<= 1;
829         tmp = (struct instr *)PyObject_Realloc(
830                                         (void *)b->b_instr, newsize);
831         if (tmp == NULL) {
832             PyErr_NoMemory();
833             return -1;
834         }
835         b->b_instr = tmp;
836         memset((char *)b->b_instr + oldsize, 0, newsize - oldsize);
837     }
838     return b->b_iused++;
839 }
840 
841 /* Set the i_lineno member of the instruction at offset off if the
842    line number for the current expression/statement has not
843    already been set.  If it has been set, the call has no effect.
844 
845    The line number is reset in the following cases:
846    - when entering a new scope
847    - on each statement
848    - on each expression that start a new line
849    - before the "except" clause
850    - before the "for" and "while" expressions
851 */
852 
853 static void
compiler_set_lineno(struct compiler * c,int off)854 compiler_set_lineno(struct compiler *c, int off)
855 {
856     basicblock *b;
857     if (c->u->u_lineno_set)
858         return;
859     c->u->u_lineno_set = 1;
860     b = c->u->u_curblock;
861     b->b_instr[off].i_lineno = c->u->u_lineno;
862 }
863 
864 /* Return the stack effect of opcode with argument oparg.
865 
866    Some opcodes have different stack effect when jump to the target and
867    when not jump. The 'jump' parameter specifies the case:
868 
869    * 0 -- when not jump
870    * 1 -- when jump
871    * -1 -- maximal
872  */
873 /* XXX Make the stack effect of WITH_CLEANUP_START and
874    WITH_CLEANUP_FINISH deterministic. */
875 static int
stack_effect(int opcode,int oparg,int jump)876 stack_effect(int opcode, int oparg, int jump)
877 {
878     switch (opcode) {
879         /* Stack manipulation */
880         case POP_TOP:
881             return -1;
882         case ROT_TWO:
883         case ROT_THREE:
884             return 0;
885         case DUP_TOP:
886             return 1;
887         case DUP_TOP_TWO:
888             return 2;
889 
890         /* Unary operators */
891         case UNARY_POSITIVE:
892         case UNARY_NEGATIVE:
893         case UNARY_NOT:
894         case UNARY_INVERT:
895             return 0;
896 
897         case SET_ADD:
898         case LIST_APPEND:
899             return -1;
900         case MAP_ADD:
901             return -2;
902 
903         /* Binary operators */
904         case BINARY_POWER:
905         case BINARY_MULTIPLY:
906         case BINARY_MATRIX_MULTIPLY:
907         case BINARY_MODULO:
908         case BINARY_ADD:
909         case BINARY_SUBTRACT:
910         case BINARY_SUBSCR:
911         case BINARY_FLOOR_DIVIDE:
912         case BINARY_TRUE_DIVIDE:
913             return -1;
914         case INPLACE_FLOOR_DIVIDE:
915         case INPLACE_TRUE_DIVIDE:
916             return -1;
917 
918         case INPLACE_ADD:
919         case INPLACE_SUBTRACT:
920         case INPLACE_MULTIPLY:
921         case INPLACE_MATRIX_MULTIPLY:
922         case INPLACE_MODULO:
923             return -1;
924         case STORE_SUBSCR:
925             return -3;
926         case DELETE_SUBSCR:
927             return -2;
928 
929         case BINARY_LSHIFT:
930         case BINARY_RSHIFT:
931         case BINARY_AND:
932         case BINARY_XOR:
933         case BINARY_OR:
934             return -1;
935         case INPLACE_POWER:
936             return -1;
937         case GET_ITER:
938             return 0;
939 
940         case PRINT_EXPR:
941             return -1;
942         case LOAD_BUILD_CLASS:
943             return 1;
944         case INPLACE_LSHIFT:
945         case INPLACE_RSHIFT:
946         case INPLACE_AND:
947         case INPLACE_XOR:
948         case INPLACE_OR:
949             return -1;
950         case BREAK_LOOP:
951             return 0;
952         case SETUP_WITH:
953             /* 1 in the normal flow.
954              * Restore the stack position and push 6 values before jumping to
955              * the handler if an exception be raised. */
956             return jump ? 6 : 1;
957         case WITH_CLEANUP_START:
958             return 2; /* or 1, depending on TOS */
959         case WITH_CLEANUP_FINISH:
960             /* Pop a variable number of values pushed by WITH_CLEANUP_START
961              * + __exit__ or __aexit__. */
962             return -3;
963         case RETURN_VALUE:
964             return -1;
965         case IMPORT_STAR:
966             return -1;
967         case SETUP_ANNOTATIONS:
968             return 0;
969         case YIELD_VALUE:
970             return 0;
971         case YIELD_FROM:
972             return -1;
973         case POP_BLOCK:
974             return 0;
975         case POP_EXCEPT:
976             return -3;
977         case END_FINALLY:
978             /* Pop 6 values when an exception was raised. */
979             return -6;
980 
981         case STORE_NAME:
982             return -1;
983         case DELETE_NAME:
984             return 0;
985         case UNPACK_SEQUENCE:
986             return oparg-1;
987         case UNPACK_EX:
988             return (oparg&0xFF) + (oparg>>8);
989         case FOR_ITER:
990             /* -1 at end of iterator, 1 if continue iterating. */
991             return jump > 0 ? -1 : 1;
992 
993         case STORE_ATTR:
994             return -2;
995         case DELETE_ATTR:
996             return -1;
997         case STORE_GLOBAL:
998             return -1;
999         case DELETE_GLOBAL:
1000             return 0;
1001         case LOAD_CONST:
1002             return 1;
1003         case LOAD_NAME:
1004             return 1;
1005         case BUILD_TUPLE:
1006         case BUILD_LIST:
1007         case BUILD_SET:
1008         case BUILD_STRING:
1009             return 1-oparg;
1010         case BUILD_LIST_UNPACK:
1011         case BUILD_TUPLE_UNPACK:
1012         case BUILD_TUPLE_UNPACK_WITH_CALL:
1013         case BUILD_SET_UNPACK:
1014         case BUILD_MAP_UNPACK:
1015         case BUILD_MAP_UNPACK_WITH_CALL:
1016             return 1 - oparg;
1017         case BUILD_MAP:
1018             return 1 - 2*oparg;
1019         case BUILD_CONST_KEY_MAP:
1020             return -oparg;
1021         case LOAD_ATTR:
1022             return 0;
1023         case COMPARE_OP:
1024             return -1;
1025         case IMPORT_NAME:
1026             return -1;
1027         case IMPORT_FROM:
1028             return 1;
1029 
1030         /* Jumps */
1031         case JUMP_FORWARD:
1032         case JUMP_ABSOLUTE:
1033             return 0;
1034 
1035         case JUMP_IF_TRUE_OR_POP:
1036         case JUMP_IF_FALSE_OR_POP:
1037             return jump ? 0 : -1;
1038 
1039         case POP_JUMP_IF_FALSE:
1040         case POP_JUMP_IF_TRUE:
1041             return -1;
1042 
1043         case LOAD_GLOBAL:
1044             return 1;
1045 
1046         case CONTINUE_LOOP:
1047             return 0;
1048         case SETUP_LOOP:
1049             return 0;
1050         case SETUP_EXCEPT:
1051         case SETUP_FINALLY:
1052             /* 0 in the normal flow.
1053              * Restore the stack position and push 6 values before jumping to
1054              * the handler if an exception be raised. */
1055             return jump ? 6 : 0;
1056 
1057         case LOAD_FAST:
1058             return 1;
1059         case STORE_FAST:
1060             return -1;
1061         case DELETE_FAST:
1062             return 0;
1063 
1064         case RAISE_VARARGS:
1065             return -oparg;
1066 
1067         /* Functions and calls */
1068         case CALL_FUNCTION:
1069             return -oparg;
1070         case CALL_METHOD:
1071             return -oparg-1;
1072         case CALL_FUNCTION_KW:
1073             return -oparg-1;
1074         case CALL_FUNCTION_EX:
1075             return -1 - ((oparg & 0x01) != 0);
1076         case MAKE_FUNCTION:
1077             return -1 - ((oparg & 0x01) != 0) - ((oparg & 0x02) != 0) -
1078                 ((oparg & 0x04) != 0) - ((oparg & 0x08) != 0);
1079         case BUILD_SLICE:
1080             if (oparg == 3)
1081                 return -2;
1082             else
1083                 return -1;
1084 
1085         /* Closures */
1086         case LOAD_CLOSURE:
1087             return 1;
1088         case LOAD_DEREF:
1089         case LOAD_CLASSDEREF:
1090             return 1;
1091         case STORE_DEREF:
1092             return -1;
1093         case DELETE_DEREF:
1094             return 0;
1095 
1096         /* Iterators and generators */
1097         case GET_AWAITABLE:
1098             return 0;
1099         case SETUP_ASYNC_WITH:
1100             /* 0 in the normal flow.
1101              * Restore the stack position to the position before the result
1102              * of __aenter__ and push 6 values before jumping to the handler
1103              * if an exception be raised. */
1104             return jump ? -1 + 6 : 0;
1105         case BEFORE_ASYNC_WITH:
1106             return 1;
1107         case GET_AITER:
1108             return 0;
1109         case GET_ANEXT:
1110             return 1;
1111         case GET_YIELD_FROM_ITER:
1112             return 0;
1113         case FORMAT_VALUE:
1114             /* If there's a fmt_spec on the stack, we go from 2->1,
1115                else 1->1. */
1116             return (oparg & FVS_MASK) == FVS_HAVE_SPEC ? -1 : 0;
1117         case LOAD_METHOD:
1118             return 1;
1119         default:
1120             return PY_INVALID_STACK_EFFECT;
1121     }
1122     return PY_INVALID_STACK_EFFECT; /* not reachable */
1123 }
1124 
1125 int
PyCompile_OpcodeStackEffect(int opcode,int oparg)1126 PyCompile_OpcodeStackEffect(int opcode, int oparg)
1127 {
1128     return stack_effect(opcode, oparg, -1);
1129 }
1130 
1131 /* Add an opcode with no argument.
1132    Returns 0 on failure, 1 on success.
1133 */
1134 
1135 static int
compiler_addop(struct compiler * c,int opcode)1136 compiler_addop(struct compiler *c, int opcode)
1137 {
1138     basicblock *b;
1139     struct instr *i;
1140     int off;
1141     assert(!HAS_ARG(opcode));
1142     off = compiler_next_instr(c, c->u->u_curblock);
1143     if (off < 0)
1144         return 0;
1145     b = c->u->u_curblock;
1146     i = &b->b_instr[off];
1147     i->i_opcode = opcode;
1148     i->i_oparg = 0;
1149     if (opcode == RETURN_VALUE)
1150         b->b_return = 1;
1151     compiler_set_lineno(c, off);
1152     return 1;
1153 }
1154 
1155 static Py_ssize_t
compiler_add_o(struct compiler * c,PyObject * dict,PyObject * o)1156 compiler_add_o(struct compiler *c, PyObject *dict, PyObject *o)
1157 {
1158     PyObject *t, *v;
1159     Py_ssize_t arg;
1160 
1161     t = _PyCode_ConstantKey(o);
1162     if (t == NULL)
1163         return -1;
1164 
1165     v = PyDict_GetItem(dict, t);
1166     if (!v) {
1167         if (PyErr_Occurred()) {
1168             Py_DECREF(t);
1169             return -1;
1170         }
1171         arg = PyDict_GET_SIZE(dict);
1172         v = PyLong_FromSsize_t(arg);
1173         if (!v) {
1174             Py_DECREF(t);
1175             return -1;
1176         }
1177         if (PyDict_SetItem(dict, t, v) < 0) {
1178             Py_DECREF(t);
1179             Py_DECREF(v);
1180             return -1;
1181         }
1182         Py_DECREF(v);
1183     }
1184     else
1185         arg = PyLong_AsLong(v);
1186     Py_DECREF(t);
1187     return arg;
1188 }
1189 
1190 static int
compiler_addop_o(struct compiler * c,int opcode,PyObject * dict,PyObject * o)1191 compiler_addop_o(struct compiler *c, int opcode, PyObject *dict,
1192                      PyObject *o)
1193 {
1194     Py_ssize_t arg = compiler_add_o(c, dict, o);
1195     if (arg < 0)
1196         return 0;
1197     return compiler_addop_i(c, opcode, arg);
1198 }
1199 
1200 static int
compiler_addop_name(struct compiler * c,int opcode,PyObject * dict,PyObject * o)1201 compiler_addop_name(struct compiler *c, int opcode, PyObject *dict,
1202                     PyObject *o)
1203 {
1204     Py_ssize_t arg;
1205     PyObject *mangled = _Py_Mangle(c->u->u_private, o);
1206     if (!mangled)
1207         return 0;
1208     arg = compiler_add_o(c, dict, mangled);
1209     Py_DECREF(mangled);
1210     if (arg < 0)
1211         return 0;
1212     return compiler_addop_i(c, opcode, arg);
1213 }
1214 
1215 /* Add an opcode with an integer argument.
1216    Returns 0 on failure, 1 on success.
1217 */
1218 
1219 static int
compiler_addop_i(struct compiler * c,int opcode,Py_ssize_t oparg)1220 compiler_addop_i(struct compiler *c, int opcode, Py_ssize_t oparg)
1221 {
1222     struct instr *i;
1223     int off;
1224 
1225     /* oparg value is unsigned, but a signed C int is usually used to store
1226        it in the C code (like Python/ceval.c).
1227 
1228        Limit to 32-bit signed C int (rather than INT_MAX) for portability.
1229 
1230        The argument of a concrete bytecode instruction is limited to 8-bit.
1231        EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */
1232     assert(HAS_ARG(opcode));
1233     assert(0 <= oparg && oparg <= 2147483647);
1234 
1235     off = compiler_next_instr(c, c->u->u_curblock);
1236     if (off < 0)
1237         return 0;
1238     i = &c->u->u_curblock->b_instr[off];
1239     i->i_opcode = opcode;
1240     i->i_oparg = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int);
1241     compiler_set_lineno(c, off);
1242     return 1;
1243 }
1244 
1245 static int
compiler_addop_j(struct compiler * c,int opcode,basicblock * b,int absolute)1246 compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute)
1247 {
1248     struct instr *i;
1249     int off;
1250 
1251     assert(HAS_ARG(opcode));
1252     assert(b != NULL);
1253     off = compiler_next_instr(c, c->u->u_curblock);
1254     if (off < 0)
1255         return 0;
1256     i = &c->u->u_curblock->b_instr[off];
1257     i->i_opcode = opcode;
1258     i->i_target = b;
1259     if (absolute)
1260         i->i_jabs = 1;
1261     else
1262         i->i_jrel = 1;
1263     compiler_set_lineno(c, off);
1264     return 1;
1265 }
1266 
1267 /* NEXT_BLOCK() creates an implicit jump from the current block
1268    to the new block.
1269 
1270    The returns inside this macro make it impossible to decref objects
1271    created in the local function. Local objects should use the arena.
1272 */
1273 #define NEXT_BLOCK(C) { \
1274     if (compiler_next_block((C)) == NULL) \
1275         return 0; \
1276 }
1277 
1278 #define ADDOP(C, OP) { \
1279     if (!compiler_addop((C), (OP))) \
1280         return 0; \
1281 }
1282 
1283 #define ADDOP_IN_SCOPE(C, OP) { \
1284     if (!compiler_addop((C), (OP))) { \
1285         compiler_exit_scope(c); \
1286         return 0; \
1287     } \
1288 }
1289 
1290 #define ADDOP_O(C, OP, O, TYPE) { \
1291     if (!compiler_addop_o((C), (OP), (C)->u->u_ ## TYPE, (O))) \
1292         return 0; \
1293 }
1294 
1295 /* Same as ADDOP_O, but steals a reference. */
1296 #define ADDOP_N(C, OP, O, TYPE) { \
1297     if (!compiler_addop_o((C), (OP), (C)->u->u_ ## TYPE, (O))) { \
1298         Py_DECREF((O)); \
1299         return 0; \
1300     } \
1301     Py_DECREF((O)); \
1302 }
1303 
1304 #define ADDOP_NAME(C, OP, O, TYPE) { \
1305     if (!compiler_addop_name((C), (OP), (C)->u->u_ ## TYPE, (O))) \
1306         return 0; \
1307 }
1308 
1309 #define ADDOP_I(C, OP, O) { \
1310     if (!compiler_addop_i((C), (OP), (O))) \
1311         return 0; \
1312 }
1313 
1314 #define ADDOP_JABS(C, OP, O) { \
1315     if (!compiler_addop_j((C), (OP), (O), 1)) \
1316         return 0; \
1317 }
1318 
1319 #define ADDOP_JREL(C, OP, O) { \
1320     if (!compiler_addop_j((C), (OP), (O), 0)) \
1321         return 0; \
1322 }
1323 
1324 /* VISIT and VISIT_SEQ takes an ASDL type as their second argument.  They use
1325    the ASDL name to synthesize the name of the C type and the visit function.
1326 */
1327 
1328 #define VISIT(C, TYPE, V) {\
1329     if (!compiler_visit_ ## TYPE((C), (V))) \
1330         return 0; \
1331 }
1332 
1333 #define VISIT_IN_SCOPE(C, TYPE, V) {\
1334     if (!compiler_visit_ ## TYPE((C), (V))) { \
1335         compiler_exit_scope(c); \
1336         return 0; \
1337     } \
1338 }
1339 
1340 #define VISIT_SLICE(C, V, CTX) {\
1341     if (!compiler_visit_slice((C), (V), (CTX))) \
1342         return 0; \
1343 }
1344 
1345 #define VISIT_SEQ(C, TYPE, SEQ) { \
1346     int _i; \
1347     asdl_seq *seq = (SEQ); /* avoid variable capture */ \
1348     for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \
1349         TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \
1350         if (!compiler_visit_ ## TYPE((C), elt)) \
1351             return 0; \
1352     } \
1353 }
1354 
1355 #define VISIT_SEQ_IN_SCOPE(C, TYPE, SEQ) { \
1356     int _i; \
1357     asdl_seq *seq = (SEQ); /* avoid variable capture */ \
1358     for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \
1359         TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \
1360         if (!compiler_visit_ ## TYPE((C), elt)) { \
1361             compiler_exit_scope(c); \
1362             return 0; \
1363         } \
1364     } \
1365 }
1366 
1367 static int
compiler_isdocstring(stmt_ty s)1368 compiler_isdocstring(stmt_ty s)
1369 {
1370     if (s->kind != Expr_kind)
1371         return 0;
1372     if (s->v.Expr.value->kind == Str_kind)
1373         return 1;
1374     if (s->v.Expr.value->kind == Constant_kind)
1375         return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value);
1376     return 0;
1377 }
1378 
1379 static int
is_const(expr_ty e)1380 is_const(expr_ty e)
1381 {
1382     switch (e->kind) {
1383     case Constant_kind:
1384     case Num_kind:
1385     case Str_kind:
1386     case Bytes_kind:
1387     case Ellipsis_kind:
1388     case NameConstant_kind:
1389         return 1;
1390     default:
1391         return 0;
1392     }
1393 }
1394 
1395 static PyObject *
get_const_value(expr_ty e)1396 get_const_value(expr_ty e)
1397 {
1398     switch (e->kind) {
1399     case Constant_kind:
1400         return e->v.Constant.value;
1401     case Num_kind:
1402         return e->v.Num.n;
1403     case Str_kind:
1404         return e->v.Str.s;
1405     case Bytes_kind:
1406         return e->v.Bytes.s;
1407     case Ellipsis_kind:
1408         return Py_Ellipsis;
1409     case NameConstant_kind:
1410         return e->v.NameConstant.value;
1411     default:
1412         Py_UNREACHABLE();
1413     }
1414 }
1415 
1416 /* Search if variable annotations are present statically in a block. */
1417 
1418 static int
find_ann(asdl_seq * stmts)1419 find_ann(asdl_seq *stmts)
1420 {
1421     int i, j, res = 0;
1422     stmt_ty st;
1423 
1424     for (i = 0; i < asdl_seq_LEN(stmts); i++) {
1425         st = (stmt_ty)asdl_seq_GET(stmts, i);
1426         switch (st->kind) {
1427         case AnnAssign_kind:
1428             return 1;
1429         case For_kind:
1430             res = find_ann(st->v.For.body) ||
1431                   find_ann(st->v.For.orelse);
1432             break;
1433         case AsyncFor_kind:
1434             res = find_ann(st->v.AsyncFor.body) ||
1435                   find_ann(st->v.AsyncFor.orelse);
1436             break;
1437         case While_kind:
1438             res = find_ann(st->v.While.body) ||
1439                   find_ann(st->v.While.orelse);
1440             break;
1441         case If_kind:
1442             res = find_ann(st->v.If.body) ||
1443                   find_ann(st->v.If.orelse);
1444             break;
1445         case With_kind:
1446             res = find_ann(st->v.With.body);
1447             break;
1448         case AsyncWith_kind:
1449             res = find_ann(st->v.AsyncWith.body);
1450             break;
1451         case Try_kind:
1452             for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) {
1453                 excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
1454                     st->v.Try.handlers, j);
1455                 if (find_ann(handler->v.ExceptHandler.body)) {
1456                     return 1;
1457                 }
1458             }
1459             res = find_ann(st->v.Try.body) ||
1460                   find_ann(st->v.Try.finalbody) ||
1461                   find_ann(st->v.Try.orelse);
1462             break;
1463         default:
1464             res = 0;
1465         }
1466         if (res) {
1467             break;
1468         }
1469     }
1470     return res;
1471 }
1472 
1473 /* Compile a sequence of statements, checking for a docstring
1474    and for annotations. */
1475 
1476 static int
compiler_body(struct compiler * c,asdl_seq * stmts)1477 compiler_body(struct compiler *c, asdl_seq *stmts)
1478 {
1479     int i = 0;
1480     stmt_ty st;
1481 
1482     /* Set current line number to the line number of first statement.
1483        This way line number for SETUP_ANNOTATIONS will always
1484        coincide with the line number of first "real" statement in module.
1485        If body is empy, then lineno will be set later in assemble. */
1486     if (c->u->u_scope_type == COMPILER_SCOPE_MODULE &&
1487         !c->u->u_lineno && asdl_seq_LEN(stmts)) {
1488         st = (stmt_ty)asdl_seq_GET(stmts, 0);
1489         c->u->u_lineno = st->lineno;
1490     }
1491     /* Every annotated class and module should have __annotations__. */
1492     if (find_ann(stmts)) {
1493         ADDOP(c, SETUP_ANNOTATIONS);
1494     }
1495     if (!asdl_seq_LEN(stmts))
1496         return 1;
1497     st = (stmt_ty)asdl_seq_GET(stmts, 0);
1498     /* if not -OO mode, set docstring */
1499     if (compiler_isdocstring(st) && c->c_optimize < 2) {
1500         /* don't generate docstrings if -OO */
1501         i = 1;
1502         VISIT(c, expr, st->v.Expr.value);
1503         if (!compiler_nameop(c, __doc__, Store))
1504             return 0;
1505     }
1506     for (; i < asdl_seq_LEN(stmts); i++)
1507         VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
1508     return 1;
1509 }
1510 
1511 static PyCodeObject *
compiler_mod(struct compiler * c,mod_ty mod)1512 compiler_mod(struct compiler *c, mod_ty mod)
1513 {
1514     PyCodeObject *co;
1515     int addNone = 1;
1516     static PyObject *module;
1517     if (!module) {
1518         module = PyUnicode_InternFromString("<module>");
1519         if (!module)
1520             return NULL;
1521     }
1522     /* Use 0 for firstlineno initially, will fixup in assemble(). */
1523     if (!compiler_enter_scope(c, module, COMPILER_SCOPE_MODULE, mod, 0))
1524         return NULL;
1525     switch (mod->kind) {
1526     case Module_kind:
1527         if (!compiler_body(c, mod->v.Module.body)) {
1528             compiler_exit_scope(c);
1529             return 0;
1530         }
1531         break;
1532     case Interactive_kind:
1533         if (find_ann(mod->v.Interactive.body)) {
1534             ADDOP(c, SETUP_ANNOTATIONS);
1535         }
1536         c->c_interactive = 1;
1537         VISIT_SEQ_IN_SCOPE(c, stmt,
1538                                 mod->v.Interactive.body);
1539         break;
1540     case Expression_kind:
1541         VISIT_IN_SCOPE(c, expr, mod->v.Expression.body);
1542         addNone = 0;
1543         break;
1544     case Suite_kind:
1545         PyErr_SetString(PyExc_SystemError,
1546                         "suite should not be possible");
1547         return 0;
1548     default:
1549         PyErr_Format(PyExc_SystemError,
1550                      "module kind %d should not be possible",
1551                      mod->kind);
1552         return 0;
1553     }
1554     co = assemble(c, addNone);
1555     compiler_exit_scope(c);
1556     return co;
1557 }
1558 
1559 /* The test for LOCAL must come before the test for FREE in order to
1560    handle classes where name is both local and free.  The local var is
1561    a method and the free var is a free var referenced within a method.
1562 */
1563 
1564 static int
get_ref_type(struct compiler * c,PyObject * name)1565 get_ref_type(struct compiler *c, PyObject *name)
1566 {
1567     int scope;
1568     if (c->u->u_scope_type == COMPILER_SCOPE_CLASS &&
1569         _PyUnicode_EqualToASCIIString(name, "__class__"))
1570         return CELL;
1571     scope = PyST_GetScope(c->u->u_ste, name);
1572     if (scope == 0) {
1573         char buf[350];
1574         PyOS_snprintf(buf, sizeof(buf),
1575                       "unknown scope for %.100s in %.100s(%s)\n"
1576                       "symbols: %s\nlocals: %s\nglobals: %s",
1577                       PyUnicode_AsUTF8(name),
1578                       PyUnicode_AsUTF8(c->u->u_name),
1579                       PyUnicode_AsUTF8(PyObject_Repr(c->u->u_ste->ste_id)),
1580                       PyUnicode_AsUTF8(PyObject_Repr(c->u->u_ste->ste_symbols)),
1581                       PyUnicode_AsUTF8(PyObject_Repr(c->u->u_varnames)),
1582                       PyUnicode_AsUTF8(PyObject_Repr(c->u->u_names))
1583         );
1584         Py_FatalError(buf);
1585     }
1586 
1587     return scope;
1588 }
1589 
1590 static int
compiler_lookup_arg(PyObject * dict,PyObject * name)1591 compiler_lookup_arg(PyObject *dict, PyObject *name)
1592 {
1593     PyObject *k, *v;
1594     k = _PyCode_ConstantKey(name);
1595     if (k == NULL)
1596         return -1;
1597     v = PyDict_GetItem(dict, k);
1598     Py_DECREF(k);
1599     if (v == NULL)
1600         return -1;
1601     return PyLong_AS_LONG(v);
1602 }
1603 
1604 static int
compiler_make_closure(struct compiler * c,PyCodeObject * co,Py_ssize_t flags,PyObject * qualname)1605 compiler_make_closure(struct compiler *c, PyCodeObject *co, Py_ssize_t flags, PyObject *qualname)
1606 {
1607     Py_ssize_t i, free = PyCode_GetNumFree(co);
1608     if (qualname == NULL)
1609         qualname = co->co_name;
1610 
1611     if (free) {
1612         for (i = 0; i < free; ++i) {
1613             /* Bypass com_addop_varname because it will generate
1614                LOAD_DEREF but LOAD_CLOSURE is needed.
1615             */
1616             PyObject *name = PyTuple_GET_ITEM(co->co_freevars, i);
1617             int arg, reftype;
1618 
1619             /* Special case: If a class contains a method with a
1620                free variable that has the same name as a method,
1621                the name will be considered free *and* local in the
1622                class.  It should be handled by the closure, as
1623                well as by the normal name loookup logic.
1624             */
1625             reftype = get_ref_type(c, name);
1626             if (reftype == CELL)
1627                 arg = compiler_lookup_arg(c->u->u_cellvars, name);
1628             else /* (reftype == FREE) */
1629                 arg = compiler_lookup_arg(c->u->u_freevars, name);
1630             if (arg == -1) {
1631                 fprintf(stderr,
1632                     "lookup %s in %s %d %d\n"
1633                     "freevars of %s: %s\n",
1634                     PyUnicode_AsUTF8(PyObject_Repr(name)),
1635                     PyUnicode_AsUTF8(c->u->u_name),
1636                     reftype, arg,
1637                     PyUnicode_AsUTF8(co->co_name),
1638                     PyUnicode_AsUTF8(PyObject_Repr(co->co_freevars)));
1639                 Py_FatalError("compiler_make_closure()");
1640             }
1641             ADDOP_I(c, LOAD_CLOSURE, arg);
1642         }
1643         flags |= 0x08;
1644         ADDOP_I(c, BUILD_TUPLE, free);
1645     }
1646     ADDOP_O(c, LOAD_CONST, (PyObject*)co, consts);
1647     ADDOP_O(c, LOAD_CONST, qualname, consts);
1648     ADDOP_I(c, MAKE_FUNCTION, flags);
1649     return 1;
1650 }
1651 
1652 static int
compiler_decorators(struct compiler * c,asdl_seq * decos)1653 compiler_decorators(struct compiler *c, asdl_seq* decos)
1654 {
1655     int i;
1656 
1657     if (!decos)
1658         return 1;
1659 
1660     for (i = 0; i < asdl_seq_LEN(decos); i++) {
1661         VISIT(c, expr, (expr_ty)asdl_seq_GET(decos, i));
1662     }
1663     return 1;
1664 }
1665 
1666 static int
compiler_visit_kwonlydefaults(struct compiler * c,asdl_seq * kwonlyargs,asdl_seq * kw_defaults)1667 compiler_visit_kwonlydefaults(struct compiler *c, asdl_seq *kwonlyargs,
1668                               asdl_seq *kw_defaults)
1669 {
1670     /* Push a dict of keyword-only default values.
1671 
1672        Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed.
1673        */
1674     int i;
1675     PyObject *keys = NULL;
1676 
1677     for (i = 0; i < asdl_seq_LEN(kwonlyargs); i++) {
1678         arg_ty arg = asdl_seq_GET(kwonlyargs, i);
1679         expr_ty default_ = asdl_seq_GET(kw_defaults, i);
1680         if (default_) {
1681             PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg);
1682             if (!mangled) {
1683                 goto error;
1684             }
1685             if (keys == NULL) {
1686                 keys = PyList_New(1);
1687                 if (keys == NULL) {
1688                     Py_DECREF(mangled);
1689                     return 0;
1690                 }
1691                 PyList_SET_ITEM(keys, 0, mangled);
1692             }
1693             else {
1694                 int res = PyList_Append(keys, mangled);
1695                 Py_DECREF(mangled);
1696                 if (res == -1) {
1697                     goto error;
1698                 }
1699             }
1700             if (!compiler_visit_expr(c, default_)) {
1701                 goto error;
1702             }
1703         }
1704     }
1705     if (keys != NULL) {
1706         Py_ssize_t default_count = PyList_GET_SIZE(keys);
1707         PyObject *keys_tuple = PyList_AsTuple(keys);
1708         Py_DECREF(keys);
1709         if (keys_tuple == NULL) {
1710             return 0;
1711         }
1712         ADDOP_N(c, LOAD_CONST, keys_tuple, consts);
1713         ADDOP_I(c, BUILD_CONST_KEY_MAP, default_count);
1714         assert(default_count > 0);
1715         return 1;
1716     }
1717     else {
1718         return -1;
1719     }
1720 
1721 error:
1722     Py_XDECREF(keys);
1723     return 0;
1724 }
1725 
1726 static int
compiler_visit_annexpr(struct compiler * c,expr_ty annotation)1727 compiler_visit_annexpr(struct compiler *c, expr_ty annotation)
1728 {
1729     PyObject *ann_as_str;
1730     ann_as_str = _PyAST_ExprAsUnicode(annotation);
1731     if (!ann_as_str) {
1732         return 0;
1733     }
1734     ADDOP_N(c, LOAD_CONST, ann_as_str, consts);
1735     return 1;
1736 }
1737 
1738 static int
compiler_visit_argannotation(struct compiler * c,identifier id,expr_ty annotation,PyObject * names)1739 compiler_visit_argannotation(struct compiler *c, identifier id,
1740     expr_ty annotation, PyObject *names)
1741 {
1742     if (annotation) {
1743         PyObject *mangled;
1744         if (c->c_future->ff_features & CO_FUTURE_ANNOTATIONS) {
1745             VISIT(c, annexpr, annotation)
1746         }
1747         else {
1748             VISIT(c, expr, annotation);
1749         }
1750         mangled = _Py_Mangle(c->u->u_private, id);
1751         if (!mangled)
1752             return 0;
1753         if (PyList_Append(names, mangled) < 0) {
1754             Py_DECREF(mangled);
1755             return 0;
1756         }
1757         Py_DECREF(mangled);
1758     }
1759     return 1;
1760 }
1761 
1762 static int
compiler_visit_argannotations(struct compiler * c,asdl_seq * args,PyObject * names)1763 compiler_visit_argannotations(struct compiler *c, asdl_seq* args,
1764                               PyObject *names)
1765 {
1766     int i;
1767     for (i = 0; i < asdl_seq_LEN(args); i++) {
1768         arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
1769         if (!compiler_visit_argannotation(
1770                         c,
1771                         arg->arg,
1772                         arg->annotation,
1773                         names))
1774             return 0;
1775     }
1776     return 1;
1777 }
1778 
1779 static int
compiler_visit_annotations(struct compiler * c,arguments_ty args,expr_ty returns)1780 compiler_visit_annotations(struct compiler *c, arguments_ty args,
1781                            expr_ty returns)
1782 {
1783     /* Push arg annotation dict.
1784        The expressions are evaluated out-of-order wrt the source code.
1785 
1786        Return 0 on error, -1 if no dict pushed, 1 if a dict is pushed.
1787        */
1788     static identifier return_str;
1789     PyObject *names;
1790     Py_ssize_t len;
1791     names = PyList_New(0);
1792     if (!names)
1793         return 0;
1794 
1795     if (!compiler_visit_argannotations(c, args->args, names))
1796         goto error;
1797     if (args->vararg && args->vararg->annotation &&
1798         !compiler_visit_argannotation(c, args->vararg->arg,
1799                                      args->vararg->annotation, names))
1800         goto error;
1801     if (!compiler_visit_argannotations(c, args->kwonlyargs, names))
1802         goto error;
1803     if (args->kwarg && args->kwarg->annotation &&
1804         !compiler_visit_argannotation(c, args->kwarg->arg,
1805                                      args->kwarg->annotation, names))
1806         goto error;
1807 
1808     if (!return_str) {
1809         return_str = PyUnicode_InternFromString("return");
1810         if (!return_str)
1811             goto error;
1812     }
1813     if (!compiler_visit_argannotation(c, return_str, returns, names)) {
1814         goto error;
1815     }
1816 
1817     len = PyList_GET_SIZE(names);
1818     if (len) {
1819         PyObject *keytuple = PyList_AsTuple(names);
1820         Py_DECREF(names);
1821         if (keytuple == NULL) {
1822             return 0;
1823         }
1824         ADDOP_N(c, LOAD_CONST, keytuple, consts);
1825         ADDOP_I(c, BUILD_CONST_KEY_MAP, len);
1826         return 1;
1827     }
1828     else {
1829         Py_DECREF(names);
1830         return -1;
1831     }
1832 
1833 error:
1834     Py_DECREF(names);
1835     return 0;
1836 }
1837 
1838 static int
compiler_visit_defaults(struct compiler * c,arguments_ty args)1839 compiler_visit_defaults(struct compiler *c, arguments_ty args)
1840 {
1841     VISIT_SEQ(c, expr, args->defaults);
1842     ADDOP_I(c, BUILD_TUPLE, asdl_seq_LEN(args->defaults));
1843     return 1;
1844 }
1845 
1846 static Py_ssize_t
compiler_default_arguments(struct compiler * c,arguments_ty args)1847 compiler_default_arguments(struct compiler *c, arguments_ty args)
1848 {
1849     Py_ssize_t funcflags = 0;
1850     if (args->defaults && asdl_seq_LEN(args->defaults) > 0) {
1851         if (!compiler_visit_defaults(c, args))
1852             return -1;
1853         funcflags |= 0x01;
1854     }
1855     if (args->kwonlyargs) {
1856         int res = compiler_visit_kwonlydefaults(c, args->kwonlyargs,
1857                                                 args->kw_defaults);
1858         if (res == 0) {
1859             return -1;
1860         }
1861         else if (res > 0) {
1862             funcflags |= 0x02;
1863         }
1864     }
1865     return funcflags;
1866 }
1867 
1868 static int
compiler_function(struct compiler * c,stmt_ty s,int is_async)1869 compiler_function(struct compiler *c, stmt_ty s, int is_async)
1870 {
1871     PyCodeObject *co;
1872     PyObject *qualname, *first_const = Py_None;
1873     arguments_ty args;
1874     expr_ty returns;
1875     identifier name;
1876     asdl_seq* decos;
1877     asdl_seq *body;
1878     stmt_ty st;
1879     Py_ssize_t i, funcflags;
1880     int docstring;
1881     int annotations;
1882     int scope_type;
1883 
1884     if (is_async) {
1885         assert(s->kind == AsyncFunctionDef_kind);
1886 
1887         args = s->v.AsyncFunctionDef.args;
1888         returns = s->v.AsyncFunctionDef.returns;
1889         decos = s->v.AsyncFunctionDef.decorator_list;
1890         name = s->v.AsyncFunctionDef.name;
1891         body = s->v.AsyncFunctionDef.body;
1892 
1893         scope_type = COMPILER_SCOPE_ASYNC_FUNCTION;
1894     } else {
1895         assert(s->kind == FunctionDef_kind);
1896 
1897         args = s->v.FunctionDef.args;
1898         returns = s->v.FunctionDef.returns;
1899         decos = s->v.FunctionDef.decorator_list;
1900         name = s->v.FunctionDef.name;
1901         body = s->v.FunctionDef.body;
1902 
1903         scope_type = COMPILER_SCOPE_FUNCTION;
1904     }
1905 
1906     if (!compiler_decorators(c, decos))
1907         return 0;
1908 
1909     funcflags = compiler_default_arguments(c, args);
1910     if (funcflags == -1) {
1911         return 0;
1912     }
1913 
1914     annotations = compiler_visit_annotations(c, args, returns);
1915     if (annotations == 0) {
1916         return 0;
1917     }
1918     else if (annotations > 0) {
1919         funcflags |= 0x04;
1920     }
1921 
1922     if (!compiler_enter_scope(c, name, scope_type, (void *)s, s->lineno)) {
1923         return 0;
1924     }
1925 
1926     st = (stmt_ty)asdl_seq_GET(body, 0);
1927     docstring = compiler_isdocstring(st);
1928     if (docstring && c->c_optimize < 2) {
1929         if (st->v.Expr.value->kind == Constant_kind)
1930             first_const = st->v.Expr.value->v.Constant.value;
1931         else
1932             first_const = st->v.Expr.value->v.Str.s;
1933     }
1934     if (compiler_add_o(c, c->u->u_consts, first_const) < 0) {
1935         compiler_exit_scope(c);
1936         return 0;
1937     }
1938 
1939     c->u->u_argcount = asdl_seq_LEN(args->args);
1940     c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
1941     VISIT_SEQ_IN_SCOPE(c, stmt, body);
1942     co = assemble(c, 1);
1943     qualname = c->u->u_qualname;
1944     Py_INCREF(qualname);
1945     compiler_exit_scope(c);
1946     if (co == NULL) {
1947         Py_XDECREF(qualname);
1948         Py_XDECREF(co);
1949         return 0;
1950     }
1951 
1952     compiler_make_closure(c, co, funcflags, qualname);
1953     Py_DECREF(qualname);
1954     Py_DECREF(co);
1955 
1956     /* decorators */
1957     for (i = 0; i < asdl_seq_LEN(decos); i++) {
1958         ADDOP_I(c, CALL_FUNCTION, 1);
1959     }
1960 
1961     return compiler_nameop(c, name, Store);
1962 }
1963 
1964 static int
compiler_class(struct compiler * c,stmt_ty s)1965 compiler_class(struct compiler *c, stmt_ty s)
1966 {
1967     PyCodeObject *co;
1968     PyObject *str;
1969     int i;
1970     asdl_seq* decos = s->v.ClassDef.decorator_list;
1971 
1972     if (!compiler_decorators(c, decos))
1973         return 0;
1974 
1975     /* ultimately generate code for:
1976          <name> = __build_class__(<func>, <name>, *<bases>, **<keywords>)
1977        where:
1978          <func> is a function/closure created from the class body;
1979             it has a single argument (__locals__) where the dict
1980             (or MutableSequence) representing the locals is passed
1981          <name> is the class name
1982          <bases> is the positional arguments and *varargs argument
1983          <keywords> is the keyword arguments and **kwds argument
1984        This borrows from compiler_call.
1985     */
1986 
1987     /* 1. compile the class body into a code object */
1988     if (!compiler_enter_scope(c, s->v.ClassDef.name,
1989                               COMPILER_SCOPE_CLASS, (void *)s, s->lineno))
1990         return 0;
1991     /* this block represents what we do in the new scope */
1992     {
1993         /* use the class name for name mangling */
1994         Py_INCREF(s->v.ClassDef.name);
1995         Py_XSETREF(c->u->u_private, s->v.ClassDef.name);
1996         /* load (global) __name__ ... */
1997         str = PyUnicode_InternFromString("__name__");
1998         if (!str || !compiler_nameop(c, str, Load)) {
1999             Py_XDECREF(str);
2000             compiler_exit_scope(c);
2001             return 0;
2002         }
2003         Py_DECREF(str);
2004         /* ... and store it as __module__ */
2005         str = PyUnicode_InternFromString("__module__");
2006         if (!str || !compiler_nameop(c, str, Store)) {
2007             Py_XDECREF(str);
2008             compiler_exit_scope(c);
2009             return 0;
2010         }
2011         Py_DECREF(str);
2012         assert(c->u->u_qualname);
2013         ADDOP_O(c, LOAD_CONST, c->u->u_qualname, consts);
2014         str = PyUnicode_InternFromString("__qualname__");
2015         if (!str || !compiler_nameop(c, str, Store)) {
2016             Py_XDECREF(str);
2017             compiler_exit_scope(c);
2018             return 0;
2019         }
2020         Py_DECREF(str);
2021         /* compile the body proper */
2022         if (!compiler_body(c, s->v.ClassDef.body)) {
2023             compiler_exit_scope(c);
2024             return 0;
2025         }
2026         /* Return __classcell__ if it is referenced, otherwise return None */
2027         if (c->u->u_ste->ste_needs_class_closure) {
2028             /* Store __classcell__ into class namespace & return it */
2029             str = PyUnicode_InternFromString("__class__");
2030             if (str == NULL) {
2031                 compiler_exit_scope(c);
2032                 return 0;
2033             }
2034             i = compiler_lookup_arg(c->u->u_cellvars, str);
2035             Py_DECREF(str);
2036             if (i < 0) {
2037                 compiler_exit_scope(c);
2038                 return 0;
2039             }
2040             assert(i == 0);
2041 
2042             ADDOP_I(c, LOAD_CLOSURE, i);
2043             ADDOP(c, DUP_TOP);
2044             str = PyUnicode_InternFromString("__classcell__");
2045             if (!str || !compiler_nameop(c, str, Store)) {
2046                 Py_XDECREF(str);
2047                 compiler_exit_scope(c);
2048                 return 0;
2049             }
2050             Py_DECREF(str);
2051         }
2052         else {
2053             /* No methods referenced __class__, so just return None */
2054             assert(PyDict_GET_SIZE(c->u->u_cellvars) == 0);
2055             ADDOP_O(c, LOAD_CONST, Py_None, consts);
2056         }
2057         ADDOP_IN_SCOPE(c, RETURN_VALUE);
2058         /* create the code object */
2059         co = assemble(c, 1);
2060     }
2061     /* leave the new scope */
2062     compiler_exit_scope(c);
2063     if (co == NULL)
2064         return 0;
2065 
2066     /* 2. load the 'build_class' function */
2067     ADDOP(c, LOAD_BUILD_CLASS);
2068 
2069     /* 3. load a function (or closure) made from the code object */
2070     compiler_make_closure(c, co, 0, NULL);
2071     Py_DECREF(co);
2072 
2073     /* 4. load class name */
2074     ADDOP_O(c, LOAD_CONST, s->v.ClassDef.name, consts);
2075 
2076     /* 5. generate the rest of the code for the call */
2077     if (!compiler_call_helper(c, 2,
2078                               s->v.ClassDef.bases,
2079                               s->v.ClassDef.keywords))
2080         return 0;
2081 
2082     /* 6. apply decorators */
2083     for (i = 0; i < asdl_seq_LEN(decos); i++) {
2084         ADDOP_I(c, CALL_FUNCTION, 1);
2085     }
2086 
2087     /* 7. store into <name> */
2088     if (!compiler_nameop(c, s->v.ClassDef.name, Store))
2089         return 0;
2090     return 1;
2091 }
2092 
2093 static int
cmpop(cmpop_ty op)2094 cmpop(cmpop_ty op)
2095 {
2096     switch (op) {
2097     case Eq:
2098         return PyCmp_EQ;
2099     case NotEq:
2100         return PyCmp_NE;
2101     case Lt:
2102         return PyCmp_LT;
2103     case LtE:
2104         return PyCmp_LE;
2105     case Gt:
2106         return PyCmp_GT;
2107     case GtE:
2108         return PyCmp_GE;
2109     case Is:
2110         return PyCmp_IS;
2111     case IsNot:
2112         return PyCmp_IS_NOT;
2113     case In:
2114         return PyCmp_IN;
2115     case NotIn:
2116         return PyCmp_NOT_IN;
2117     default:
2118         return PyCmp_BAD;
2119     }
2120 }
2121 
2122 static int
compiler_jump_if(struct compiler * c,expr_ty e,basicblock * next,int cond)2123 compiler_jump_if(struct compiler *c, expr_ty e, basicblock *next, int cond)
2124 {
2125     switch (e->kind) {
2126     case UnaryOp_kind:
2127         if (e->v.UnaryOp.op == Not)
2128             return compiler_jump_if(c, e->v.UnaryOp.operand, next, !cond);
2129         /* fallback to general implementation */
2130         break;
2131     case BoolOp_kind: {
2132         asdl_seq *s = e->v.BoolOp.values;
2133         Py_ssize_t i, n = asdl_seq_LEN(s) - 1;
2134         assert(n >= 0);
2135         int cond2 = e->v.BoolOp.op == Or;
2136         basicblock *next2 = next;
2137         if (!cond2 != !cond) {
2138             next2 = compiler_new_block(c);
2139             if (next2 == NULL)
2140                 return 0;
2141         }
2142         for (i = 0; i < n; ++i) {
2143             if (!compiler_jump_if(c, (expr_ty)asdl_seq_GET(s, i), next2, cond2))
2144                 return 0;
2145         }
2146         if (!compiler_jump_if(c, (expr_ty)asdl_seq_GET(s, n), next, cond))
2147             return 0;
2148         if (next2 != next)
2149             compiler_use_next_block(c, next2);
2150         return 1;
2151     }
2152     case IfExp_kind: {
2153         basicblock *end, *next2;
2154         end = compiler_new_block(c);
2155         if (end == NULL)
2156             return 0;
2157         next2 = compiler_new_block(c);
2158         if (next2 == NULL)
2159             return 0;
2160         if (!compiler_jump_if(c, e->v.IfExp.test, next2, 0))
2161             return 0;
2162         if (!compiler_jump_if(c, e->v.IfExp.body, next, cond))
2163             return 0;
2164         ADDOP_JREL(c, JUMP_FORWARD, end);
2165         compiler_use_next_block(c, next2);
2166         if (!compiler_jump_if(c, e->v.IfExp.orelse, next, cond))
2167             return 0;
2168         compiler_use_next_block(c, end);
2169         return 1;
2170     }
2171     case Compare_kind: {
2172         Py_ssize_t i, n = asdl_seq_LEN(e->v.Compare.ops) - 1;
2173         if (n > 0) {
2174             basicblock *cleanup = compiler_new_block(c);
2175             if (cleanup == NULL)
2176                 return 0;
2177             VISIT(c, expr, e->v.Compare.left);
2178             for (i = 0; i < n; i++) {
2179                 VISIT(c, expr,
2180                     (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i));
2181                 ADDOP(c, DUP_TOP);
2182                 ADDOP(c, ROT_THREE);
2183                 ADDOP_I(c, COMPARE_OP,
2184                     cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, i))));
2185                 ADDOP_JABS(c, POP_JUMP_IF_FALSE, cleanup);
2186                 NEXT_BLOCK(c);
2187             }
2188             VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n));
2189             ADDOP_I(c, COMPARE_OP,
2190                    cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, n))));
2191             ADDOP_JABS(c, cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next);
2192             basicblock *end = compiler_new_block(c);
2193             if (end == NULL)
2194                 return 0;
2195             ADDOP_JREL(c, JUMP_FORWARD, end);
2196             compiler_use_next_block(c, cleanup);
2197             ADDOP(c, POP_TOP);
2198             if (!cond) {
2199                 ADDOP_JREL(c, JUMP_FORWARD, next);
2200             }
2201             compiler_use_next_block(c, end);
2202             return 1;
2203         }
2204         /* fallback to general implementation */
2205         break;
2206     }
2207     default:
2208         /* fallback to general implementation */
2209         break;
2210     }
2211 
2212     /* general implementation */
2213     VISIT(c, expr, e);
2214     ADDOP_JABS(c, cond ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE, next);
2215     return 1;
2216 }
2217 
2218 static int
compiler_ifexp(struct compiler * c,expr_ty e)2219 compiler_ifexp(struct compiler *c, expr_ty e)
2220 {
2221     basicblock *end, *next;
2222 
2223     assert(e->kind == IfExp_kind);
2224     end = compiler_new_block(c);
2225     if (end == NULL)
2226         return 0;
2227     next = compiler_new_block(c);
2228     if (next == NULL)
2229         return 0;
2230     if (!compiler_jump_if(c, e->v.IfExp.test, next, 0))
2231         return 0;
2232     VISIT(c, expr, e->v.IfExp.body);
2233     ADDOP_JREL(c, JUMP_FORWARD, end);
2234     compiler_use_next_block(c, next);
2235     VISIT(c, expr, e->v.IfExp.orelse);
2236     compiler_use_next_block(c, end);
2237     return 1;
2238 }
2239 
2240 static int
compiler_lambda(struct compiler * c,expr_ty e)2241 compiler_lambda(struct compiler *c, expr_ty e)
2242 {
2243     PyCodeObject *co;
2244     PyObject *qualname;
2245     static identifier name;
2246     Py_ssize_t funcflags;
2247     arguments_ty args = e->v.Lambda.args;
2248     assert(e->kind == Lambda_kind);
2249 
2250     if (!name) {
2251         name = PyUnicode_InternFromString("<lambda>");
2252         if (!name)
2253             return 0;
2254     }
2255 
2256     funcflags = compiler_default_arguments(c, args);
2257     if (funcflags == -1) {
2258         return 0;
2259     }
2260 
2261     if (!compiler_enter_scope(c, name, COMPILER_SCOPE_LAMBDA,
2262                               (void *)e, e->lineno))
2263         return 0;
2264 
2265     /* Make None the first constant, so the lambda can't have a
2266        docstring. */
2267     if (compiler_add_o(c, c->u->u_consts, Py_None) < 0)
2268         return 0;
2269 
2270     c->u->u_argcount = asdl_seq_LEN(args->args);
2271     c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
2272     VISIT_IN_SCOPE(c, expr, e->v.Lambda.body);
2273     if (c->u->u_ste->ste_generator) {
2274         co = assemble(c, 0);
2275     }
2276     else {
2277         ADDOP_IN_SCOPE(c, RETURN_VALUE);
2278         co = assemble(c, 1);
2279     }
2280     qualname = c->u->u_qualname;
2281     Py_INCREF(qualname);
2282     compiler_exit_scope(c);
2283     if (co == NULL)
2284         return 0;
2285 
2286     compiler_make_closure(c, co, funcflags, qualname);
2287     Py_DECREF(qualname);
2288     Py_DECREF(co);
2289 
2290     return 1;
2291 }
2292 
2293 static int
compiler_if(struct compiler * c,stmt_ty s)2294 compiler_if(struct compiler *c, stmt_ty s)
2295 {
2296     basicblock *end, *next;
2297     int constant;
2298     assert(s->kind == If_kind);
2299     end = compiler_new_block(c);
2300     if (end == NULL)
2301         return 0;
2302 
2303     constant = expr_constant(s->v.If.test);
2304     /* constant = 0: "if 0"
2305      * constant = 1: "if 1", "if 2", ...
2306      * constant = -1: rest */
2307     if (constant == 0) {
2308         if (s->v.If.orelse)
2309             VISIT_SEQ(c, stmt, s->v.If.orelse);
2310     } else if (constant == 1) {
2311         VISIT_SEQ(c, stmt, s->v.If.body);
2312     } else {
2313         if (asdl_seq_LEN(s->v.If.orelse)) {
2314             next = compiler_new_block(c);
2315             if (next == NULL)
2316                 return 0;
2317         }
2318         else
2319             next = end;
2320         if (!compiler_jump_if(c, s->v.If.test, next, 0))
2321             return 0;
2322         VISIT_SEQ(c, stmt, s->v.If.body);
2323         if (asdl_seq_LEN(s->v.If.orelse)) {
2324             ADDOP_JREL(c, JUMP_FORWARD, end);
2325             compiler_use_next_block(c, next);
2326             VISIT_SEQ(c, stmt, s->v.If.orelse);
2327         }
2328     }
2329     compiler_use_next_block(c, end);
2330     return 1;
2331 }
2332 
2333 static int
compiler_for(struct compiler * c,stmt_ty s)2334 compiler_for(struct compiler *c, stmt_ty s)
2335 {
2336     basicblock *start, *cleanup, *end;
2337 
2338     start = compiler_new_block(c);
2339     cleanup = compiler_new_block(c);
2340     end = compiler_new_block(c);
2341     if (start == NULL || end == NULL || cleanup == NULL)
2342         return 0;
2343     ADDOP_JREL(c, SETUP_LOOP, end);
2344     if (!compiler_push_fblock(c, LOOP, start))
2345         return 0;
2346     VISIT(c, expr, s->v.For.iter);
2347     ADDOP(c, GET_ITER);
2348     compiler_use_next_block(c, start);
2349     ADDOP_JREL(c, FOR_ITER, cleanup);
2350     VISIT(c, expr, s->v.For.target);
2351     VISIT_SEQ(c, stmt, s->v.For.body);
2352     ADDOP_JABS(c, JUMP_ABSOLUTE, start);
2353     compiler_use_next_block(c, cleanup);
2354     ADDOP(c, POP_BLOCK);
2355     compiler_pop_fblock(c, LOOP, start);
2356     VISIT_SEQ(c, stmt, s->v.For.orelse);
2357     compiler_use_next_block(c, end);
2358     return 1;
2359 }
2360 
2361 
2362 static int
compiler_async_for(struct compiler * c,stmt_ty s)2363 compiler_async_for(struct compiler *c, stmt_ty s)
2364 {
2365     _Py_IDENTIFIER(StopAsyncIteration);
2366 
2367     basicblock *try, *except, *end, *after_try, *try_cleanup,
2368                *after_loop_else;
2369 
2370     if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
2371         return compiler_error(c, "'async for' outside async function");
2372     }
2373 
2374     PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration);
2375     if (stop_aiter_error == NULL) {
2376         return 0;
2377     }
2378 
2379     try = compiler_new_block(c);
2380     except = compiler_new_block(c);
2381     end = compiler_new_block(c);
2382     after_try = compiler_new_block(c);
2383     try_cleanup = compiler_new_block(c);
2384     after_loop_else = compiler_new_block(c);
2385 
2386     if (try == NULL || except == NULL || end == NULL
2387             || after_try == NULL || try_cleanup == NULL
2388             || after_loop_else == NULL)
2389         return 0;
2390 
2391     ADDOP_JREL(c, SETUP_LOOP, end);
2392     if (!compiler_push_fblock(c, LOOP, try))
2393         return 0;
2394 
2395     VISIT(c, expr, s->v.AsyncFor.iter);
2396     ADDOP(c, GET_AITER);
2397 
2398     compiler_use_next_block(c, try);
2399 
2400 
2401     ADDOP_JREL(c, SETUP_EXCEPT, except);
2402     if (!compiler_push_fblock(c, EXCEPT, try))
2403         return 0;
2404 
2405     ADDOP(c, GET_ANEXT);
2406     ADDOP_O(c, LOAD_CONST, Py_None, consts);
2407     ADDOP(c, YIELD_FROM);
2408     VISIT(c, expr, s->v.AsyncFor.target);
2409     ADDOP(c, POP_BLOCK);
2410     compiler_pop_fblock(c, EXCEPT, try);
2411     ADDOP_JREL(c, JUMP_FORWARD, after_try);
2412 
2413 
2414     compiler_use_next_block(c, except);
2415     ADDOP(c, DUP_TOP);
2416     ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names);
2417     ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH);
2418     ADDOP_JABS(c, POP_JUMP_IF_TRUE, try_cleanup);
2419     ADDOP(c, END_FINALLY);
2420 
2421     compiler_use_next_block(c, after_try);
2422     VISIT_SEQ(c, stmt, s->v.AsyncFor.body);
2423     ADDOP_JABS(c, JUMP_ABSOLUTE, try);
2424 
2425     compiler_use_next_block(c, try_cleanup);
2426     ADDOP(c, POP_TOP);
2427     ADDOP(c, POP_TOP);
2428     ADDOP(c, POP_TOP);
2429     ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */
2430     ADDOP(c, POP_TOP); /* for correct calculation of stack effect */
2431     ADDOP(c, POP_BLOCK); /* for SETUP_LOOP */
2432     compiler_pop_fblock(c, LOOP, try);
2433 
2434     compiler_use_next_block(c, after_loop_else);
2435     VISIT_SEQ(c, stmt, s->v.For.orelse);
2436 
2437     compiler_use_next_block(c, end);
2438 
2439     return 1;
2440 }
2441 
2442 static int
compiler_while(struct compiler * c,stmt_ty s)2443 compiler_while(struct compiler *c, stmt_ty s)
2444 {
2445     basicblock *loop, *orelse, *end, *anchor = NULL;
2446     int constant = expr_constant(s->v.While.test);
2447 
2448     if (constant == 0) {
2449         if (s->v.While.orelse)
2450             VISIT_SEQ(c, stmt, s->v.While.orelse);
2451         return 1;
2452     }
2453     loop = compiler_new_block(c);
2454     end = compiler_new_block(c);
2455     if (constant == -1) {
2456         anchor = compiler_new_block(c);
2457         if (anchor == NULL)
2458             return 0;
2459     }
2460     if (loop == NULL || end == NULL)
2461         return 0;
2462     if (s->v.While.orelse) {
2463         orelse = compiler_new_block(c);
2464         if (orelse == NULL)
2465             return 0;
2466     }
2467     else
2468         orelse = NULL;
2469 
2470     ADDOP_JREL(c, SETUP_LOOP, end);
2471     compiler_use_next_block(c, loop);
2472     if (!compiler_push_fblock(c, LOOP, loop))
2473         return 0;
2474     if (constant == -1) {
2475         if (!compiler_jump_if(c, s->v.While.test, anchor, 0))
2476             return 0;
2477     }
2478     VISIT_SEQ(c, stmt, s->v.While.body);
2479     ADDOP_JABS(c, JUMP_ABSOLUTE, loop);
2480 
2481     /* XXX should the two POP instructions be in a separate block
2482        if there is no else clause ?
2483     */
2484 
2485     if (constant == -1)
2486         compiler_use_next_block(c, anchor);
2487     ADDOP(c, POP_BLOCK);
2488     compiler_pop_fblock(c, LOOP, loop);
2489     if (orelse != NULL) /* what if orelse is just pass? */
2490         VISIT_SEQ(c, stmt, s->v.While.orelse);
2491     compiler_use_next_block(c, end);
2492 
2493     return 1;
2494 }
2495 
2496 static int
compiler_continue(struct compiler * c)2497 compiler_continue(struct compiler *c)
2498 {
2499     static const char LOOP_ERROR_MSG[] = "'continue' not properly in loop";
2500     static const char IN_FINALLY_ERROR_MSG[] =
2501                     "'continue' not supported inside 'finally' clause";
2502     int i;
2503 
2504     if (!c->u->u_nfblocks)
2505         return compiler_error(c, LOOP_ERROR_MSG);
2506     i = c->u->u_nfblocks - 1;
2507     switch (c->u->u_fblock[i].fb_type) {
2508     case LOOP:
2509         ADDOP_JABS(c, JUMP_ABSOLUTE, c->u->u_fblock[i].fb_block);
2510         break;
2511     case EXCEPT:
2512     case FINALLY_TRY:
2513         while (--i >= 0 && c->u->u_fblock[i].fb_type != LOOP) {
2514             /* Prevent continue anywhere under a finally
2515                   even if hidden in a sub-try or except. */
2516             if (c->u->u_fblock[i].fb_type == FINALLY_END)
2517                 return compiler_error(c, IN_FINALLY_ERROR_MSG);
2518         }
2519         if (i == -1)
2520             return compiler_error(c, LOOP_ERROR_MSG);
2521         ADDOP_JABS(c, CONTINUE_LOOP, c->u->u_fblock[i].fb_block);
2522         break;
2523     case FINALLY_END:
2524         return compiler_error(c, IN_FINALLY_ERROR_MSG);
2525     }
2526 
2527     return 1;
2528 }
2529 
2530 /* Code generated for "try: <body> finally: <finalbody>" is as follows:
2531 
2532         SETUP_FINALLY           L
2533         <code for body>
2534         POP_BLOCK
2535         LOAD_CONST              <None>
2536     L:          <code for finalbody>
2537         END_FINALLY
2538 
2539    The special instructions use the block stack.  Each block
2540    stack entry contains the instruction that created it (here
2541    SETUP_FINALLY), the level of the value stack at the time the
2542    block stack entry was created, and a label (here L).
2543 
2544    SETUP_FINALLY:
2545     Pushes the current value stack level and the label
2546     onto the block stack.
2547    POP_BLOCK:
2548     Pops en entry from the block stack, and pops the value
2549     stack until its level is the same as indicated on the
2550     block stack.  (The label is ignored.)
2551    END_FINALLY:
2552     Pops a variable number of entries from the *value* stack
2553     and re-raises the exception they specify.  The number of
2554     entries popped depends on the (pseudo) exception type.
2555 
2556    The block stack is unwound when an exception is raised:
2557    when a SETUP_FINALLY entry is found, the exception is pushed
2558    onto the value stack (and the exception condition is cleared),
2559    and the interpreter jumps to the label gotten from the block
2560    stack.
2561 */
2562 
2563 static int
compiler_try_finally(struct compiler * c,stmt_ty s)2564 compiler_try_finally(struct compiler *c, stmt_ty s)
2565 {
2566     basicblock *body, *end;
2567     body = compiler_new_block(c);
2568     end = compiler_new_block(c);
2569     if (body == NULL || end == NULL)
2570         return 0;
2571 
2572     ADDOP_JREL(c, SETUP_FINALLY, end);
2573     compiler_use_next_block(c, body);
2574     if (!compiler_push_fblock(c, FINALLY_TRY, body))
2575         return 0;
2576     if (s->v.Try.handlers && asdl_seq_LEN(s->v.Try.handlers)) {
2577         if (!compiler_try_except(c, s))
2578             return 0;
2579     }
2580     else {
2581         VISIT_SEQ(c, stmt, s->v.Try.body);
2582     }
2583     ADDOP(c, POP_BLOCK);
2584     compiler_pop_fblock(c, FINALLY_TRY, body);
2585 
2586     ADDOP_O(c, LOAD_CONST, Py_None, consts);
2587     compiler_use_next_block(c, end);
2588     if (!compiler_push_fblock(c, FINALLY_END, end))
2589         return 0;
2590     VISIT_SEQ(c, stmt, s->v.Try.finalbody);
2591     ADDOP(c, END_FINALLY);
2592     compiler_pop_fblock(c, FINALLY_END, end);
2593 
2594     return 1;
2595 }
2596 
2597 /*
2598    Code generated for "try: S except E1 as V1: S1 except E2 as V2: S2 ...":
2599    (The contents of the value stack is shown in [], with the top
2600    at the right; 'tb' is trace-back info, 'val' the exception's
2601    associated value, and 'exc' the exception.)
2602 
2603    Value stack          Label   Instruction     Argument
2604    []                           SETUP_EXCEPT    L1
2605    []                           <code for S>
2606    []                           POP_BLOCK
2607    []                           JUMP_FORWARD    L0
2608 
2609    [tb, val, exc]       L1:     DUP                             )
2610    [tb, val, exc, exc]          <evaluate E1>                   )
2611    [tb, val, exc, exc, E1]      COMPARE_OP      EXC_MATCH       ) only if E1
2612    [tb, val, exc, 1-or-0]       POP_JUMP_IF_FALSE       L2      )
2613    [tb, val, exc]               POP
2614    [tb, val]                    <assign to V1>  (or POP if no V1)
2615    [tb]                         POP
2616    []                           <code for S1>
2617                                 JUMP_FORWARD    L0
2618 
2619    [tb, val, exc]       L2:     DUP
2620    .............................etc.......................
2621 
2622    [tb, val, exc]       Ln+1:   END_FINALLY     # re-raise exception
2623 
2624    []                   L0:     <next statement>
2625 
2626    Of course, parts are not generated if Vi or Ei is not present.
2627 */
2628 static int
compiler_try_except(struct compiler * c,stmt_ty s)2629 compiler_try_except(struct compiler *c, stmt_ty s)
2630 {
2631     basicblock *body, *orelse, *except, *end;
2632     Py_ssize_t i, n;
2633 
2634     body = compiler_new_block(c);
2635     except = compiler_new_block(c);
2636     orelse = compiler_new_block(c);
2637     end = compiler_new_block(c);
2638     if (body == NULL || except == NULL || orelse == NULL || end == NULL)
2639         return 0;
2640     ADDOP_JREL(c, SETUP_EXCEPT, except);
2641     compiler_use_next_block(c, body);
2642     if (!compiler_push_fblock(c, EXCEPT, body))
2643         return 0;
2644     VISIT_SEQ(c, stmt, s->v.Try.body);
2645     ADDOP(c, POP_BLOCK);
2646     compiler_pop_fblock(c, EXCEPT, body);
2647     ADDOP_JREL(c, JUMP_FORWARD, orelse);
2648     n = asdl_seq_LEN(s->v.Try.handlers);
2649     compiler_use_next_block(c, except);
2650     for (i = 0; i < n; i++) {
2651         excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
2652             s->v.Try.handlers, i);
2653         if (!handler->v.ExceptHandler.type && i < n-1)
2654             return compiler_error(c, "default 'except:' must be last");
2655         c->u->u_lineno_set = 0;
2656         c->u->u_lineno = handler->lineno;
2657         c->u->u_col_offset = handler->col_offset;
2658         except = compiler_new_block(c);
2659         if (except == NULL)
2660             return 0;
2661         if (handler->v.ExceptHandler.type) {
2662             ADDOP(c, DUP_TOP);
2663             VISIT(c, expr, handler->v.ExceptHandler.type);
2664             ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH);
2665             ADDOP_JABS(c, POP_JUMP_IF_FALSE, except);
2666         }
2667         ADDOP(c, POP_TOP);
2668         if (handler->v.ExceptHandler.name) {
2669             basicblock *cleanup_end, *cleanup_body;
2670 
2671             cleanup_end = compiler_new_block(c);
2672             cleanup_body = compiler_new_block(c);
2673             if (cleanup_end == NULL || cleanup_body == NULL) {
2674                 return 0;
2675             }
2676 
2677             compiler_nameop(c, handler->v.ExceptHandler.name, Store);
2678             ADDOP(c, POP_TOP);
2679 
2680             /*
2681               try:
2682                   # body
2683               except type as name:
2684                   try:
2685                       # body
2686                   finally:
2687                       name = None
2688                       del name
2689             */
2690 
2691             /* second try: */
2692             ADDOP_JREL(c, SETUP_FINALLY, cleanup_end);
2693             compiler_use_next_block(c, cleanup_body);
2694             if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
2695                 return 0;
2696 
2697             /* second # body */
2698             VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
2699             ADDOP(c, POP_BLOCK);
2700             compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
2701 
2702             /* finally: */
2703             ADDOP_O(c, LOAD_CONST, Py_None, consts);
2704             compiler_use_next_block(c, cleanup_end);
2705             if (!compiler_push_fblock(c, FINALLY_END, cleanup_end))
2706                 return 0;
2707 
2708             /* name = None */
2709             ADDOP_O(c, LOAD_CONST, Py_None, consts);
2710             compiler_nameop(c, handler->v.ExceptHandler.name, Store);
2711 
2712             /* del name */
2713             compiler_nameop(c, handler->v.ExceptHandler.name, Del);
2714 
2715             ADDOP(c, END_FINALLY);
2716             ADDOP(c, POP_EXCEPT);
2717             compiler_pop_fblock(c, FINALLY_END, cleanup_end);
2718         }
2719         else {
2720             basicblock *cleanup_body;
2721 
2722             cleanup_body = compiler_new_block(c);
2723             if (!cleanup_body)
2724                 return 0;
2725 
2726             ADDOP(c, POP_TOP);
2727             ADDOP(c, POP_TOP);
2728             compiler_use_next_block(c, cleanup_body);
2729             if (!compiler_push_fblock(c, FINALLY_TRY, cleanup_body))
2730                 return 0;
2731             VISIT_SEQ(c, stmt, handler->v.ExceptHandler.body);
2732             ADDOP(c, POP_EXCEPT);
2733             compiler_pop_fblock(c, FINALLY_TRY, cleanup_body);
2734         }
2735         ADDOP_JREL(c, JUMP_FORWARD, end);
2736         compiler_use_next_block(c, except);
2737     }
2738     ADDOP(c, END_FINALLY);
2739     compiler_use_next_block(c, orelse);
2740     VISIT_SEQ(c, stmt, s->v.Try.orelse);
2741     compiler_use_next_block(c, end);
2742     return 1;
2743 }
2744 
2745 static int
compiler_try(struct compiler * c,stmt_ty s)2746 compiler_try(struct compiler *c, stmt_ty s) {
2747     if (s->v.Try.finalbody && asdl_seq_LEN(s->v.Try.finalbody))
2748         return compiler_try_finally(c, s);
2749     else
2750         return compiler_try_except(c, s);
2751 }
2752 
2753 
2754 static int
compiler_import_as(struct compiler * c,identifier name,identifier asname)2755 compiler_import_as(struct compiler *c, identifier name, identifier asname)
2756 {
2757     /* The IMPORT_NAME opcode was already generated.  This function
2758        merely needs to bind the result to a name.
2759 
2760        If there is a dot in name, we need to split it and emit a
2761        IMPORT_FROM for each name.
2762     */
2763     Py_ssize_t len = PyUnicode_GET_LENGTH(name);
2764     Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, len, 1);
2765     if (dot == -2)
2766         return 0;
2767     if (dot != -1) {
2768         /* Consume the base module name to get the first attribute */
2769         while (1) {
2770             Py_ssize_t pos = dot + 1;
2771             PyObject *attr;
2772             dot = PyUnicode_FindChar(name, '.', pos, len, 1);
2773             if (dot == -2)
2774                 return 0;
2775             attr = PyUnicode_Substring(name, pos, (dot != -1) ? dot : len);
2776             if (!attr)
2777                 return 0;
2778             ADDOP_N(c, IMPORT_FROM, attr, names);
2779             if (dot == -1) {
2780                 break;
2781             }
2782             ADDOP(c, ROT_TWO);
2783             ADDOP(c, POP_TOP);
2784         }
2785         if (!compiler_nameop(c, asname, Store)) {
2786             return 0;
2787         }
2788         ADDOP(c, POP_TOP);
2789         return 1;
2790     }
2791     return compiler_nameop(c, asname, Store);
2792 }
2793 
2794 static int
compiler_import(struct compiler * c,stmt_ty s)2795 compiler_import(struct compiler *c, stmt_ty s)
2796 {
2797     /* The Import node stores a module name like a.b.c as a single
2798        string.  This is convenient for all cases except
2799          import a.b.c as d
2800        where we need to parse that string to extract the individual
2801        module names.
2802        XXX Perhaps change the representation to make this case simpler?
2803      */
2804     Py_ssize_t i, n = asdl_seq_LEN(s->v.Import.names);
2805 
2806     for (i = 0; i < n; i++) {
2807         alias_ty alias = (alias_ty)asdl_seq_GET(s->v.Import.names, i);
2808         int r;
2809 
2810         ADDOP_O(c, LOAD_CONST, _PyLong_Zero, consts);
2811         ADDOP_O(c, LOAD_CONST, Py_None, consts);
2812         ADDOP_NAME(c, IMPORT_NAME, alias->name, names);
2813 
2814         if (alias->asname) {
2815             r = compiler_import_as(c, alias->name, alias->asname);
2816             if (!r)
2817                 return r;
2818         }
2819         else {
2820             identifier tmp = alias->name;
2821             Py_ssize_t dot = PyUnicode_FindChar(
2822                 alias->name, '.', 0, PyUnicode_GET_LENGTH(alias->name), 1);
2823             if (dot != -1) {
2824                 tmp = PyUnicode_Substring(alias->name, 0, dot);
2825                 if (tmp == NULL)
2826                     return 0;
2827             }
2828             r = compiler_nameop(c, tmp, Store);
2829             if (dot != -1) {
2830                 Py_DECREF(tmp);
2831             }
2832             if (!r)
2833                 return r;
2834         }
2835     }
2836     return 1;
2837 }
2838 
2839 static int
compiler_from_import(struct compiler * c,stmt_ty s)2840 compiler_from_import(struct compiler *c, stmt_ty s)
2841 {
2842     Py_ssize_t i, n = asdl_seq_LEN(s->v.ImportFrom.names);
2843     PyObject *level, *names;
2844     static PyObject *empty_string;
2845 
2846     if (!empty_string) {
2847         empty_string = PyUnicode_FromString("");
2848         if (!empty_string)
2849             return 0;
2850     }
2851 
2852     level = PyLong_FromLong(s->v.ImportFrom.level);
2853     if (!level) {
2854         return 0;
2855     }
2856     ADDOP_N(c, LOAD_CONST, level, consts);
2857 
2858     names = PyTuple_New(n);
2859     if (!names)
2860         return 0;
2861 
2862     /* build up the names */
2863     for (i = 0; i < n; i++) {
2864         alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i);
2865         Py_INCREF(alias->name);
2866         PyTuple_SET_ITEM(names, i, alias->name);
2867     }
2868 
2869     if (s->lineno > c->c_future->ff_lineno && s->v.ImportFrom.module &&
2870         _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) {
2871         Py_DECREF(names);
2872         return compiler_error(c, "from __future__ imports must occur "
2873                               "at the beginning of the file");
2874     }
2875     ADDOP_N(c, LOAD_CONST, names, consts);
2876 
2877     if (s->v.ImportFrom.module) {
2878         ADDOP_NAME(c, IMPORT_NAME, s->v.ImportFrom.module, names);
2879     }
2880     else {
2881         ADDOP_NAME(c, IMPORT_NAME, empty_string, names);
2882     }
2883     for (i = 0; i < n; i++) {
2884         alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i);
2885         identifier store_name;
2886 
2887         if (i == 0 && PyUnicode_READ_CHAR(alias->name, 0) == '*') {
2888             assert(n == 1);
2889             ADDOP(c, IMPORT_STAR);
2890             return 1;
2891         }
2892 
2893         ADDOP_NAME(c, IMPORT_FROM, alias->name, names);
2894         store_name = alias->name;
2895         if (alias->asname)
2896             store_name = alias->asname;
2897 
2898         if (!compiler_nameop(c, store_name, Store)) {
2899             return 0;
2900         }
2901     }
2902     /* remove imported module */
2903     ADDOP(c, POP_TOP);
2904     return 1;
2905 }
2906 
2907 static int
compiler_assert(struct compiler * c,stmt_ty s)2908 compiler_assert(struct compiler *c, stmt_ty s)
2909 {
2910     static PyObject *assertion_error = NULL;
2911     basicblock *end;
2912     PyObject* msg;
2913 
2914     if (c->c_optimize)
2915         return 1;
2916     if (assertion_error == NULL) {
2917         assertion_error = PyUnicode_InternFromString("AssertionError");
2918         if (assertion_error == NULL)
2919             return 0;
2920     }
2921     if (s->v.Assert.test->kind == Tuple_kind &&
2922         asdl_seq_LEN(s->v.Assert.test->v.Tuple.elts) > 0) {
2923         msg = PyUnicode_FromString("assertion is always true, "
2924                                    "perhaps remove parentheses?");
2925         if (msg == NULL)
2926             return 0;
2927         if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg,
2928                                      c->c_filename, c->u->u_lineno,
2929                                      NULL, NULL) == -1) {
2930             Py_DECREF(msg);
2931             return 0;
2932         }
2933         Py_DECREF(msg);
2934     }
2935     end = compiler_new_block(c);
2936     if (end == NULL)
2937         return 0;
2938     if (!compiler_jump_if(c, s->v.Assert.test, end, 1))
2939         return 0;
2940     ADDOP_O(c, LOAD_GLOBAL, assertion_error, names);
2941     if (s->v.Assert.msg) {
2942         VISIT(c, expr, s->v.Assert.msg);
2943         ADDOP_I(c, CALL_FUNCTION, 1);
2944     }
2945     ADDOP_I(c, RAISE_VARARGS, 1);
2946     compiler_use_next_block(c, end);
2947     return 1;
2948 }
2949 
2950 static int
compiler_visit_stmt_expr(struct compiler * c,expr_ty value)2951 compiler_visit_stmt_expr(struct compiler *c, expr_ty value)
2952 {
2953     if (c->c_interactive && c->c_nestlevel <= 1) {
2954         VISIT(c, expr, value);
2955         ADDOP(c, PRINT_EXPR);
2956         return 1;
2957     }
2958 
2959     if (is_const(value)) {
2960         /* ignore constant statement */
2961         return 1;
2962     }
2963 
2964     VISIT(c, expr, value);
2965     ADDOP(c, POP_TOP);
2966     return 1;
2967 }
2968 
2969 static int
compiler_visit_stmt(struct compiler * c,stmt_ty s)2970 compiler_visit_stmt(struct compiler *c, stmt_ty s)
2971 {
2972     Py_ssize_t i, n;
2973 
2974     /* Always assign a lineno to the next instruction for a stmt. */
2975     c->u->u_lineno = s->lineno;
2976     c->u->u_col_offset = s->col_offset;
2977     c->u->u_lineno_set = 0;
2978 
2979     switch (s->kind) {
2980     case FunctionDef_kind:
2981         return compiler_function(c, s, 0);
2982     case ClassDef_kind:
2983         return compiler_class(c, s);
2984     case Return_kind:
2985         if (c->u->u_ste->ste_type != FunctionBlock)
2986             return compiler_error(c, "'return' outside function");
2987         if (s->v.Return.value) {
2988             if (c->u->u_ste->ste_coroutine && c->u->u_ste->ste_generator)
2989                 return compiler_error(
2990                     c, "'return' with value in async generator");
2991             VISIT(c, expr, s->v.Return.value);
2992         }
2993         else
2994             ADDOP_O(c, LOAD_CONST, Py_None, consts);
2995         ADDOP(c, RETURN_VALUE);
2996         break;
2997     case Delete_kind:
2998         VISIT_SEQ(c, expr, s->v.Delete.targets)
2999         break;
3000     case Assign_kind:
3001         n = asdl_seq_LEN(s->v.Assign.targets);
3002         VISIT(c, expr, s->v.Assign.value);
3003         for (i = 0; i < n; i++) {
3004             if (i < n - 1)
3005                 ADDOP(c, DUP_TOP);
3006             VISIT(c, expr,
3007                   (expr_ty)asdl_seq_GET(s->v.Assign.targets, i));
3008         }
3009         break;
3010     case AugAssign_kind:
3011         return compiler_augassign(c, s);
3012     case AnnAssign_kind:
3013         return compiler_annassign(c, s);
3014     case For_kind:
3015         return compiler_for(c, s);
3016     case While_kind:
3017         return compiler_while(c, s);
3018     case If_kind:
3019         return compiler_if(c, s);
3020     case Raise_kind:
3021         n = 0;
3022         if (s->v.Raise.exc) {
3023             VISIT(c, expr, s->v.Raise.exc);
3024             n++;
3025             if (s->v.Raise.cause) {
3026                 VISIT(c, expr, s->v.Raise.cause);
3027                 n++;
3028             }
3029         }
3030         ADDOP_I(c, RAISE_VARARGS, (int)n);
3031         break;
3032     case Try_kind:
3033         return compiler_try(c, s);
3034     case Assert_kind:
3035         return compiler_assert(c, s);
3036     case Import_kind:
3037         return compiler_import(c, s);
3038     case ImportFrom_kind:
3039         return compiler_from_import(c, s);
3040     case Global_kind:
3041     case Nonlocal_kind:
3042         break;
3043     case Expr_kind:
3044         return compiler_visit_stmt_expr(c, s->v.Expr.value);
3045     case Pass_kind:
3046         break;
3047     case Break_kind:
3048         if (!compiler_in_loop(c))
3049             return compiler_error(c, "'break' outside loop");
3050         ADDOP(c, BREAK_LOOP);
3051         break;
3052     case Continue_kind:
3053         return compiler_continue(c);
3054     case With_kind:
3055         return compiler_with(c, s, 0);
3056     case AsyncFunctionDef_kind:
3057         return compiler_function(c, s, 1);
3058     case AsyncWith_kind:
3059         return compiler_async_with(c, s, 0);
3060     case AsyncFor_kind:
3061         return compiler_async_for(c, s);
3062     }
3063 
3064     return 1;
3065 }
3066 
3067 static int
unaryop(unaryop_ty op)3068 unaryop(unaryop_ty op)
3069 {
3070     switch (op) {
3071     case Invert:
3072         return UNARY_INVERT;
3073     case Not:
3074         return UNARY_NOT;
3075     case UAdd:
3076         return UNARY_POSITIVE;
3077     case USub:
3078         return UNARY_NEGATIVE;
3079     default:
3080         PyErr_Format(PyExc_SystemError,
3081             "unary op %d should not be possible", op);
3082         return 0;
3083     }
3084 }
3085 
3086 static int
binop(struct compiler * c,operator_ty op)3087 binop(struct compiler *c, operator_ty op)
3088 {
3089     switch (op) {
3090     case Add:
3091         return BINARY_ADD;
3092     case Sub:
3093         return BINARY_SUBTRACT;
3094     case Mult:
3095         return BINARY_MULTIPLY;
3096     case MatMult:
3097         return BINARY_MATRIX_MULTIPLY;
3098     case Div:
3099         return BINARY_TRUE_DIVIDE;
3100     case Mod:
3101         return BINARY_MODULO;
3102     case Pow:
3103         return BINARY_POWER;
3104     case LShift:
3105         return BINARY_LSHIFT;
3106     case RShift:
3107         return BINARY_RSHIFT;
3108     case BitOr:
3109         return BINARY_OR;
3110     case BitXor:
3111         return BINARY_XOR;
3112     case BitAnd:
3113         return BINARY_AND;
3114     case FloorDiv:
3115         return BINARY_FLOOR_DIVIDE;
3116     default:
3117         PyErr_Format(PyExc_SystemError,
3118             "binary op %d should not be possible", op);
3119         return 0;
3120     }
3121 }
3122 
3123 static int
inplace_binop(struct compiler * c,operator_ty op)3124 inplace_binop(struct compiler *c, operator_ty op)
3125 {
3126     switch (op) {
3127     case Add:
3128         return INPLACE_ADD;
3129     case Sub:
3130         return INPLACE_SUBTRACT;
3131     case Mult:
3132         return INPLACE_MULTIPLY;
3133     case MatMult:
3134         return INPLACE_MATRIX_MULTIPLY;
3135     case Div:
3136         return INPLACE_TRUE_DIVIDE;
3137     case Mod:
3138         return INPLACE_MODULO;
3139     case Pow:
3140         return INPLACE_POWER;
3141     case LShift:
3142         return INPLACE_LSHIFT;
3143     case RShift:
3144         return INPLACE_RSHIFT;
3145     case BitOr:
3146         return INPLACE_OR;
3147     case BitXor:
3148         return INPLACE_XOR;
3149     case BitAnd:
3150         return INPLACE_AND;
3151     case FloorDiv:
3152         return INPLACE_FLOOR_DIVIDE;
3153     default:
3154         PyErr_Format(PyExc_SystemError,
3155             "inplace binary op %d should not be possible", op);
3156         return 0;
3157     }
3158 }
3159 
3160 static int
compiler_nameop(struct compiler * c,identifier name,expr_context_ty ctx)3161 compiler_nameop(struct compiler *c, identifier name, expr_context_ty ctx)
3162 {
3163     int op, scope;
3164     Py_ssize_t arg;
3165     enum { OP_FAST, OP_GLOBAL, OP_DEREF, OP_NAME } optype;
3166 
3167     PyObject *dict = c->u->u_names;
3168     PyObject *mangled;
3169     /* XXX AugStore isn't used anywhere! */
3170 
3171     assert(!_PyUnicode_EqualToASCIIString(name, "None") &&
3172            !_PyUnicode_EqualToASCIIString(name, "True") &&
3173            !_PyUnicode_EqualToASCIIString(name, "False"));
3174 
3175     mangled = _Py_Mangle(c->u->u_private, name);
3176     if (!mangled)
3177         return 0;
3178 
3179     op = 0;
3180     optype = OP_NAME;
3181     scope = PyST_GetScope(c->u->u_ste, mangled);
3182     switch (scope) {
3183     case FREE:
3184         dict = c->u->u_freevars;
3185         optype = OP_DEREF;
3186         break;
3187     case CELL:
3188         dict = c->u->u_cellvars;
3189         optype = OP_DEREF;
3190         break;
3191     case LOCAL:
3192         if (c->u->u_ste->ste_type == FunctionBlock)
3193             optype = OP_FAST;
3194         break;
3195     case GLOBAL_IMPLICIT:
3196         if (c->u->u_ste->ste_type == FunctionBlock)
3197             optype = OP_GLOBAL;
3198         break;
3199     case GLOBAL_EXPLICIT:
3200         optype = OP_GLOBAL;
3201         break;
3202     default:
3203         /* scope can be 0 */
3204         break;
3205     }
3206 
3207     /* XXX Leave assert here, but handle __doc__ and the like better */
3208     assert(scope || PyUnicode_READ_CHAR(name, 0) == '_');
3209 
3210     switch (optype) {
3211     case OP_DEREF:
3212         switch (ctx) {
3213         case Load:
3214             op = (c->u->u_ste->ste_type == ClassBlock) ? LOAD_CLASSDEREF : LOAD_DEREF;
3215             break;
3216         case Store: op = STORE_DEREF; break;
3217         case AugLoad:
3218         case AugStore:
3219             break;
3220         case Del: op = DELETE_DEREF; break;
3221         case Param:
3222         default:
3223             PyErr_SetString(PyExc_SystemError,
3224                             "param invalid for deref variable");
3225             return 0;
3226         }
3227         break;
3228     case OP_FAST:
3229         switch (ctx) {
3230         case Load: op = LOAD_FAST; break;
3231         case Store: op = STORE_FAST; break;
3232         case Del: op = DELETE_FAST; break;
3233         case AugLoad:
3234         case AugStore:
3235             break;
3236         case Param:
3237         default:
3238             PyErr_SetString(PyExc_SystemError,
3239                             "param invalid for local variable");
3240             return 0;
3241         }
3242         ADDOP_N(c, op, mangled, varnames);
3243         return 1;
3244     case OP_GLOBAL:
3245         switch (ctx) {
3246         case Load: op = LOAD_GLOBAL; break;
3247         case Store: op = STORE_GLOBAL; break;
3248         case Del: op = DELETE_GLOBAL; break;
3249         case AugLoad:
3250         case AugStore:
3251             break;
3252         case Param:
3253         default:
3254             PyErr_SetString(PyExc_SystemError,
3255                             "param invalid for global variable");
3256             return 0;
3257         }
3258         break;
3259     case OP_NAME:
3260         switch (ctx) {
3261         case Load: op = LOAD_NAME; break;
3262         case Store: op = STORE_NAME; break;
3263         case Del: op = DELETE_NAME; break;
3264         case AugLoad:
3265         case AugStore:
3266             break;
3267         case Param:
3268         default:
3269             PyErr_SetString(PyExc_SystemError,
3270                             "param invalid for name variable");
3271             return 0;
3272         }
3273         break;
3274     }
3275 
3276     assert(op);
3277     arg = compiler_add_o(c, dict, mangled);
3278     Py_DECREF(mangled);
3279     if (arg < 0)
3280         return 0;
3281     return compiler_addop_i(c, op, arg);
3282 }
3283 
3284 static int
compiler_boolop(struct compiler * c,expr_ty e)3285 compiler_boolop(struct compiler *c, expr_ty e)
3286 {
3287     basicblock *end;
3288     int jumpi;
3289     Py_ssize_t i, n;
3290     asdl_seq *s;
3291 
3292     assert(e->kind == BoolOp_kind);
3293     if (e->v.BoolOp.op == And)
3294         jumpi = JUMP_IF_FALSE_OR_POP;
3295     else
3296         jumpi = JUMP_IF_TRUE_OR_POP;
3297     end = compiler_new_block(c);
3298     if (end == NULL)
3299         return 0;
3300     s = e->v.BoolOp.values;
3301     n = asdl_seq_LEN(s) - 1;
3302     assert(n >= 0);
3303     for (i = 0; i < n; ++i) {
3304         VISIT(c, expr, (expr_ty)asdl_seq_GET(s, i));
3305         ADDOP_JABS(c, jumpi, end);
3306     }
3307     VISIT(c, expr, (expr_ty)asdl_seq_GET(s, n));
3308     compiler_use_next_block(c, end);
3309     return 1;
3310 }
3311 
3312 static int
starunpack_helper(struct compiler * c,asdl_seq * elts,int single_op,int inner_op,int outer_op)3313 starunpack_helper(struct compiler *c, asdl_seq *elts,
3314                   int single_op, int inner_op, int outer_op)
3315 {
3316     Py_ssize_t n = asdl_seq_LEN(elts);
3317     Py_ssize_t i, nsubitems = 0, nseen = 0;
3318     for (i = 0; i < n; i++) {
3319         expr_ty elt = asdl_seq_GET(elts, i);
3320         if (elt->kind == Starred_kind) {
3321             if (nseen) {
3322                 ADDOP_I(c, inner_op, nseen);
3323                 nseen = 0;
3324                 nsubitems++;
3325             }
3326             VISIT(c, expr, elt->v.Starred.value);
3327             nsubitems++;
3328         }
3329         else {
3330             VISIT(c, expr, elt);
3331             nseen++;
3332         }
3333     }
3334     if (nsubitems) {
3335         if (nseen) {
3336             ADDOP_I(c, inner_op, nseen);
3337             nsubitems++;
3338         }
3339         ADDOP_I(c, outer_op, nsubitems);
3340     }
3341     else
3342         ADDOP_I(c, single_op, nseen);
3343     return 1;
3344 }
3345 
3346 static int
assignment_helper(struct compiler * c,asdl_seq * elts)3347 assignment_helper(struct compiler *c, asdl_seq *elts)
3348 {
3349     Py_ssize_t n = asdl_seq_LEN(elts);
3350     Py_ssize_t i;
3351     int seen_star = 0;
3352     for (i = 0; i < n; i++) {
3353         expr_ty elt = asdl_seq_GET(elts, i);
3354         if (elt->kind == Starred_kind && !seen_star) {
3355             if ((i >= (1 << 8)) ||
3356                 (n-i-1 >= (INT_MAX >> 8)))
3357                 return compiler_error(c,
3358                     "too many expressions in "
3359                     "star-unpacking assignment");
3360             ADDOP_I(c, UNPACK_EX, (i + ((n-i-1) << 8)));
3361             seen_star = 1;
3362             asdl_seq_SET(elts, i, elt->v.Starred.value);
3363         }
3364         else if (elt->kind == Starred_kind) {
3365             return compiler_error(c,
3366                 "two starred expressions in assignment");
3367         }
3368     }
3369     if (!seen_star) {
3370         ADDOP_I(c, UNPACK_SEQUENCE, n);
3371     }
3372     VISIT_SEQ(c, expr, elts);
3373     return 1;
3374 }
3375 
3376 static int
compiler_list(struct compiler * c,expr_ty e)3377 compiler_list(struct compiler *c, expr_ty e)
3378 {
3379     asdl_seq *elts = e->v.List.elts;
3380     if (e->v.List.ctx == Store) {
3381         return assignment_helper(c, elts);
3382     }
3383     else if (e->v.List.ctx == Load) {
3384         return starunpack_helper(c, elts,
3385                                  BUILD_LIST, BUILD_TUPLE, BUILD_LIST_UNPACK);
3386     }
3387     else
3388         VISIT_SEQ(c, expr, elts);
3389     return 1;
3390 }
3391 
3392 static int
compiler_tuple(struct compiler * c,expr_ty e)3393 compiler_tuple(struct compiler *c, expr_ty e)
3394 {
3395     asdl_seq *elts = e->v.Tuple.elts;
3396     if (e->v.Tuple.ctx == Store) {
3397         return assignment_helper(c, elts);
3398     }
3399     else if (e->v.Tuple.ctx == Load) {
3400         return starunpack_helper(c, elts,
3401                                  BUILD_TUPLE, BUILD_TUPLE, BUILD_TUPLE_UNPACK);
3402     }
3403     else
3404         VISIT_SEQ(c, expr, elts);
3405     return 1;
3406 }
3407 
3408 static int
compiler_set(struct compiler * c,expr_ty e)3409 compiler_set(struct compiler *c, expr_ty e)
3410 {
3411     return starunpack_helper(c, e->v.Set.elts, BUILD_SET,
3412                              BUILD_SET, BUILD_SET_UNPACK);
3413 }
3414 
3415 static int
are_all_items_const(asdl_seq * seq,Py_ssize_t begin,Py_ssize_t end)3416 are_all_items_const(asdl_seq *seq, Py_ssize_t begin, Py_ssize_t end)
3417 {
3418     Py_ssize_t i;
3419     for (i = begin; i < end; i++) {
3420         expr_ty key = (expr_ty)asdl_seq_GET(seq, i);
3421         if (key == NULL || !is_const(key))
3422             return 0;
3423     }
3424     return 1;
3425 }
3426 
3427 static int
compiler_subdict(struct compiler * c,expr_ty e,Py_ssize_t begin,Py_ssize_t end)3428 compiler_subdict(struct compiler *c, expr_ty e, Py_ssize_t begin, Py_ssize_t end)
3429 {
3430     Py_ssize_t i, n = end - begin;
3431     PyObject *keys, *key;
3432     if (n > 1 && are_all_items_const(e->v.Dict.keys, begin, end)) {
3433         for (i = begin; i < end; i++) {
3434             VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
3435         }
3436         keys = PyTuple_New(n);
3437         if (keys == NULL) {
3438             return 0;
3439         }
3440         for (i = begin; i < end; i++) {
3441             key = get_const_value((expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
3442             Py_INCREF(key);
3443             PyTuple_SET_ITEM(keys, i - begin, key);
3444         }
3445         ADDOP_N(c, LOAD_CONST, keys, consts);
3446         ADDOP_I(c, BUILD_CONST_KEY_MAP, n);
3447     }
3448     else {
3449         for (i = begin; i < end; i++) {
3450             VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
3451             VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
3452         }
3453         ADDOP_I(c, BUILD_MAP, n);
3454     }
3455     return 1;
3456 }
3457 
3458 static int
compiler_dict(struct compiler * c,expr_ty e)3459 compiler_dict(struct compiler *c, expr_ty e)
3460 {
3461     Py_ssize_t i, n, elements;
3462     int containers;
3463     int is_unpacking = 0;
3464     n = asdl_seq_LEN(e->v.Dict.values);
3465     containers = 0;
3466     elements = 0;
3467     for (i = 0; i < n; i++) {
3468         is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL;
3469         if (elements == 0xFFFF || (elements && is_unpacking)) {
3470             if (!compiler_subdict(c, e, i - elements, i))
3471                 return 0;
3472             containers++;
3473             elements = 0;
3474         }
3475         if (is_unpacking) {
3476             VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
3477             containers++;
3478         }
3479         else {
3480             elements++;
3481         }
3482     }
3483     if (elements || containers == 0) {
3484         if (!compiler_subdict(c, e, n - elements, n))
3485             return 0;
3486         containers++;
3487     }
3488     /* If there is more than one dict, they need to be merged into a new
3489      * dict.  If there is one dict and it's an unpacking, then it needs
3490      * to be copied into a new dict." */
3491     if (containers > 1 || is_unpacking) {
3492         ADDOP_I(c, BUILD_MAP_UNPACK, containers);
3493     }
3494     return 1;
3495 }
3496 
3497 static int
compiler_compare(struct compiler * c,expr_ty e)3498 compiler_compare(struct compiler *c, expr_ty e)
3499 {
3500     Py_ssize_t i, n;
3501 
3502     VISIT(c, expr, e->v.Compare.left);
3503     assert(asdl_seq_LEN(e->v.Compare.ops) > 0);
3504     n = asdl_seq_LEN(e->v.Compare.ops) - 1;
3505     if (n == 0) {
3506         VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, 0));
3507         ADDOP_I(c, COMPARE_OP,
3508             cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, 0))));
3509     }
3510     else {
3511         basicblock *cleanup = compiler_new_block(c);
3512         if (cleanup == NULL)
3513             return 0;
3514         for (i = 0; i < n; i++) {
3515             VISIT(c, expr,
3516                 (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i));
3517             ADDOP(c, DUP_TOP);
3518             ADDOP(c, ROT_THREE);
3519             ADDOP_I(c, COMPARE_OP,
3520                 cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, i))));
3521             ADDOP_JABS(c, JUMP_IF_FALSE_OR_POP, cleanup);
3522             NEXT_BLOCK(c);
3523         }
3524         VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n));
3525         ADDOP_I(c, COMPARE_OP,
3526             cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, n))));
3527         basicblock *end = compiler_new_block(c);
3528         if (end == NULL)
3529             return 0;
3530         ADDOP_JREL(c, JUMP_FORWARD, end);
3531         compiler_use_next_block(c, cleanup);
3532         ADDOP(c, ROT_TWO);
3533         ADDOP(c, POP_TOP);
3534         compiler_use_next_block(c, end);
3535     }
3536     return 1;
3537 }
3538 
3539 static int
maybe_optimize_method_call(struct compiler * c,expr_ty e)3540 maybe_optimize_method_call(struct compiler *c, expr_ty e)
3541 {
3542     Py_ssize_t argsl, i;
3543     expr_ty meth = e->v.Call.func;
3544     asdl_seq *args = e->v.Call.args;
3545 
3546     /* Check that the call node is an attribute access, and that
3547        the call doesn't have keyword parameters. */
3548     if (meth->kind != Attribute_kind || meth->v.Attribute.ctx != Load ||
3549             asdl_seq_LEN(e->v.Call.keywords))
3550         return -1;
3551 
3552     /* Check that there are no *varargs types of arguments. */
3553     argsl = asdl_seq_LEN(args);
3554     for (i = 0; i < argsl; i++) {
3555         expr_ty elt = asdl_seq_GET(args, i);
3556         if (elt->kind == Starred_kind) {
3557             return -1;
3558         }
3559     }
3560 
3561     /* Alright, we can optimize the code. */
3562     VISIT(c, expr, meth->v.Attribute.value);
3563     ADDOP_NAME(c, LOAD_METHOD, meth->v.Attribute.attr, names);
3564     VISIT_SEQ(c, expr, e->v.Call.args);
3565     ADDOP_I(c, CALL_METHOD, asdl_seq_LEN(e->v.Call.args));
3566     return 1;
3567 }
3568 
3569 static int
compiler_call(struct compiler * c,expr_ty e)3570 compiler_call(struct compiler *c, expr_ty e)
3571 {
3572     if (maybe_optimize_method_call(c, e) > 0)
3573         return 1;
3574 
3575     VISIT(c, expr, e->v.Call.func);
3576     return compiler_call_helper(c, 0,
3577                                 e->v.Call.args,
3578                                 e->v.Call.keywords);
3579 }
3580 
3581 static int
compiler_joined_str(struct compiler * c,expr_ty e)3582 compiler_joined_str(struct compiler *c, expr_ty e)
3583 {
3584     VISIT_SEQ(c, expr, e->v.JoinedStr.values);
3585     if (asdl_seq_LEN(e->v.JoinedStr.values) != 1)
3586         ADDOP_I(c, BUILD_STRING, asdl_seq_LEN(e->v.JoinedStr.values));
3587     return 1;
3588 }
3589 
3590 /* Used to implement f-strings. Format a single value. */
3591 static int
compiler_formatted_value(struct compiler * c,expr_ty e)3592 compiler_formatted_value(struct compiler *c, expr_ty e)
3593 {
3594     /* Our oparg encodes 2 pieces of information: the conversion
3595        character, and whether or not a format_spec was provided.
3596 
3597        Convert the conversion char to 2 bits:
3598        None: 000  0x0  FVC_NONE
3599        !s  : 001  0x1  FVC_STR
3600        !r  : 010  0x2  FVC_REPR
3601        !a  : 011  0x3  FVC_ASCII
3602 
3603        next bit is whether or not we have a format spec:
3604        yes : 100  0x4
3605        no  : 000  0x0
3606     */
3607 
3608     int oparg;
3609 
3610     /* Evaluate the expression to be formatted. */
3611     VISIT(c, expr, e->v.FormattedValue.value);
3612 
3613     switch (e->v.FormattedValue.conversion) {
3614     case 's': oparg = FVC_STR;   break;
3615     case 'r': oparg = FVC_REPR;  break;
3616     case 'a': oparg = FVC_ASCII; break;
3617     case -1:  oparg = FVC_NONE;  break;
3618     default:
3619         PyErr_SetString(PyExc_SystemError,
3620                         "Unrecognized conversion character");
3621         return 0;
3622     }
3623     if (e->v.FormattedValue.format_spec) {
3624         /* Evaluate the format spec, and update our opcode arg. */
3625         VISIT(c, expr, e->v.FormattedValue.format_spec);
3626         oparg |= FVS_HAVE_SPEC;
3627     }
3628 
3629     /* And push our opcode and oparg */
3630     ADDOP_I(c, FORMAT_VALUE, oparg);
3631     return 1;
3632 }
3633 
3634 static int
compiler_subkwargs(struct compiler * c,asdl_seq * keywords,Py_ssize_t begin,Py_ssize_t end)3635 compiler_subkwargs(struct compiler *c, asdl_seq *keywords, Py_ssize_t begin, Py_ssize_t end)
3636 {
3637     Py_ssize_t i, n = end - begin;
3638     keyword_ty kw;
3639     PyObject *keys, *key;
3640     assert(n > 0);
3641     if (n > 1) {
3642         for (i = begin; i < end; i++) {
3643             kw = asdl_seq_GET(keywords, i);
3644             VISIT(c, expr, kw->value);
3645         }
3646         keys = PyTuple_New(n);
3647         if (keys == NULL) {
3648             return 0;
3649         }
3650         for (i = begin; i < end; i++) {
3651             key = ((keyword_ty) asdl_seq_GET(keywords, i))->arg;
3652             Py_INCREF(key);
3653             PyTuple_SET_ITEM(keys, i - begin, key);
3654         }
3655         ADDOP_N(c, LOAD_CONST, keys, consts);
3656         ADDOP_I(c, BUILD_CONST_KEY_MAP, n);
3657     }
3658     else {
3659         /* a for loop only executes once */
3660         for (i = begin; i < end; i++) {
3661             kw = asdl_seq_GET(keywords, i);
3662             ADDOP_O(c, LOAD_CONST, kw->arg, consts);
3663             VISIT(c, expr, kw->value);
3664         }
3665         ADDOP_I(c, BUILD_MAP, n);
3666     }
3667     return 1;
3668 }
3669 
3670 /* shared code between compiler_call and compiler_class */
3671 static int
compiler_call_helper(struct compiler * c,int n,asdl_seq * args,asdl_seq * keywords)3672 compiler_call_helper(struct compiler *c,
3673                      int n, /* Args already pushed */
3674                      asdl_seq *args,
3675                      asdl_seq *keywords)
3676 {
3677     Py_ssize_t i, nseen, nelts, nkwelts;
3678     int mustdictunpack = 0;
3679 
3680     /* the number of tuples and dictionaries on the stack */
3681     Py_ssize_t nsubargs = 0, nsubkwargs = 0;
3682 
3683     nelts = asdl_seq_LEN(args);
3684     nkwelts = asdl_seq_LEN(keywords);
3685 
3686     for (i = 0; i < nkwelts; i++) {
3687         keyword_ty kw = asdl_seq_GET(keywords, i);
3688         if (kw->arg == NULL) {
3689             mustdictunpack = 1;
3690             break;
3691         }
3692     }
3693 
3694     nseen = n;  /* the number of positional arguments on the stack */
3695     for (i = 0; i < nelts; i++) {
3696         expr_ty elt = asdl_seq_GET(args, i);
3697         if (elt->kind == Starred_kind) {
3698             /* A star-arg. If we've seen positional arguments,
3699                pack the positional arguments into a tuple. */
3700             if (nseen) {
3701                 ADDOP_I(c, BUILD_TUPLE, nseen);
3702                 nseen = 0;
3703                 nsubargs++;
3704             }
3705             VISIT(c, expr, elt->v.Starred.value);
3706             nsubargs++;
3707         }
3708         else {
3709             VISIT(c, expr, elt);
3710             nseen++;
3711         }
3712     }
3713 
3714     /* Same dance again for keyword arguments */
3715     if (nsubargs || mustdictunpack) {
3716         if (nseen) {
3717             /* Pack up any trailing positional arguments. */
3718             ADDOP_I(c, BUILD_TUPLE, nseen);
3719             nsubargs++;
3720         }
3721         if (nsubargs > 1) {
3722             /* If we ended up with more than one stararg, we need
3723                to concatenate them into a single sequence. */
3724             ADDOP_I(c, BUILD_TUPLE_UNPACK_WITH_CALL, nsubargs);
3725         }
3726         else if (nsubargs == 0) {
3727             ADDOP_I(c, BUILD_TUPLE, 0);
3728         }
3729         nseen = 0;  /* the number of keyword arguments on the stack following */
3730         for (i = 0; i < nkwelts; i++) {
3731             keyword_ty kw = asdl_seq_GET(keywords, i);
3732             if (kw->arg == NULL) {
3733                 /* A keyword argument unpacking. */
3734                 if (nseen) {
3735                     if (!compiler_subkwargs(c, keywords, i - nseen, i))
3736                         return 0;
3737                     nsubkwargs++;
3738                     nseen = 0;
3739                 }
3740                 VISIT(c, expr, kw->value);
3741                 nsubkwargs++;
3742             }
3743             else {
3744                 nseen++;
3745             }
3746         }
3747         if (nseen) {
3748             /* Pack up any trailing keyword arguments. */
3749             if (!compiler_subkwargs(c, keywords, nkwelts - nseen, nkwelts))
3750                 return 0;
3751             nsubkwargs++;
3752         }
3753         if (nsubkwargs > 1) {
3754             /* Pack it all up */
3755             ADDOP_I(c, BUILD_MAP_UNPACK_WITH_CALL, nsubkwargs);
3756         }
3757         ADDOP_I(c, CALL_FUNCTION_EX, nsubkwargs > 0);
3758         return 1;
3759     }
3760     else if (nkwelts) {
3761         PyObject *names;
3762         VISIT_SEQ(c, keyword, keywords);
3763         names = PyTuple_New(nkwelts);
3764         if (names == NULL) {
3765             return 0;
3766         }
3767         for (i = 0; i < nkwelts; i++) {
3768             keyword_ty kw = asdl_seq_GET(keywords, i);
3769             Py_INCREF(kw->arg);
3770             PyTuple_SET_ITEM(names, i, kw->arg);
3771         }
3772         ADDOP_N(c, LOAD_CONST, names, consts);
3773         ADDOP_I(c, CALL_FUNCTION_KW, n + nelts + nkwelts);
3774         return 1;
3775     }
3776     else {
3777         ADDOP_I(c, CALL_FUNCTION, n + nelts);
3778         return 1;
3779     }
3780 }
3781 
3782 
3783 /* List and set comprehensions and generator expressions work by creating a
3784   nested function to perform the actual iteration. This means that the
3785   iteration variables don't leak into the current scope.
3786   The defined function is called immediately following its definition, with the
3787   result of that call being the result of the expression.
3788   The LC/SC version returns the populated container, while the GE version is
3789   flagged in symtable.c as a generator, so it returns the generator object
3790   when the function is called.
3791   This code *knows* that the loop cannot contain break, continue, or return,
3792   so it cheats and skips the SETUP_LOOP/POP_BLOCK steps used in normal loops.
3793 
3794   Possible cleanups:
3795     - iterate over the generator sequence instead of using recursion
3796 */
3797 
3798 
3799 static int
compiler_comprehension_generator(struct compiler * c,asdl_seq * generators,int gen_index,expr_ty elt,expr_ty val,int type)3800 compiler_comprehension_generator(struct compiler *c,
3801                                  asdl_seq *generators, int gen_index,
3802                                  expr_ty elt, expr_ty val, int type)
3803 {
3804     comprehension_ty gen;
3805     gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
3806     if (gen->is_async) {
3807         return compiler_async_comprehension_generator(
3808             c, generators, gen_index, elt, val, type);
3809     } else {
3810         return compiler_sync_comprehension_generator(
3811             c, generators, gen_index, elt, val, type);
3812     }
3813 }
3814 
3815 static int
compiler_sync_comprehension_generator(struct compiler * c,asdl_seq * generators,int gen_index,expr_ty elt,expr_ty val,int type)3816 compiler_sync_comprehension_generator(struct compiler *c,
3817                                       asdl_seq *generators, int gen_index,
3818                                       expr_ty elt, expr_ty val, int type)
3819 {
3820     /* generate code for the iterator, then each of the ifs,
3821        and then write to the element */
3822 
3823     comprehension_ty gen;
3824     basicblock *start, *anchor, *skip, *if_cleanup;
3825     Py_ssize_t i, n;
3826 
3827     start = compiler_new_block(c);
3828     skip = compiler_new_block(c);
3829     if_cleanup = compiler_new_block(c);
3830     anchor = compiler_new_block(c);
3831 
3832     if (start == NULL || skip == NULL || if_cleanup == NULL ||
3833         anchor == NULL)
3834         return 0;
3835 
3836     gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
3837 
3838     if (gen_index == 0) {
3839         /* Receive outermost iter as an implicit argument */
3840         c->u->u_argcount = 1;
3841         ADDOP_I(c, LOAD_FAST, 0);
3842     }
3843     else {
3844         /* Sub-iter - calculate on the fly */
3845         VISIT(c, expr, gen->iter);
3846         ADDOP(c, GET_ITER);
3847     }
3848     compiler_use_next_block(c, start);
3849     ADDOP_JREL(c, FOR_ITER, anchor);
3850     NEXT_BLOCK(c);
3851     VISIT(c, expr, gen->target);
3852 
3853     /* XXX this needs to be cleaned up...a lot! */
3854     n = asdl_seq_LEN(gen->ifs);
3855     for (i = 0; i < n; i++) {
3856         expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i);
3857         if (!compiler_jump_if(c, e, if_cleanup, 0))
3858             return 0;
3859         NEXT_BLOCK(c);
3860     }
3861 
3862     if (++gen_index < asdl_seq_LEN(generators))
3863         if (!compiler_comprehension_generator(c,
3864                                               generators, gen_index,
3865                                               elt, val, type))
3866         return 0;
3867 
3868     /* only append after the last for generator */
3869     if (gen_index >= asdl_seq_LEN(generators)) {
3870         /* comprehension specific code */
3871         switch (type) {
3872         case COMP_GENEXP:
3873             VISIT(c, expr, elt);
3874             ADDOP(c, YIELD_VALUE);
3875             ADDOP(c, POP_TOP);
3876             break;
3877         case COMP_LISTCOMP:
3878             VISIT(c, expr, elt);
3879             ADDOP_I(c, LIST_APPEND, gen_index + 1);
3880             break;
3881         case COMP_SETCOMP:
3882             VISIT(c, expr, elt);
3883             ADDOP_I(c, SET_ADD, gen_index + 1);
3884             break;
3885         case COMP_DICTCOMP:
3886             /* With 'd[k] = v', v is evaluated before k, so we do
3887                the same. */
3888             VISIT(c, expr, val);
3889             VISIT(c, expr, elt);
3890             ADDOP_I(c, MAP_ADD, gen_index + 1);
3891             break;
3892         default:
3893             return 0;
3894         }
3895 
3896         compiler_use_next_block(c, skip);
3897     }
3898     compiler_use_next_block(c, if_cleanup);
3899     ADDOP_JABS(c, JUMP_ABSOLUTE, start);
3900     compiler_use_next_block(c, anchor);
3901 
3902     return 1;
3903 }
3904 
3905 static int
compiler_async_comprehension_generator(struct compiler * c,asdl_seq * generators,int gen_index,expr_ty elt,expr_ty val,int type)3906 compiler_async_comprehension_generator(struct compiler *c,
3907                                       asdl_seq *generators, int gen_index,
3908                                       expr_ty elt, expr_ty val, int type)
3909 {
3910     _Py_IDENTIFIER(StopAsyncIteration);
3911 
3912     comprehension_ty gen;
3913     basicblock *if_cleanup, *try,
3914                *after_try, *except, *try_cleanup;
3915     Py_ssize_t i, n;
3916 
3917     PyObject *stop_aiter_error = _PyUnicode_FromId(&PyId_StopAsyncIteration);
3918     if (stop_aiter_error == NULL) {
3919         return 0;
3920     }
3921 
3922     try = compiler_new_block(c);
3923     after_try = compiler_new_block(c);
3924     except = compiler_new_block(c);
3925     if_cleanup = compiler_new_block(c);
3926     try_cleanup = compiler_new_block(c);
3927 
3928     if (if_cleanup == NULL ||
3929             try == NULL || after_try == NULL ||
3930             except == NULL || try_cleanup == NULL) {
3931         return 0;
3932     }
3933 
3934     gen = (comprehension_ty)asdl_seq_GET(generators, gen_index);
3935 
3936     if (gen_index == 0) {
3937         /* Receive outermost iter as an implicit argument */
3938         c->u->u_argcount = 1;
3939         ADDOP_I(c, LOAD_FAST, 0);
3940     }
3941     else {
3942         /* Sub-iter - calculate on the fly */
3943         VISIT(c, expr, gen->iter);
3944         ADDOP(c, GET_AITER);
3945     }
3946 
3947     compiler_use_next_block(c, try);
3948 
3949 
3950     ADDOP_JREL(c, SETUP_EXCEPT, except);
3951     if (!compiler_push_fblock(c, EXCEPT, try))
3952         return 0;
3953 
3954     ADDOP(c, GET_ANEXT);
3955     ADDOP_O(c, LOAD_CONST, Py_None, consts);
3956     ADDOP(c, YIELD_FROM);
3957     VISIT(c, expr, gen->target);
3958     ADDOP(c, POP_BLOCK);
3959     compiler_pop_fblock(c, EXCEPT, try);
3960     ADDOP_JREL(c, JUMP_FORWARD, after_try);
3961 
3962 
3963     compiler_use_next_block(c, except);
3964     ADDOP(c, DUP_TOP);
3965     ADDOP_O(c, LOAD_GLOBAL, stop_aiter_error, names);
3966     ADDOP_I(c, COMPARE_OP, PyCmp_EXC_MATCH);
3967     ADDOP_JABS(c, POP_JUMP_IF_TRUE, try_cleanup);
3968     ADDOP(c, END_FINALLY);
3969 
3970     compiler_use_next_block(c, after_try);
3971 
3972     n = asdl_seq_LEN(gen->ifs);
3973     for (i = 0; i < n; i++) {
3974         expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i);
3975         if (!compiler_jump_if(c, e, if_cleanup, 0))
3976             return 0;
3977         NEXT_BLOCK(c);
3978     }
3979 
3980     if (++gen_index < asdl_seq_LEN(generators))
3981         if (!compiler_comprehension_generator(c,
3982                                               generators, gen_index,
3983                                               elt, val, type))
3984         return 0;
3985 
3986     /* only append after the last for generator */
3987     if (gen_index >= asdl_seq_LEN(generators)) {
3988         /* comprehension specific code */
3989         switch (type) {
3990         case COMP_GENEXP:
3991             VISIT(c, expr, elt);
3992             ADDOP(c, YIELD_VALUE);
3993             ADDOP(c, POP_TOP);
3994             break;
3995         case COMP_LISTCOMP:
3996             VISIT(c, expr, elt);
3997             ADDOP_I(c, LIST_APPEND, gen_index + 1);
3998             break;
3999         case COMP_SETCOMP:
4000             VISIT(c, expr, elt);
4001             ADDOP_I(c, SET_ADD, gen_index + 1);
4002             break;
4003         case COMP_DICTCOMP:
4004             /* With 'd[k] = v', v is evaluated before k, so we do
4005                the same. */
4006             VISIT(c, expr, val);
4007             VISIT(c, expr, elt);
4008             ADDOP_I(c, MAP_ADD, gen_index + 1);
4009             break;
4010         default:
4011             return 0;
4012         }
4013     }
4014     compiler_use_next_block(c, if_cleanup);
4015     ADDOP_JABS(c, JUMP_ABSOLUTE, try);
4016 
4017     compiler_use_next_block(c, try_cleanup);
4018     ADDOP(c, POP_TOP);
4019     ADDOP(c, POP_TOP);
4020     ADDOP(c, POP_TOP);
4021     ADDOP(c, POP_EXCEPT); /* for SETUP_EXCEPT */
4022     ADDOP(c, POP_TOP);
4023 
4024     return 1;
4025 }
4026 
4027 static int
compiler_comprehension(struct compiler * c,expr_ty e,int type,identifier name,asdl_seq * generators,expr_ty elt,expr_ty val)4028 compiler_comprehension(struct compiler *c, expr_ty e, int type,
4029                        identifier name, asdl_seq *generators, expr_ty elt,
4030                        expr_ty val)
4031 {
4032     PyCodeObject *co = NULL;
4033     comprehension_ty outermost;
4034     PyObject *qualname = NULL;
4035     int is_async_function = c->u->u_ste->ste_coroutine;
4036     int is_async_generator = 0;
4037 
4038     outermost = (comprehension_ty) asdl_seq_GET(generators, 0);
4039 
4040     if (!compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION,
4041                               (void *)e, e->lineno))
4042     {
4043         goto error;
4044     }
4045 
4046     is_async_generator = c->u->u_ste->ste_coroutine;
4047 
4048     if (is_async_generator && !is_async_function && type != COMP_GENEXP) {
4049         if (e->lineno > c->u->u_lineno) {
4050             c->u->u_lineno = e->lineno;
4051             c->u->u_lineno_set = 0;
4052         }
4053         compiler_error(c, "asynchronous comprehension outside of "
4054                           "an asynchronous function");
4055         goto error_in_scope;
4056     }
4057 
4058     if (type != COMP_GENEXP) {
4059         int op;
4060         switch (type) {
4061         case COMP_LISTCOMP:
4062             op = BUILD_LIST;
4063             break;
4064         case COMP_SETCOMP:
4065             op = BUILD_SET;
4066             break;
4067         case COMP_DICTCOMP:
4068             op = BUILD_MAP;
4069             break;
4070         default:
4071             PyErr_Format(PyExc_SystemError,
4072                          "unknown comprehension type %d", type);
4073             goto error_in_scope;
4074         }
4075 
4076         ADDOP_I(c, op, 0);
4077     }
4078 
4079     if (!compiler_comprehension_generator(c, generators, 0, elt,
4080                                           val, type))
4081         goto error_in_scope;
4082 
4083     if (type != COMP_GENEXP) {
4084         ADDOP(c, RETURN_VALUE);
4085     }
4086 
4087     co = assemble(c, 1);
4088     qualname = c->u->u_qualname;
4089     Py_INCREF(qualname);
4090     compiler_exit_scope(c);
4091     if (co == NULL)
4092         goto error;
4093 
4094     if (!compiler_make_closure(c, co, 0, qualname))
4095         goto error;
4096     Py_DECREF(qualname);
4097     Py_DECREF(co);
4098 
4099     VISIT(c, expr, outermost->iter);
4100 
4101     if (outermost->is_async) {
4102         ADDOP(c, GET_AITER);
4103     } else {
4104         ADDOP(c, GET_ITER);
4105     }
4106 
4107     ADDOP_I(c, CALL_FUNCTION, 1);
4108 
4109     if (is_async_generator && type != COMP_GENEXP) {
4110         ADDOP(c, GET_AWAITABLE);
4111         ADDOP_O(c, LOAD_CONST, Py_None, consts);
4112         ADDOP(c, YIELD_FROM);
4113     }
4114 
4115     return 1;
4116 error_in_scope:
4117     compiler_exit_scope(c);
4118 error:
4119     Py_XDECREF(qualname);
4120     Py_XDECREF(co);
4121     return 0;
4122 }
4123 
4124 static int
compiler_genexp(struct compiler * c,expr_ty e)4125 compiler_genexp(struct compiler *c, expr_ty e)
4126 {
4127     static identifier name;
4128     if (!name) {
4129         name = PyUnicode_FromString("<genexpr>");
4130         if (!name)
4131             return 0;
4132     }
4133     assert(e->kind == GeneratorExp_kind);
4134     return compiler_comprehension(c, e, COMP_GENEXP, name,
4135                                   e->v.GeneratorExp.generators,
4136                                   e->v.GeneratorExp.elt, NULL);
4137 }
4138 
4139 static int
compiler_listcomp(struct compiler * c,expr_ty e)4140 compiler_listcomp(struct compiler *c, expr_ty e)
4141 {
4142     static identifier name;
4143     if (!name) {
4144         name = PyUnicode_FromString("<listcomp>");
4145         if (!name)
4146             return 0;
4147     }
4148     assert(e->kind == ListComp_kind);
4149     return compiler_comprehension(c, e, COMP_LISTCOMP, name,
4150                                   e->v.ListComp.generators,
4151                                   e->v.ListComp.elt, NULL);
4152 }
4153 
4154 static int
compiler_setcomp(struct compiler * c,expr_ty e)4155 compiler_setcomp(struct compiler *c, expr_ty e)
4156 {
4157     static identifier name;
4158     if (!name) {
4159         name = PyUnicode_FromString("<setcomp>");
4160         if (!name)
4161             return 0;
4162     }
4163     assert(e->kind == SetComp_kind);
4164     return compiler_comprehension(c, e, COMP_SETCOMP, name,
4165                                   e->v.SetComp.generators,
4166                                   e->v.SetComp.elt, NULL);
4167 }
4168 
4169 
4170 static int
compiler_dictcomp(struct compiler * c,expr_ty e)4171 compiler_dictcomp(struct compiler *c, expr_ty e)
4172 {
4173     static identifier name;
4174     if (!name) {
4175         name = PyUnicode_FromString("<dictcomp>");
4176         if (!name)
4177             return 0;
4178     }
4179     assert(e->kind == DictComp_kind);
4180     return compiler_comprehension(c, e, COMP_DICTCOMP, name,
4181                                   e->v.DictComp.generators,
4182                                   e->v.DictComp.key, e->v.DictComp.value);
4183 }
4184 
4185 
4186 static int
compiler_visit_keyword(struct compiler * c,keyword_ty k)4187 compiler_visit_keyword(struct compiler *c, keyword_ty k)
4188 {
4189     VISIT(c, expr, k->value);
4190     return 1;
4191 }
4192 
4193 /* Test whether expression is constant.  For constants, report
4194    whether they are true or false.
4195 
4196    Return values: 1 for true, 0 for false, -1 for non-constant.
4197  */
4198 
4199 static int
expr_constant(expr_ty e)4200 expr_constant(expr_ty e)
4201 {
4202     if (is_const(e)) {
4203         return PyObject_IsTrue(get_const_value(e));
4204     }
4205     return -1;
4206 }
4207 
4208 
4209 /*
4210    Implements the async with statement.
4211 
4212    The semantics outlined in that PEP are as follows:
4213 
4214    async with EXPR as VAR:
4215        BLOCK
4216 
4217    It is implemented roughly as:
4218 
4219    context = EXPR
4220    exit = context.__aexit__  # not calling it
4221    value = await context.__aenter__()
4222    try:
4223        VAR = value  # if VAR present in the syntax
4224        BLOCK
4225    finally:
4226        if an exception was raised:
4227            exc = copy of (exception, instance, traceback)
4228        else:
4229            exc = (None, None, None)
4230        if not (await exit(*exc)):
4231            raise
4232  */
4233 static int
compiler_async_with(struct compiler * c,stmt_ty s,int pos)4234 compiler_async_with(struct compiler *c, stmt_ty s, int pos)
4235 {
4236     basicblock *block, *finally;
4237     withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
4238 
4239     assert(s->kind == AsyncWith_kind);
4240     if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
4241         return compiler_error(c, "'async with' outside async function");
4242     }
4243 
4244     block = compiler_new_block(c);
4245     finally = compiler_new_block(c);
4246     if (!block || !finally)
4247         return 0;
4248 
4249     /* Evaluate EXPR */
4250     VISIT(c, expr, item->context_expr);
4251 
4252     ADDOP(c, BEFORE_ASYNC_WITH);
4253     ADDOP(c, GET_AWAITABLE);
4254     ADDOP_O(c, LOAD_CONST, Py_None, consts);
4255     ADDOP(c, YIELD_FROM);
4256 
4257     ADDOP_JREL(c, SETUP_ASYNC_WITH, finally);
4258 
4259     /* SETUP_ASYNC_WITH pushes a finally block. */
4260     compiler_use_next_block(c, block);
4261     if (!compiler_push_fblock(c, FINALLY_TRY, block)) {
4262         return 0;
4263     }
4264 
4265     if (item->optional_vars) {
4266         VISIT(c, expr, item->optional_vars);
4267     }
4268     else {
4269     /* Discard result from context.__aenter__() */
4270         ADDOP(c, POP_TOP);
4271     }
4272 
4273     pos++;
4274     if (pos == asdl_seq_LEN(s->v.AsyncWith.items))
4275         /* BLOCK code */
4276         VISIT_SEQ(c, stmt, s->v.AsyncWith.body)
4277     else if (!compiler_async_with(c, s, pos))
4278             return 0;
4279 
4280     /* End of try block; start the finally block */
4281     ADDOP(c, POP_BLOCK);
4282     compiler_pop_fblock(c, FINALLY_TRY, block);
4283 
4284     ADDOP_O(c, LOAD_CONST, Py_None, consts);
4285     compiler_use_next_block(c, finally);
4286     if (!compiler_push_fblock(c, FINALLY_END, finally))
4287         return 0;
4288 
4289     /* Finally block starts; context.__exit__ is on the stack under
4290        the exception or return information. Just issue our magic
4291        opcode. */
4292     ADDOP(c, WITH_CLEANUP_START);
4293 
4294     ADDOP(c, GET_AWAITABLE);
4295     ADDOP_O(c, LOAD_CONST, Py_None, consts);
4296     ADDOP(c, YIELD_FROM);
4297 
4298     ADDOP(c, WITH_CLEANUP_FINISH);
4299 
4300     /* Finally block ends. */
4301     ADDOP(c, END_FINALLY);
4302     compiler_pop_fblock(c, FINALLY_END, finally);
4303     return 1;
4304 }
4305 
4306 
4307 /*
4308    Implements the with statement from PEP 343.
4309 
4310    The semantics outlined in that PEP are as follows:
4311 
4312    with EXPR as VAR:
4313        BLOCK
4314 
4315    It is implemented roughly as:
4316 
4317    context = EXPR
4318    exit = context.__exit__  # not calling it
4319    value = context.__enter__()
4320    try:
4321        VAR = value  # if VAR present in the syntax
4322        BLOCK
4323    finally:
4324        if an exception was raised:
4325            exc = copy of (exception, instance, traceback)
4326        else:
4327            exc = (None, None, None)
4328        exit(*exc)
4329  */
4330 static int
compiler_with(struct compiler * c,stmt_ty s,int pos)4331 compiler_with(struct compiler *c, stmt_ty s, int pos)
4332 {
4333     basicblock *block, *finally;
4334     withitem_ty item = asdl_seq_GET(s->v.With.items, pos);
4335 
4336     assert(s->kind == With_kind);
4337 
4338     block = compiler_new_block(c);
4339     finally = compiler_new_block(c);
4340     if (!block || !finally)
4341         return 0;
4342 
4343     /* Evaluate EXPR */
4344     VISIT(c, expr, item->context_expr);
4345     ADDOP_JREL(c, SETUP_WITH, finally);
4346 
4347     /* SETUP_WITH pushes a finally block. */
4348     compiler_use_next_block(c, block);
4349     if (!compiler_push_fblock(c, FINALLY_TRY, block)) {
4350         return 0;
4351     }
4352 
4353     if (item->optional_vars) {
4354         VISIT(c, expr, item->optional_vars);
4355     }
4356     else {
4357     /* Discard result from context.__enter__() */
4358         ADDOP(c, POP_TOP);
4359     }
4360 
4361     pos++;
4362     if (pos == asdl_seq_LEN(s->v.With.items))
4363         /* BLOCK code */
4364         VISIT_SEQ(c, stmt, s->v.With.body)
4365     else if (!compiler_with(c, s, pos))
4366             return 0;
4367 
4368     /* End of try block; start the finally block */
4369     ADDOP(c, POP_BLOCK);
4370     compiler_pop_fblock(c, FINALLY_TRY, block);
4371 
4372     ADDOP_O(c, LOAD_CONST, Py_None, consts);
4373     compiler_use_next_block(c, finally);
4374     if (!compiler_push_fblock(c, FINALLY_END, finally))
4375         return 0;
4376 
4377     /* Finally block starts; context.__exit__ is on the stack under
4378        the exception or return information. Just issue our magic
4379        opcode. */
4380     ADDOP(c, WITH_CLEANUP_START);
4381     ADDOP(c, WITH_CLEANUP_FINISH);
4382 
4383     /* Finally block ends. */
4384     ADDOP(c, END_FINALLY);
4385     compiler_pop_fblock(c, FINALLY_END, finally);
4386     return 1;
4387 }
4388 
4389 static int
compiler_visit_expr(struct compiler * c,expr_ty e)4390 compiler_visit_expr(struct compiler *c, expr_ty e)
4391 {
4392     /* If expr e has a different line number than the last expr/stmt,
4393        set a new line number for the next instruction.
4394     */
4395     if (e->lineno > c->u->u_lineno) {
4396         c->u->u_lineno = e->lineno;
4397         c->u->u_lineno_set = 0;
4398     }
4399     /* Updating the column offset is always harmless. */
4400     c->u->u_col_offset = e->col_offset;
4401     switch (e->kind) {
4402     case BoolOp_kind:
4403         return compiler_boolop(c, e);
4404     case BinOp_kind:
4405         VISIT(c, expr, e->v.BinOp.left);
4406         VISIT(c, expr, e->v.BinOp.right);
4407         ADDOP(c, binop(c, e->v.BinOp.op));
4408         break;
4409     case UnaryOp_kind:
4410         VISIT(c, expr, e->v.UnaryOp.operand);
4411         ADDOP(c, unaryop(e->v.UnaryOp.op));
4412         break;
4413     case Lambda_kind:
4414         return compiler_lambda(c, e);
4415     case IfExp_kind:
4416         return compiler_ifexp(c, e);
4417     case Dict_kind:
4418         return compiler_dict(c, e);
4419     case Set_kind:
4420         return compiler_set(c, e);
4421     case GeneratorExp_kind:
4422         return compiler_genexp(c, e);
4423     case ListComp_kind:
4424         return compiler_listcomp(c, e);
4425     case SetComp_kind:
4426         return compiler_setcomp(c, e);
4427     case DictComp_kind:
4428         return compiler_dictcomp(c, e);
4429     case Yield_kind:
4430         if (c->u->u_ste->ste_type != FunctionBlock)
4431             return compiler_error(c, "'yield' outside function");
4432         if (e->v.Yield.value) {
4433             VISIT(c, expr, e->v.Yield.value);
4434         }
4435         else {
4436             ADDOP_O(c, LOAD_CONST, Py_None, consts);
4437         }
4438         ADDOP(c, YIELD_VALUE);
4439         break;
4440     case YieldFrom_kind:
4441         if (c->u->u_ste->ste_type != FunctionBlock)
4442             return compiler_error(c, "'yield' outside function");
4443 
4444         if (c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION)
4445             return compiler_error(c, "'yield from' inside async function");
4446 
4447         VISIT(c, expr, e->v.YieldFrom.value);
4448         ADDOP(c, GET_YIELD_FROM_ITER);
4449         ADDOP_O(c, LOAD_CONST, Py_None, consts);
4450         ADDOP(c, YIELD_FROM);
4451         break;
4452     case Await_kind:
4453         if (c->u->u_ste->ste_type != FunctionBlock)
4454             return compiler_error(c, "'await' outside function");
4455 
4456         if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
4457                 c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION)
4458             return compiler_error(c, "'await' outside async function");
4459 
4460         VISIT(c, expr, e->v.Await.value);
4461         ADDOP(c, GET_AWAITABLE);
4462         ADDOP_O(c, LOAD_CONST, Py_None, consts);
4463         ADDOP(c, YIELD_FROM);
4464         break;
4465     case Compare_kind:
4466         return compiler_compare(c, e);
4467     case Call_kind:
4468         return compiler_call(c, e);
4469     case Constant_kind:
4470         ADDOP_O(c, LOAD_CONST, e->v.Constant.value, consts);
4471         break;
4472     case Num_kind:
4473         ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts);
4474         break;
4475     case Str_kind:
4476         ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts);
4477         break;
4478     case JoinedStr_kind:
4479         return compiler_joined_str(c, e);
4480     case FormattedValue_kind:
4481         return compiler_formatted_value(c, e);
4482     case Bytes_kind:
4483         ADDOP_O(c, LOAD_CONST, e->v.Bytes.s, consts);
4484         break;
4485     case Ellipsis_kind:
4486         ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts);
4487         break;
4488     case NameConstant_kind:
4489         ADDOP_O(c, LOAD_CONST, e->v.NameConstant.value, consts);
4490         break;
4491     /* The following exprs can be assignment targets. */
4492     case Attribute_kind:
4493         if (e->v.Attribute.ctx != AugStore)
4494             VISIT(c, expr, e->v.Attribute.value);
4495         switch (e->v.Attribute.ctx) {
4496         case AugLoad:
4497             ADDOP(c, DUP_TOP);
4498             /* Fall through */
4499         case Load:
4500             ADDOP_NAME(c, LOAD_ATTR, e->v.Attribute.attr, names);
4501             break;
4502         case AugStore:
4503             ADDOP(c, ROT_TWO);
4504             /* Fall through */
4505         case Store:
4506             ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names);
4507             break;
4508         case Del:
4509             ADDOP_NAME(c, DELETE_ATTR, e->v.Attribute.attr, names);
4510             break;
4511         case Param:
4512         default:
4513             PyErr_SetString(PyExc_SystemError,
4514                             "param invalid in attribute expression");
4515             return 0;
4516         }
4517         break;
4518     case Subscript_kind:
4519         switch (e->v.Subscript.ctx) {
4520         case AugLoad:
4521             VISIT(c, expr, e->v.Subscript.value);
4522             VISIT_SLICE(c, e->v.Subscript.slice, AugLoad);
4523             break;
4524         case Load:
4525             VISIT(c, expr, e->v.Subscript.value);
4526             VISIT_SLICE(c, e->v.Subscript.slice, Load);
4527             break;
4528         case AugStore:
4529             VISIT_SLICE(c, e->v.Subscript.slice, AugStore);
4530             break;
4531         case Store:
4532             VISIT(c, expr, e->v.Subscript.value);
4533             VISIT_SLICE(c, e->v.Subscript.slice, Store);
4534             break;
4535         case Del:
4536             VISIT(c, expr, e->v.Subscript.value);
4537             VISIT_SLICE(c, e->v.Subscript.slice, Del);
4538             break;
4539         case Param:
4540         default:
4541             PyErr_SetString(PyExc_SystemError,
4542                 "param invalid in subscript expression");
4543             return 0;
4544         }
4545         break;
4546     case Starred_kind:
4547         switch (e->v.Starred.ctx) {
4548         case Store:
4549             /* In all legitimate cases, the Starred node was already replaced
4550              * by compiler_list/compiler_tuple. XXX: is that okay? */
4551             return compiler_error(c,
4552                 "starred assignment target must be in a list or tuple");
4553         default:
4554             return compiler_error(c,
4555                 "can't use starred expression here");
4556         }
4557         break;
4558     case Name_kind:
4559         return compiler_nameop(c, e->v.Name.id, e->v.Name.ctx);
4560     /* child nodes of List and Tuple will have expr_context set */
4561     case List_kind:
4562         return compiler_list(c, e);
4563     case Tuple_kind:
4564         return compiler_tuple(c, e);
4565     }
4566     return 1;
4567 }
4568 
4569 static int
compiler_augassign(struct compiler * c,stmt_ty s)4570 compiler_augassign(struct compiler *c, stmt_ty s)
4571 {
4572     expr_ty e = s->v.AugAssign.target;
4573     expr_ty auge;
4574 
4575     assert(s->kind == AugAssign_kind);
4576 
4577     switch (e->kind) {
4578     case Attribute_kind:
4579         auge = Attribute(e->v.Attribute.value, e->v.Attribute.attr,
4580                          AugLoad, e->lineno, e->col_offset, c->c_arena);
4581         if (auge == NULL)
4582             return 0;
4583         VISIT(c, expr, auge);
4584         VISIT(c, expr, s->v.AugAssign.value);
4585         ADDOP(c, inplace_binop(c, s->v.AugAssign.op));
4586         auge->v.Attribute.ctx = AugStore;
4587         VISIT(c, expr, auge);
4588         break;
4589     case Subscript_kind:
4590         auge = Subscript(e->v.Subscript.value, e->v.Subscript.slice,
4591                          AugLoad, e->lineno, e->col_offset, c->c_arena);
4592         if (auge == NULL)
4593             return 0;
4594         VISIT(c, expr, auge);
4595         VISIT(c, expr, s->v.AugAssign.value);
4596         ADDOP(c, inplace_binop(c, s->v.AugAssign.op));
4597         auge->v.Subscript.ctx = AugStore;
4598         VISIT(c, expr, auge);
4599         break;
4600     case Name_kind:
4601         if (!compiler_nameop(c, e->v.Name.id, Load))
4602             return 0;
4603         VISIT(c, expr, s->v.AugAssign.value);
4604         ADDOP(c, inplace_binop(c, s->v.AugAssign.op));
4605         return compiler_nameop(c, e->v.Name.id, Store);
4606     default:
4607         PyErr_Format(PyExc_SystemError,
4608             "invalid node type (%d) for augmented assignment",
4609             e->kind);
4610         return 0;
4611     }
4612     return 1;
4613 }
4614 
4615 static int
check_ann_expr(struct compiler * c,expr_ty e)4616 check_ann_expr(struct compiler *c, expr_ty e)
4617 {
4618     VISIT(c, expr, e);
4619     ADDOP(c, POP_TOP);
4620     return 1;
4621 }
4622 
4623 static int
check_annotation(struct compiler * c,stmt_ty s)4624 check_annotation(struct compiler *c, stmt_ty s)
4625 {
4626     /* Annotations are only evaluated in a module or class. */
4627     if (c->u->u_scope_type == COMPILER_SCOPE_MODULE ||
4628         c->u->u_scope_type == COMPILER_SCOPE_CLASS) {
4629         return check_ann_expr(c, s->v.AnnAssign.annotation);
4630     }
4631     return 1;
4632 }
4633 
4634 static int
check_ann_slice(struct compiler * c,slice_ty sl)4635 check_ann_slice(struct compiler *c, slice_ty sl)
4636 {
4637     switch(sl->kind) {
4638     case Index_kind:
4639         return check_ann_expr(c, sl->v.Index.value);
4640     case Slice_kind:
4641         if (sl->v.Slice.lower && !check_ann_expr(c, sl->v.Slice.lower)) {
4642             return 0;
4643         }
4644         if (sl->v.Slice.upper && !check_ann_expr(c, sl->v.Slice.upper)) {
4645             return 0;
4646         }
4647         if (sl->v.Slice.step && !check_ann_expr(c, sl->v.Slice.step)) {
4648             return 0;
4649         }
4650         break;
4651     default:
4652         PyErr_SetString(PyExc_SystemError,
4653                         "unexpected slice kind");
4654         return 0;
4655     }
4656     return 1;
4657 }
4658 
4659 static int
check_ann_subscr(struct compiler * c,slice_ty sl)4660 check_ann_subscr(struct compiler *c, slice_ty sl)
4661 {
4662     /* We check that everything in a subscript is defined at runtime. */
4663     Py_ssize_t i, n;
4664 
4665     switch (sl->kind) {
4666     case Index_kind:
4667     case Slice_kind:
4668         if (!check_ann_slice(c, sl)) {
4669             return 0;
4670         }
4671         break;
4672     case ExtSlice_kind:
4673         n = asdl_seq_LEN(sl->v.ExtSlice.dims);
4674         for (i = 0; i < n; i++) {
4675             slice_ty subsl = (slice_ty)asdl_seq_GET(sl->v.ExtSlice.dims, i);
4676             switch (subsl->kind) {
4677             case Index_kind:
4678             case Slice_kind:
4679                 if (!check_ann_slice(c, subsl)) {
4680                     return 0;
4681                 }
4682                 break;
4683             case ExtSlice_kind:
4684             default:
4685                 PyErr_SetString(PyExc_SystemError,
4686                                 "extended slice invalid in nested slice");
4687                 return 0;
4688             }
4689         }
4690         break;
4691     default:
4692         PyErr_Format(PyExc_SystemError,
4693                      "invalid subscript kind %d", sl->kind);
4694         return 0;
4695     }
4696     return 1;
4697 }
4698 
4699 static int
compiler_annassign(struct compiler * c,stmt_ty s)4700 compiler_annassign(struct compiler *c, stmt_ty s)
4701 {
4702     expr_ty targ = s->v.AnnAssign.target;
4703     PyObject* mangled;
4704 
4705     assert(s->kind == AnnAssign_kind);
4706 
4707     /* We perform the actual assignment first. */
4708     if (s->v.AnnAssign.value) {
4709         VISIT(c, expr, s->v.AnnAssign.value);
4710         VISIT(c, expr, targ);
4711     }
4712     switch (targ->kind) {
4713     case Name_kind:
4714         /* If we have a simple name in a module or class, store annotation. */
4715         if (s->v.AnnAssign.simple &&
4716             (c->u->u_scope_type == COMPILER_SCOPE_MODULE ||
4717              c->u->u_scope_type == COMPILER_SCOPE_CLASS)) {
4718             if (c->c_future->ff_features & CO_FUTURE_ANNOTATIONS) {
4719                 VISIT(c, annexpr, s->v.AnnAssign.annotation)
4720             }
4721             else {
4722                 VISIT(c, expr, s->v.AnnAssign.annotation);
4723             }
4724             ADDOP_NAME(c, LOAD_NAME, __annotations__, names);
4725             mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id);
4726             if (!mangled) {
4727                 return 0;
4728             }
4729             ADDOP_N(c, LOAD_CONST, mangled, consts);
4730             ADDOP(c, STORE_SUBSCR);
4731         }
4732         break;
4733     case Attribute_kind:
4734         if (!s->v.AnnAssign.value &&
4735             !check_ann_expr(c, targ->v.Attribute.value)) {
4736             return 0;
4737         }
4738         break;
4739     case Subscript_kind:
4740         if (!s->v.AnnAssign.value &&
4741             (!check_ann_expr(c, targ->v.Subscript.value) ||
4742              !check_ann_subscr(c, targ->v.Subscript.slice))) {
4743                 return 0;
4744         }
4745         break;
4746     default:
4747         PyErr_Format(PyExc_SystemError,
4748                      "invalid node type (%d) for annotated assignment",
4749                      targ->kind);
4750             return 0;
4751     }
4752     /* Annotation is evaluated last. */
4753     if (!s->v.AnnAssign.simple && !check_annotation(c, s)) {
4754         return 0;
4755     }
4756     return 1;
4757 }
4758 
4759 static int
compiler_push_fblock(struct compiler * c,enum fblocktype t,basicblock * b)4760 compiler_push_fblock(struct compiler *c, enum fblocktype t, basicblock *b)
4761 {
4762     struct fblockinfo *f;
4763     if (c->u->u_nfblocks >= CO_MAXBLOCKS) {
4764         PyErr_SetString(PyExc_SyntaxError,
4765                         "too many statically nested blocks");
4766         return 0;
4767     }
4768     f = &c->u->u_fblock[c->u->u_nfblocks++];
4769     f->fb_type = t;
4770     f->fb_block = b;
4771     return 1;
4772 }
4773 
4774 static void
compiler_pop_fblock(struct compiler * c,enum fblocktype t,basicblock * b)4775 compiler_pop_fblock(struct compiler *c, enum fblocktype t, basicblock *b)
4776 {
4777     struct compiler_unit *u = c->u;
4778     assert(u->u_nfblocks > 0);
4779     u->u_nfblocks--;
4780     assert(u->u_fblock[u->u_nfblocks].fb_type == t);
4781     assert(u->u_fblock[u->u_nfblocks].fb_block == b);
4782 }
4783 
4784 static int
compiler_in_loop(struct compiler * c)4785 compiler_in_loop(struct compiler *c) {
4786     int i;
4787     struct compiler_unit *u = c->u;
4788     for (i = 0; i < u->u_nfblocks; ++i) {
4789         if (u->u_fblock[i].fb_type == LOOP)
4790             return 1;
4791     }
4792     return 0;
4793 }
4794 /* Raises a SyntaxError and returns 0.
4795    If something goes wrong, a different exception may be raised.
4796 */
4797 
4798 static int
compiler_error(struct compiler * c,const char * errstr)4799 compiler_error(struct compiler *c, const char *errstr)
4800 {
4801     PyObject *loc;
4802     PyObject *u = NULL, *v = NULL;
4803 
4804     loc = PyErr_ProgramTextObject(c->c_filename, c->u->u_lineno);
4805     if (!loc) {
4806         Py_INCREF(Py_None);
4807         loc = Py_None;
4808     }
4809     u = Py_BuildValue("(OiiO)", c->c_filename, c->u->u_lineno,
4810                       c->u->u_col_offset, loc);
4811     if (!u)
4812         goto exit;
4813     v = Py_BuildValue("(zO)", errstr, u);
4814     if (!v)
4815         goto exit;
4816     PyErr_SetObject(PyExc_SyntaxError, v);
4817  exit:
4818     Py_DECREF(loc);
4819     Py_XDECREF(u);
4820     Py_XDECREF(v);
4821     return 0;
4822 }
4823 
4824 static int
compiler_handle_subscr(struct compiler * c,const char * kind,expr_context_ty ctx)4825 compiler_handle_subscr(struct compiler *c, const char *kind,
4826                        expr_context_ty ctx)
4827 {
4828     int op = 0;
4829 
4830     /* XXX this code is duplicated */
4831     switch (ctx) {
4832         case AugLoad: /* fall through to Load */
4833         case Load:    op = BINARY_SUBSCR; break;
4834         case AugStore:/* fall through to Store */
4835         case Store:   op = STORE_SUBSCR; break;
4836         case Del:     op = DELETE_SUBSCR; break;
4837         case Param:
4838             PyErr_Format(PyExc_SystemError,
4839                          "invalid %s kind %d in subscript\n",
4840                          kind, ctx);
4841             return 0;
4842     }
4843     if (ctx == AugLoad) {
4844         ADDOP(c, DUP_TOP_TWO);
4845     }
4846     else if (ctx == AugStore) {
4847         ADDOP(c, ROT_THREE);
4848     }
4849     ADDOP(c, op);
4850     return 1;
4851 }
4852 
4853 static int
compiler_slice(struct compiler * c,slice_ty s,expr_context_ty ctx)4854 compiler_slice(struct compiler *c, slice_ty s, expr_context_ty ctx)
4855 {
4856     int n = 2;
4857     assert(s->kind == Slice_kind);
4858 
4859     /* only handles the cases where BUILD_SLICE is emitted */
4860     if (s->v.Slice.lower) {
4861         VISIT(c, expr, s->v.Slice.lower);
4862     }
4863     else {
4864         ADDOP_O(c, LOAD_CONST, Py_None, consts);
4865     }
4866 
4867     if (s->v.Slice.upper) {
4868         VISIT(c, expr, s->v.Slice.upper);
4869     }
4870     else {
4871         ADDOP_O(c, LOAD_CONST, Py_None, consts);
4872     }
4873 
4874     if (s->v.Slice.step) {
4875         n++;
4876         VISIT(c, expr, s->v.Slice.step);
4877     }
4878     ADDOP_I(c, BUILD_SLICE, n);
4879     return 1;
4880 }
4881 
4882 static int
compiler_visit_nested_slice(struct compiler * c,slice_ty s,expr_context_ty ctx)4883 compiler_visit_nested_slice(struct compiler *c, slice_ty s,
4884                             expr_context_ty ctx)
4885 {
4886     switch (s->kind) {
4887     case Slice_kind:
4888         return compiler_slice(c, s, ctx);
4889     case Index_kind:
4890         VISIT(c, expr, s->v.Index.value);
4891         break;
4892     case ExtSlice_kind:
4893     default:
4894         PyErr_SetString(PyExc_SystemError,
4895                         "extended slice invalid in nested slice");
4896         return 0;
4897     }
4898     return 1;
4899 }
4900 
4901 static int
compiler_visit_slice(struct compiler * c,slice_ty s,expr_context_ty ctx)4902 compiler_visit_slice(struct compiler *c, slice_ty s, expr_context_ty ctx)
4903 {
4904     const char * kindname = NULL;
4905     switch (s->kind) {
4906     case Index_kind:
4907         kindname = "index";
4908         if (ctx != AugStore) {
4909             VISIT(c, expr, s->v.Index.value);
4910         }
4911         break;
4912     case Slice_kind:
4913         kindname = "slice";
4914         if (ctx != AugStore) {
4915             if (!compiler_slice(c, s, ctx))
4916                 return 0;
4917         }
4918         break;
4919     case ExtSlice_kind:
4920         kindname = "extended slice";
4921         if (ctx != AugStore) {
4922             Py_ssize_t i, n = asdl_seq_LEN(s->v.ExtSlice.dims);
4923             for (i = 0; i < n; i++) {
4924                 slice_ty sub = (slice_ty)asdl_seq_GET(
4925                     s->v.ExtSlice.dims, i);
4926                 if (!compiler_visit_nested_slice(c, sub, ctx))
4927                     return 0;
4928             }
4929             ADDOP_I(c, BUILD_TUPLE, n);
4930         }
4931         break;
4932     default:
4933         PyErr_Format(PyExc_SystemError,
4934                      "invalid subscript kind %d", s->kind);
4935         return 0;
4936     }
4937     return compiler_handle_subscr(c, kindname, ctx);
4938 }
4939 
4940 /* End of the compiler section, beginning of the assembler section */
4941 
4942 /* do depth-first search of basic block graph, starting with block.
4943    post records the block indices in post-order.
4944 
4945    XXX must handle implicit jumps from one block to next
4946 */
4947 
4948 struct assembler {
4949     PyObject *a_bytecode;  /* string containing bytecode */
4950     int a_offset;              /* offset into bytecode */
4951     int a_nblocks;             /* number of reachable blocks */
4952     basicblock **a_postorder; /* list of blocks in dfs postorder */
4953     PyObject *a_lnotab;    /* string containing lnotab */
4954     int a_lnotab_off;      /* offset into lnotab */
4955     int a_lineno;              /* last lineno of emitted instruction */
4956     int a_lineno_off;      /* bytecode offset of last lineno */
4957 };
4958 
4959 static void
dfs(struct compiler * c,basicblock * b,struct assembler * a,int end)4960 dfs(struct compiler *c, basicblock *b, struct assembler *a, int end)
4961 {
4962     int i, j;
4963 
4964     /* Get rid of recursion for normal control flow.
4965        Since the number of blocks is limited, unused space in a_postorder
4966        (from a_nblocks to end) can be used as a stack for still not ordered
4967        blocks. */
4968     for (j = end; b && !b->b_seen; b = b->b_next) {
4969         b->b_seen = 1;
4970         assert(a->a_nblocks < j);
4971         a->a_postorder[--j] = b;
4972     }
4973     while (j < end) {
4974         b = a->a_postorder[j++];
4975         for (i = 0; i < b->b_iused; i++) {
4976             struct instr *instr = &b->b_instr[i];
4977             if (instr->i_jrel || instr->i_jabs)
4978                 dfs(c, instr->i_target, a, j);
4979         }
4980         assert(a->a_nblocks < j);
4981         a->a_postorder[a->a_nblocks++] = b;
4982     }
4983 }
4984 
4985 Py_LOCAL_INLINE(void)
stackdepth_push(basicblock *** sp,basicblock * b,int depth)4986 stackdepth_push(basicblock ***sp, basicblock *b, int depth)
4987 {
4988     /* XXX b->b_startdepth > depth only for the target of SETUP_FINALLY,
4989      * SETUP_WITH and SETUP_ASYNC_WITH. */
4990     assert(b->b_startdepth < 0 || b->b_startdepth >= depth);
4991     if (b->b_startdepth < depth) {
4992         assert(b->b_startdepth < 0);
4993         b->b_startdepth = depth;
4994         *(*sp)++ = b;
4995     }
4996 }
4997 
4998 /* Find the flow path that needs the largest stack.  We assume that
4999  * cycles in the flow graph have no net effect on the stack depth.
5000  */
5001 static int
stackdepth(struct compiler * c)5002 stackdepth(struct compiler *c)
5003 {
5004     basicblock *b, *entryblock = NULL;
5005     basicblock **stack, **sp;
5006     int nblocks = 0, maxdepth = 0;
5007     for (b = c->u->u_blocks; b != NULL; b = b->b_list) {
5008         b->b_startdepth = INT_MIN;
5009         entryblock = b;
5010         nblocks++;
5011     }
5012     if (!entryblock)
5013         return 0;
5014     stack = (basicblock **)PyObject_Malloc(sizeof(basicblock *) * nblocks);
5015     if (!stack) {
5016         PyErr_NoMemory();
5017         return -1;
5018     }
5019 
5020     sp = stack;
5021     stackdepth_push(&sp, entryblock, 0);
5022     while (sp != stack) {
5023         b = *--sp;
5024         int depth = b->b_startdepth;
5025         assert(depth >= 0);
5026         basicblock *next = b->b_next;
5027         for (int i = 0; i < b->b_iused; i++) {
5028             struct instr *instr = &b->b_instr[i];
5029             int effect = stack_effect(instr->i_opcode, instr->i_oparg, 0);
5030             if (effect == PY_INVALID_STACK_EFFECT) {
5031                 fprintf(stderr, "opcode = %d\n", instr->i_opcode);
5032                 Py_FatalError("PyCompile_OpcodeStackEffect()");
5033             }
5034             int new_depth = depth + effect;
5035             if (new_depth > maxdepth) {
5036                 maxdepth = new_depth;
5037             }
5038             assert(depth >= 0); /* invalid code or bug in stackdepth() */
5039             if (instr->i_jrel || instr->i_jabs) {
5040                 effect = stack_effect(instr->i_opcode, instr->i_oparg, 1);
5041                 assert(effect != PY_INVALID_STACK_EFFECT);
5042                 int target_depth = depth + effect;
5043                 if (target_depth > maxdepth) {
5044                     maxdepth = target_depth;
5045                 }
5046                 assert(target_depth >= 0); /* invalid code or bug in stackdepth() */
5047                 if (instr->i_opcode == CONTINUE_LOOP) {
5048                     /* Pops a variable number of values from the stack,
5049                      * but the target should be already proceeding.
5050                      */
5051                     assert(instr->i_target->b_startdepth >= 0);
5052                     assert(instr->i_target->b_startdepth <= depth);
5053                     /* remaining code is dead */
5054                     next = NULL;
5055                     break;
5056                 }
5057                 stackdepth_push(&sp, instr->i_target, target_depth);
5058             }
5059             depth = new_depth;
5060             if (instr->i_opcode == JUMP_ABSOLUTE ||
5061                 instr->i_opcode == JUMP_FORWARD ||
5062                 instr->i_opcode == RETURN_VALUE ||
5063                 instr->i_opcode == RAISE_VARARGS ||
5064                 instr->i_opcode == BREAK_LOOP)
5065             {
5066                 /* remaining code is dead */
5067                 next = NULL;
5068                 break;
5069             }
5070         }
5071         if (next != NULL) {
5072             stackdepth_push(&sp, next, depth);
5073         }
5074     }
5075     PyObject_Free(stack);
5076     return maxdepth;
5077 }
5078 
5079 static int
assemble_init(struct assembler * a,int nblocks,int firstlineno)5080 assemble_init(struct assembler *a, int nblocks, int firstlineno)
5081 {
5082     memset(a, 0, sizeof(struct assembler));
5083     a->a_lineno = firstlineno;
5084     a->a_bytecode = PyBytes_FromStringAndSize(NULL, DEFAULT_CODE_SIZE);
5085     if (!a->a_bytecode)
5086         return 0;
5087     a->a_lnotab = PyBytes_FromStringAndSize(NULL, DEFAULT_LNOTAB_SIZE);
5088     if (!a->a_lnotab)
5089         return 0;
5090     if ((size_t)nblocks > SIZE_MAX / sizeof(basicblock *)) {
5091         PyErr_NoMemory();
5092         return 0;
5093     }
5094     a->a_postorder = (basicblock **)PyObject_Malloc(
5095                                         sizeof(basicblock *) * nblocks);
5096     if (!a->a_postorder) {
5097         PyErr_NoMemory();
5098         return 0;
5099     }
5100     return 1;
5101 }
5102 
5103 static void
assemble_free(struct assembler * a)5104 assemble_free(struct assembler *a)
5105 {
5106     Py_XDECREF(a->a_bytecode);
5107     Py_XDECREF(a->a_lnotab);
5108     if (a->a_postorder)
5109         PyObject_Free(a->a_postorder);
5110 }
5111 
5112 static int
blocksize(basicblock * b)5113 blocksize(basicblock *b)
5114 {
5115     int i;
5116     int size = 0;
5117 
5118     for (i = 0; i < b->b_iused; i++)
5119         size += instrsize(b->b_instr[i].i_oparg);
5120     return size;
5121 }
5122 
5123 /* Appends a pair to the end of the line number table, a_lnotab, representing
5124    the instruction's bytecode offset and line number.  See
5125    Objects/lnotab_notes.txt for the description of the line number table. */
5126 
5127 static int
assemble_lnotab(struct assembler * a,struct instr * i)5128 assemble_lnotab(struct assembler *a, struct instr *i)
5129 {
5130     int d_bytecode, d_lineno;
5131     Py_ssize_t len;
5132     unsigned char *lnotab;
5133 
5134     d_bytecode = (a->a_offset - a->a_lineno_off) * sizeof(_Py_CODEUNIT);
5135     d_lineno = i->i_lineno - a->a_lineno;
5136 
5137     assert(d_bytecode >= 0);
5138 
5139     if(d_bytecode == 0 && d_lineno == 0)
5140         return 1;
5141 
5142     if (d_bytecode > 255) {
5143         int j, nbytes, ncodes = d_bytecode / 255;
5144         nbytes = a->a_lnotab_off + 2 * ncodes;
5145         len = PyBytes_GET_SIZE(a->a_lnotab);
5146         if (nbytes >= len) {
5147             if ((len <= INT_MAX / 2) && (len * 2 < nbytes))
5148                 len = nbytes;
5149             else if (len <= INT_MAX / 2)
5150                 len *= 2;
5151             else {
5152                 PyErr_NoMemory();
5153                 return 0;
5154             }
5155             if (_PyBytes_Resize(&a->a_lnotab, len) < 0)
5156                 return 0;
5157         }
5158         lnotab = (unsigned char *)
5159                    PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
5160         for (j = 0; j < ncodes; j++) {
5161             *lnotab++ = 255;
5162             *lnotab++ = 0;
5163         }
5164         d_bytecode -= ncodes * 255;
5165         a->a_lnotab_off += ncodes * 2;
5166     }
5167     assert(0 <= d_bytecode && d_bytecode <= 255);
5168 
5169     if (d_lineno < -128 || 127 < d_lineno) {
5170         int j, nbytes, ncodes, k;
5171         if (d_lineno < 0) {
5172             k = -128;
5173             /* use division on positive numbers */
5174             ncodes = (-d_lineno) / 128;
5175         }
5176         else {
5177             k = 127;
5178             ncodes = d_lineno / 127;
5179         }
5180         d_lineno -= ncodes * k;
5181         assert(ncodes >= 1);
5182         nbytes = a->a_lnotab_off + 2 * ncodes;
5183         len = PyBytes_GET_SIZE(a->a_lnotab);
5184         if (nbytes >= len) {
5185             if ((len <= INT_MAX / 2) && len * 2 < nbytes)
5186                 len = nbytes;
5187             else if (len <= INT_MAX / 2)
5188                 len *= 2;
5189             else {
5190                 PyErr_NoMemory();
5191                 return 0;
5192             }
5193             if (_PyBytes_Resize(&a->a_lnotab, len) < 0)
5194                 return 0;
5195         }
5196         lnotab = (unsigned char *)
5197                    PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
5198         *lnotab++ = d_bytecode;
5199         *lnotab++ = k;
5200         d_bytecode = 0;
5201         for (j = 1; j < ncodes; j++) {
5202             *lnotab++ = 0;
5203             *lnotab++ = k;
5204         }
5205         a->a_lnotab_off += ncodes * 2;
5206     }
5207     assert(-128 <= d_lineno && d_lineno <= 127);
5208 
5209     len = PyBytes_GET_SIZE(a->a_lnotab);
5210     if (a->a_lnotab_off + 2 >= len) {
5211         if (_PyBytes_Resize(&a->a_lnotab, len * 2) < 0)
5212             return 0;
5213     }
5214     lnotab = (unsigned char *)
5215                     PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
5216 
5217     a->a_lnotab_off += 2;
5218     if (d_bytecode) {
5219         *lnotab++ = d_bytecode;
5220         *lnotab++ = d_lineno;
5221     }
5222     else {      /* First line of a block; def stmt, etc. */
5223         *lnotab++ = 0;
5224         *lnotab++ = d_lineno;
5225     }
5226     a->a_lineno = i->i_lineno;
5227     a->a_lineno_off = a->a_offset;
5228     return 1;
5229 }
5230 
5231 /* assemble_emit()
5232    Extend the bytecode with a new instruction.
5233    Update lnotab if necessary.
5234 */
5235 
5236 static int
assemble_emit(struct assembler * a,struct instr * i)5237 assemble_emit(struct assembler *a, struct instr *i)
5238 {
5239     int size, arg = 0;
5240     Py_ssize_t len = PyBytes_GET_SIZE(a->a_bytecode);
5241     _Py_CODEUNIT *code;
5242 
5243     arg = i->i_oparg;
5244     size = instrsize(arg);
5245     if (i->i_lineno && !assemble_lnotab(a, i))
5246         return 0;
5247     if (a->a_offset + size >= len / (int)sizeof(_Py_CODEUNIT)) {
5248         if (len > PY_SSIZE_T_MAX / 2)
5249             return 0;
5250         if (_PyBytes_Resize(&a->a_bytecode, len * 2) < 0)
5251             return 0;
5252     }
5253     code = (_Py_CODEUNIT *)PyBytes_AS_STRING(a->a_bytecode) + a->a_offset;
5254     a->a_offset += size;
5255     write_op_arg(code, i->i_opcode, arg, size);
5256     return 1;
5257 }
5258 
5259 static void
assemble_jump_offsets(struct assembler * a,struct compiler * c)5260 assemble_jump_offsets(struct assembler *a, struct compiler *c)
5261 {
5262     basicblock *b;
5263     int bsize, totsize, extended_arg_recompile;
5264     int i;
5265 
5266     /* Compute the size of each block and fixup jump args.
5267        Replace block pointer with position in bytecode. */
5268     do {
5269         totsize = 0;
5270         for (i = a->a_nblocks - 1; i >= 0; i--) {
5271             b = a->a_postorder[i];
5272             bsize = blocksize(b);
5273             b->b_offset = totsize;
5274             totsize += bsize;
5275         }
5276         extended_arg_recompile = 0;
5277         for (b = c->u->u_blocks; b != NULL; b = b->b_list) {
5278             bsize = b->b_offset;
5279             for (i = 0; i < b->b_iused; i++) {
5280                 struct instr *instr = &b->b_instr[i];
5281                 int isize = instrsize(instr->i_oparg);
5282                 /* Relative jumps are computed relative to
5283                    the instruction pointer after fetching
5284                    the jump instruction.
5285                 */
5286                 bsize += isize;
5287                 if (instr->i_jabs || instr->i_jrel) {
5288                     instr->i_oparg = instr->i_target->b_offset;
5289                     if (instr->i_jrel) {
5290                         instr->i_oparg -= bsize;
5291                     }
5292                     instr->i_oparg *= sizeof(_Py_CODEUNIT);
5293                     if (instrsize(instr->i_oparg) != isize) {
5294                         extended_arg_recompile = 1;
5295                     }
5296                 }
5297             }
5298         }
5299 
5300     /* XXX: This is an awful hack that could hurt performance, but
5301         on the bright side it should work until we come up
5302         with a better solution.
5303 
5304         The issue is that in the first loop blocksize() is called
5305         which calls instrsize() which requires i_oparg be set
5306         appropriately. There is a bootstrap problem because
5307         i_oparg is calculated in the second loop above.
5308 
5309         So we loop until we stop seeing new EXTENDED_ARGs.
5310         The only EXTENDED_ARGs that could be popping up are
5311         ones in jump instructions.  So this should converge
5312         fairly quickly.
5313     */
5314     } while (extended_arg_recompile);
5315 }
5316 
5317 static PyObject *
dict_keys_inorder(PyObject * dict,Py_ssize_t offset)5318 dict_keys_inorder(PyObject *dict, Py_ssize_t offset)
5319 {
5320     PyObject *tuple, *k, *v;
5321     Py_ssize_t i, pos = 0, size = PyDict_GET_SIZE(dict);
5322 
5323     tuple = PyTuple_New(size);
5324     if (tuple == NULL)
5325         return NULL;
5326     while (PyDict_Next(dict, &pos, &k, &v)) {
5327         i = PyLong_AS_LONG(v);
5328         /* The keys of the dictionary are tuples. (see compiler_add_o
5329          * and _PyCode_ConstantKey). The object we want is always second,
5330          * though. */
5331         k = PyTuple_GET_ITEM(k, 1);
5332         Py_INCREF(k);
5333         assert((i - offset) < size);
5334         assert((i - offset) >= 0);
5335         PyTuple_SET_ITEM(tuple, i - offset, k);
5336     }
5337     return tuple;
5338 }
5339 
5340 static int
compute_code_flags(struct compiler * c)5341 compute_code_flags(struct compiler *c)
5342 {
5343     PySTEntryObject *ste = c->u->u_ste;
5344     int flags = 0;
5345     if (ste->ste_type == FunctionBlock) {
5346         flags |= CO_NEWLOCALS | CO_OPTIMIZED;
5347         if (ste->ste_nested)
5348             flags |= CO_NESTED;
5349         if (ste->ste_generator && !ste->ste_coroutine)
5350             flags |= CO_GENERATOR;
5351         if (!ste->ste_generator && ste->ste_coroutine)
5352             flags |= CO_COROUTINE;
5353         if (ste->ste_generator && ste->ste_coroutine)
5354             flags |= CO_ASYNC_GENERATOR;
5355         if (ste->ste_varargs)
5356             flags |= CO_VARARGS;
5357         if (ste->ste_varkeywords)
5358             flags |= CO_VARKEYWORDS;
5359     }
5360 
5361     /* (Only) inherit compilerflags in PyCF_MASK */
5362     flags |= (c->c_flags->cf_flags & PyCF_MASK);
5363 
5364     return flags;
5365 }
5366 
5367 static PyCodeObject *
makecode(struct compiler * c,struct assembler * a)5368 makecode(struct compiler *c, struct assembler *a)
5369 {
5370     PyObject *tmp;
5371     PyCodeObject *co = NULL;
5372     PyObject *consts = NULL;
5373     PyObject *names = NULL;
5374     PyObject *varnames = NULL;
5375     PyObject *name = NULL;
5376     PyObject *freevars = NULL;
5377     PyObject *cellvars = NULL;
5378     PyObject *bytecode = NULL;
5379     Py_ssize_t nlocals;
5380     int nlocals_int;
5381     int flags;
5382     int argcount, kwonlyargcount, maxdepth;
5383 
5384     tmp = dict_keys_inorder(c->u->u_consts, 0);
5385     if (!tmp)
5386         goto error;
5387     consts = PySequence_List(tmp); /* optimize_code requires a list */
5388     Py_DECREF(tmp);
5389 
5390     names = dict_keys_inorder(c->u->u_names, 0);
5391     varnames = dict_keys_inorder(c->u->u_varnames, 0);
5392     if (!consts || !names || !varnames)
5393         goto error;
5394 
5395     cellvars = dict_keys_inorder(c->u->u_cellvars, 0);
5396     if (!cellvars)
5397         goto error;
5398     freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars));
5399     if (!freevars)
5400         goto error;
5401 
5402     nlocals = PyDict_GET_SIZE(c->u->u_varnames);
5403     assert(nlocals < INT_MAX);
5404     nlocals_int = Py_SAFE_DOWNCAST(nlocals, Py_ssize_t, int);
5405 
5406     flags = compute_code_flags(c);
5407     if (flags < 0)
5408         goto error;
5409 
5410     bytecode = PyCode_Optimize(a->a_bytecode, consts, names, a->a_lnotab);
5411     if (!bytecode)
5412         goto error;
5413 
5414     tmp = PyList_AsTuple(consts); /* PyCode_New requires a tuple */
5415     if (!tmp)
5416         goto error;
5417     Py_DECREF(consts);
5418     consts = tmp;
5419 
5420     argcount = Py_SAFE_DOWNCAST(c->u->u_argcount, Py_ssize_t, int);
5421     kwonlyargcount = Py_SAFE_DOWNCAST(c->u->u_kwonlyargcount, Py_ssize_t, int);
5422     maxdepth = stackdepth(c);
5423     if (maxdepth < 0) {
5424         goto error;
5425     }
5426     co = PyCode_New(argcount, kwonlyargcount,
5427                     nlocals_int, maxdepth, flags,
5428                     bytecode, consts, names, varnames,
5429                     freevars, cellvars,
5430                     c->c_filename, c->u->u_name,
5431                     c->u->u_firstlineno,
5432                     a->a_lnotab);
5433  error:
5434     Py_XDECREF(consts);
5435     Py_XDECREF(names);
5436     Py_XDECREF(varnames);
5437     Py_XDECREF(name);
5438     Py_XDECREF(freevars);
5439     Py_XDECREF(cellvars);
5440     Py_XDECREF(bytecode);
5441     return co;
5442 }
5443 
5444 
5445 /* For debugging purposes only */
5446 #if 0
5447 static void
5448 dump_instr(const struct instr *i)
5449 {
5450     const char *jrel = i->i_jrel ? "jrel " : "";
5451     const char *jabs = i->i_jabs ? "jabs " : "";
5452     char arg[128];
5453 
5454     *arg = '\0';
5455     if (HAS_ARG(i->i_opcode)) {
5456         sprintf(arg, "arg: %d ", i->i_oparg);
5457     }
5458     fprintf(stderr, "line: %d, opcode: %d %s%s%s\n",
5459                     i->i_lineno, i->i_opcode, arg, jabs, jrel);
5460 }
5461 
5462 static void
5463 dump_basicblock(const basicblock *b)
5464 {
5465     const char *seen = b->b_seen ? "seen " : "";
5466     const char *b_return = b->b_return ? "return " : "";
5467     fprintf(stderr, "used: %d, depth: %d, offset: %d %s%s\n",
5468         b->b_iused, b->b_startdepth, b->b_offset, seen, b_return);
5469     if (b->b_instr) {
5470         int i;
5471         for (i = 0; i < b->b_iused; i++) {
5472             fprintf(stderr, "  [%02d] ", i);
5473             dump_instr(b->b_instr + i);
5474         }
5475     }
5476 }
5477 #endif
5478 
5479 static PyCodeObject *
assemble(struct compiler * c,int addNone)5480 assemble(struct compiler *c, int addNone)
5481 {
5482     basicblock *b, *entryblock;
5483     struct assembler a;
5484     int i, j, nblocks;
5485     PyCodeObject *co = NULL;
5486 
5487     /* Make sure every block that falls off the end returns None.
5488        XXX NEXT_BLOCK() isn't quite right, because if the last
5489        block ends with a jump or return b_next shouldn't set.
5490      */
5491     if (!c->u->u_curblock->b_return) {
5492         NEXT_BLOCK(c);
5493         if (addNone)
5494             ADDOP_O(c, LOAD_CONST, Py_None, consts);
5495         ADDOP(c, RETURN_VALUE);
5496     }
5497 
5498     nblocks = 0;
5499     entryblock = NULL;
5500     for (b = c->u->u_blocks; b != NULL; b = b->b_list) {
5501         nblocks++;
5502         entryblock = b;
5503     }
5504 
5505     /* Set firstlineno if it wasn't explicitly set. */
5506     if (!c->u->u_firstlineno) {
5507         if (entryblock && entryblock->b_instr && entryblock->b_instr->i_lineno)
5508             c->u->u_firstlineno = entryblock->b_instr->i_lineno;
5509         else
5510             c->u->u_firstlineno = 1;
5511     }
5512     if (!assemble_init(&a, nblocks, c->u->u_firstlineno))
5513         goto error;
5514     dfs(c, entryblock, &a, nblocks);
5515 
5516     /* Can't modify the bytecode after computing jump offsets. */
5517     assemble_jump_offsets(&a, c);
5518 
5519     /* Emit code in reverse postorder from dfs. */
5520     for (i = a.a_nblocks - 1; i >= 0; i--) {
5521         b = a.a_postorder[i];
5522         for (j = 0; j < b->b_iused; j++)
5523             if (!assemble_emit(&a, &b->b_instr[j]))
5524                 goto error;
5525     }
5526 
5527     if (_PyBytes_Resize(&a.a_lnotab, a.a_lnotab_off) < 0)
5528         goto error;
5529     if (_PyBytes_Resize(&a.a_bytecode, a.a_offset * sizeof(_Py_CODEUNIT)) < 0)
5530         goto error;
5531 
5532     co = makecode(c, &a);
5533  error:
5534     assemble_free(&a);
5535     return co;
5536 }
5537 
5538 #undef PyAST_Compile
5539 PyAPI_FUNC(PyCodeObject *)
PyAST_Compile(mod_ty mod,const char * filename,PyCompilerFlags * flags,PyArena * arena)5540 PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags,
5541               PyArena *arena)
5542 {
5543     return PyAST_CompileEx(mod, filename, flags, -1, arena);
5544 }
5545